diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 01ed43cdc0..98ccac2f77 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 54.9.1 +current_version = 55.2.0 commit = True tag = True tag_name = v{new_version} diff --git a/alembic/versions/2023_12_27_584840c706a0_remove_uploaded_to_vogue_at.py b/alembic/versions/2023_12_27_584840c706a0_remove_uploaded_to_vogue_at.py new file mode 100644 index 0000000000..1e8207c42b --- /dev/null +++ b/alembic/versions/2023_12_27_584840c706a0_remove_uploaded_to_vogue_at.py @@ -0,0 +1,24 @@ +"""remove_uploaded_to_vogue_at + +Revision ID: 584840c706a0 +Revises: 27ec5c4c0380 +Create Date: 2023-12-27 11:50:22.278213 + +""" +import sqlalchemy as sa + +from alembic import op + +# revision identifiers, used by Alembic. +revision = "584840c706a0" +down_revision = "27ec5c4c0380" +branch_labels = None +depends_on = None + + +def upgrade(): + op.drop_column("analysis", "uploaded_to_vogue_at") + + +def downgrade(): + op.add_column("analysis", sa.Column("uploaded_to_vogue_at", sa.DateTime(), nullable=True)) diff --git a/cg/__init__.py b/cg/__init__.py index 9926266232..1caaab50d0 100644 --- a/cg/__init__.py +++ b/cg/__init__.py @@ -1,2 +1,2 @@ __title__ = "cg" -__version__ = "54.9.1" +__version__ = "55.2.0" diff --git a/cg/apps/crunchy/crunchy.py b/cg/apps/crunchy/crunchy.py index 406515e0be..d0c2c10e04 100644 --- a/cg/apps/crunchy/crunchy.py +++ b/cg/apps/crunchy/crunchy.py @@ -64,25 +64,24 @@ def is_compression_pending(compression_obj: CompressionData) -> bool: def is_fastq_compression_possible(compression_obj: CompressionData) -> bool: """Check if FASTQ compression is possible. - There are three possible answers to this question: - - - Compression is running -> Compression NOT possible - - SPRING archive exists -> Compression NOT possible - - Data is external -> Compression NOT possible - - Not compressed and not running -> Compression IS possible + - Compression is running -> Compression NOT possible + - SPRING file exists on Hasta -> Compression NOT possible + - Data is external -> Compression NOT possible + - Not compressed and + not running -> Compression IS possible """ if CrunchyAPI.is_compression_pending(compression_obj): return False if compression_obj.spring_exists(): - LOG.info("SPRING file found") + LOG.debug("SPRING file found") return False if "external-data" in str(compression_obj.fastq_first): - LOG.info("File is external data and should not be compressed") + LOG.debug("File is external data and should not be compressed") return False - LOG.info("FASTQ compression is possible") + LOG.debug("FASTQ compression is possible") return True diff --git a/cg/apps/lims/api.py b/cg/apps/lims/api.py index 4e02127cc5..caacc6e58d 100644 --- a/cg/apps/lims/api.py +++ b/cg/apps/lims/api.py @@ -90,6 +90,7 @@ def _export_sample(self, lims_sample): else None ), "comment": udfs.get("comment"), + "concentration_ng_ul": udfs.get("Concentration (ng/ul)"), } def get_received_date(self, lims_id: str) -> dt.date: diff --git a/cg/apps/tb/api.py b/cg/apps/tb/api.py index b4c836b2eb..107051809c 100644 --- a/cg/apps/tb/api.py +++ b/cg/apps/tb/api.py @@ -91,20 +91,6 @@ def is_latest_analysis_completed(self, case_id: str) -> bool: def is_latest_analysis_qc(self, case_id: str) -> bool: return self.get_latest_analysis_status(case_id=case_id) == AnalysisStatus.QC - def mark_analyses_deleted(self, case_id: str) -> list | None: - """Mark all analyses for case deleted without removing analysis files""" - request_body = { - "case_id": case_id, - } - response = self.query_trailblazer( - command="mark-analyses-deleted", request_body=request_body - ) - if response: - if isinstance(response, list): - return [TrailblazerAnalysis.model_validate(analysis) for analysis in response] - if isinstance(response, dict): - return [TrailblazerAnalysis.model_validate(response)] - def add_pending_analysis( self, case_id: str, diff --git a/cg/cli/demultiplex/demux.py b/cg/cli/demultiplex/demux.py index 47adf1bd95..3a03e0e604 100644 --- a/cg/cli/demultiplex/demux.py +++ b/cg/cli/demultiplex/demux.py @@ -59,7 +59,8 @@ def demultiplex_all(context: CGConfig, flow_cells_directory: click.Path, dry_run if not flow_cell.validate_sample_sheet(): LOG.warning( - f"Malformed sample sheet. Run cg demultiplex samplesheet validate {flow_cell.sample_sheet_path}", + "Malformed sample sheet. " + f"Run cg demultiplex sample sheet validate {flow_cell.sample_sheet_path}", ) continue diff --git a/cg/constants/gene_panel.py b/cg/constants/gene_panel.py index 854ffc9d5c..889650966b 100644 --- a/cg/constants/gene_panel.py +++ b/cg/constants/gene_panel.py @@ -32,6 +32,7 @@ class GenePanelMasterList(StrEnum): NEURODEG: str = "NEURODEG" NMD: str = "NMD" OMIM_AUTO: str = "OMIM-AUTO" + OPTIC: str = "OPTIC" PANELAPP_GREEN: str = "PANELAPP-GREEN" PEDHEP: str = "PEDHEP" PID: str = "PID" diff --git a/cg/constants/lims.py b/cg/constants/lims.py index 90b1ce5e6f..86671b2fac 100644 --- a/cg/constants/lims.py +++ b/cg/constants/lims.py @@ -12,6 +12,7 @@ "comment": "Comment", "control": "Control", "concentration": "Concentration (nM)", + "concentration_ng_ul": "Concentration (ng/ul)", "concentration_sample": "Sample Conc.", "customer": "customer", "data_analysis": "Data Analysis", diff --git a/cg/constants/scout.py b/cg/constants/scout.py index ef2bec36b9..39f22145a4 100644 --- a/cg/constants/scout.py +++ b/cg/constants/scout.py @@ -14,6 +14,11 @@ class ScoutExportFileName(StrEnum): PANELS: str = f"gene_panels{FileExtensions.BED}" +class UploadTrack(StrEnum): + RARE_DISEASE: str = "rare" + CANCER: str = "cancer" + + class ScoutCustomCaseReportTags(StrEnum): DELIVERY: str = "delivery_report" CNV: str = "cnv_report" diff --git a/cg/meta/archive/archive.py b/cg/meta/archive/archive.py index 030369b488..39dc513a84 100644 --- a/cg/meta/archive/archive.py +++ b/cg/meta/archive/archive.py @@ -40,11 +40,11 @@ def __init__( self.status_db: Store = status_db self.data_flow_config: DataFlowConfig = data_flow_config - def archive_files_to_location( - self, files_and_samples: list[FileAndSample], archive_location: ArchiveLocations + def archive_file_to_location( + self, file_and_sample: FileAndSample, archive_location: ArchiveLocations ) -> int: archive_handler: ArchiveHandler = ARCHIVE_HANDLERS[archive_location](self.data_flow_config) - return archive_handler.archive_files(files_and_samples=files_and_samples) + return archive_handler.archive_file(file_and_sample=file_and_sample) def archive_spring_files_and_add_archives_to_housekeeper( self, spring_file_count_limit: int | None @@ -55,25 +55,37 @@ def archive_spring_files_and_add_archives_to_housekeeper( LOG.warning("Please do not provide a non-positive integer as limit - exiting.") return for archive_location in ArchiveLocations: - files_to_archive: list[File] = self.housekeeper_api.get_non_archived_spring_files( - tags=[archive_location], - limit=spring_file_count_limit, + self.archive_files_to_location( + archive_location=archive_location, file_limit=spring_file_count_limit ) - if files_to_archive: - files_and_samples_for_location = self.add_samples_to_files(files_to_archive) - job_id = self.archive_files_to_location( - files_and_samples=files_and_samples_for_location, - archive_location=archive_location, - ) - LOG.info(f"Files submitted to {archive_location} with archival task id {job_id}.") - self.housekeeper_api.add_archives( - files=[ - file_and_sample.file for file_and_sample in files_and_samples_for_location - ], - archive_task_id=job_id, + + def archive_files_to_location(self, archive_location: str, file_limit: int | None) -> None: + """Archives up to spring file count limit number of files to the provided archive location.""" + files_to_archive: list[File] = self.housekeeper_api.get_non_archived_spring_files( + tags=[archive_location], + limit=file_limit, + ) + if files_to_archive: + files_and_samples_for_location = self.add_samples_to_files(files_to_archive) + for file_and_sample in files_and_samples_for_location: + self.archive_file( + file_and_sample=file_and_sample, archive_location=archive_location ) - else: - LOG.info(f"No files to archive for location {archive_location}.") + + else: + LOG.info(f"No files to archive for location {archive_location}.") + + def archive_file( + self, file_and_sample: FileAndSample, archive_location: ArchiveLocations + ) -> None: + job_id: int = self.archive_file_to_location( + file_and_sample=file_and_sample, archive_location=archive_location + ) + LOG.info(f"File submitted to {archive_location} with archival task id {job_id}.") + self.housekeeper_api.add_archives( + files=[file_and_sample.file], + archive_task_id=job_id, + ) def retrieve_case(self, case_id: str) -> None: """Submits jobs to retrieve any archived files belonging to the given case, and updates the Archive entries diff --git a/cg/meta/archive/ddn/constants.py b/cg/meta/archive/ddn/constants.py index 2267622286..d7c4b00bd4 100644 --- a/cg/meta/archive/ddn/constants.py +++ b/cg/meta/archive/ddn/constants.py @@ -49,3 +49,15 @@ class JobStatus(StrEnum): JobStatus.ON_VALIDATION, JobStatus.RUNNING, ] + +METADATA_LIST = "metadataList" + + +class MetadataFields(StrEnum): + CUSTOMER_NAME: str = "customer_name" + PREP_CATEGORY: str = "prep_category" + SAMPLE_NAME: str = "sample_name" + SEQUENCED_AT: str = "sequenced_at" + TICKET_NUMBER: str = "ticket_number" + NAME = "metadataName" + VALUE = "value" diff --git a/cg/meta/archive/ddn/ddn_data_flow_client.py b/cg/meta/archive/ddn/ddn_data_flow_client.py index dea5bc10eb..c8b5239028 100644 --- a/cg/meta/archive/ddn/ddn_data_flow_client.py +++ b/cg/meta/archive/ddn/ddn_data_flow_client.py @@ -27,6 +27,7 @@ RefreshPayload, TransferPayload, ) +from cg.meta.archive.ddn.utils import get_metadata from cg.meta.archive.models import ArchiveHandler, FileAndSample from cg.models.cg_config import DataFlowConfig @@ -100,14 +101,15 @@ def auth_header(self) -> dict[str, str]: self._refresh_auth_token() return {"Authorization": f"Bearer {self.auth_token}"} - def archive_files(self, files_and_samples: list[FileAndSample]) -> int: + def archive_file(self, file_and_sample: FileAndSample) -> int: """Archives all files provided, to their corresponding destination, as given by sources and destination in TransferData. Returns the job ID of the archiving task.""" miria_file_data: list[MiriaObject] = self.convert_into_transfer_data( - files_and_samples, is_archiving=True + [file_and_sample], is_archiving=True ) + metadata: list[dict] = get_metadata(file_and_sample.sample) archival_request: TransferPayload = self.create_transfer_request( - miria_file_data=miria_file_data, is_archiving_request=True + miria_file_data=miria_file_data, is_archiving_request=True, metadata=metadata ) return archival_request.post_request( headers=dict(self.headers, **self.auth_header), @@ -117,15 +119,9 @@ def archive_files(self, files_and_samples: list[FileAndSample]) -> int: def retrieve_files(self, files_and_samples: list[FileAndSample]) -> int: """Retrieves the provided files and stores them in the corresponding sample bundle in Housekeeper.""" - miria_file_data: list[MiriaObject] = [] - for file_and_sample in files_and_samples: - LOG.info( - f"Will retrieve file {file_and_sample.file.path} for sample {file_and_sample.sample.internal_id} via Miria." - ) - miria_object: MiriaObject = MiriaObject.create_from_file_and_sample( - file=file_and_sample.file, sample=file_and_sample.sample, is_archiving=False - ) - miria_file_data.append(miria_object) + miria_file_data: list[MiriaObject] = self.convert_into_transfer_data( + files_and_samples=files_and_samples, is_archiving=False + ) retrieval_request: TransferPayload = self.create_transfer_request( miria_file_data=miria_file_data, is_archiving_request=False ) @@ -135,7 +131,10 @@ def retrieve_files(self, files_and_samples: list[FileAndSample]) -> int: ).job_id def create_transfer_request( - self, miria_file_data: list[MiriaObject], is_archiving_request: bool + self, + miria_file_data: list[MiriaObject], + is_archiving_request: bool, + metadata: list[dict] = [], ) -> TransferPayload: """Performs the necessary curation of paths for the request to be valid, depending on if it is an archiving or a retrieve request. @@ -151,7 +150,9 @@ def create_transfer_request( ) transfer_request = TransferPayload( - files_to_transfer=miria_file_data, createFolder=is_archiving_request + files_to_transfer=miria_file_data, + createFolder=is_archiving_request, + metadataList=metadata, ) transfer_request.trim_paths(attribute_to_trim=attribute) transfer_request.add_repositories( diff --git a/cg/meta/archive/ddn/models.py b/cg/meta/archive/ddn/models.py index c7d07b77d7..ac413bcdcc 100644 --- a/cg/meta/archive/ddn/models.py +++ b/cg/meta/archive/ddn/models.py @@ -21,7 +21,6 @@ def get_request_log(headers: dict, body: dict): class MiriaObject(FileTransferData): """Model for representing a singular object transfer.""" - _metadata = None destination: str source: str @@ -58,6 +57,7 @@ class TransferPayload(BaseModel): osType: str = OSTYPE createFolder: bool = True settings: list[dict] = [] + metadataList: list[dict] = [] def trim_paths(self, attribute_to_trim: str): """Trims the source path from its root directory for all objects in the transfer.""" @@ -76,7 +76,6 @@ def model_dump(self, **kwargs) -> dict: """Creates a correctly structured dict to be used as the request payload.""" payload: dict = super().model_dump(exclude={"files_to_transfer"}) payload["pathInfo"] = [miria_file.model_dump() for miria_file in self.files_to_transfer] - payload["metadataList"] = [] return payload def post_request(self, url: str, headers: dict) -> "TransferResponse": diff --git a/cg/meta/archive/ddn/utils.py b/cg/meta/archive/ddn/utils.py new file mode 100644 index 0000000000..e820e5c2b0 --- /dev/null +++ b/cg/meta/archive/ddn/utils.py @@ -0,0 +1,29 @@ +from cg.meta.archive.ddn.constants import MetadataFields +from cg.store.models import Sample + + +def get_metadata(sample: Sample) -> list[dict]: + """Returns metadata generated from a sample.""" + metadata: list[dict] = [ + { + MetadataFields.NAME.value: MetadataFields.CUSTOMER_NAME.value, + MetadataFields.VALUE.value: sample.customer.name, + }, + { + MetadataFields.NAME.value: MetadataFields.PREP_CATEGORY.value, + MetadataFields.VALUE.value: sample.prep_category, + }, + { + MetadataFields.NAME.value: MetadataFields.SAMPLE_NAME.value, + MetadataFields.VALUE.value: sample.name, + }, + { + MetadataFields.NAME.value: MetadataFields.SEQUENCED_AT.value, + MetadataFields.VALUE.value: sample.last_sequenced_at, + }, + { + MetadataFields.NAME.value: MetadataFields.TICKET_NUMBER.value, + MetadataFields.VALUE.value: sample.original_ticket, + }, + ] + return metadata diff --git a/cg/meta/archive/models.py b/cg/meta/archive/models.py index 0bf46e9c30..0b2131a30e 100644 --- a/cg/meta/archive/models.py +++ b/cg/meta/archive/models.py @@ -35,7 +35,7 @@ def __init__(self, config: DataFlowConfig): pass @abstractmethod - def archive_files(self, files_and_samples: list[FileAndSample]): + def archive_file(self, file_and_sample: FileAndSample): """Archives all folders provided, to their corresponding destination, as given by sources and destination parameter.""" pass diff --git a/cg/meta/compress/compress.py b/cg/meta/compress/compress.py index c2674b9037..f941479e2c 100644 --- a/cg/meta/compress/compress.py +++ b/cg/meta/compress/compress.py @@ -14,7 +14,7 @@ from cg.constants import SequencingFileTag from cg.meta.backup.backup import SpringBackupAPI from cg.meta.compress import files -from cg.models import CompressionData, FileData +from cg.models import CompressionData from cg.store.models import Sample LOG = logging.getLogger(__name__) @@ -73,16 +73,11 @@ def compress_fastq(self, sample_id: str) -> bool: for run_name in sample_fastq: LOG.info(f"Check if compression possible for run {run_name}") compression: CompressionData = sample_fastq[run_name]["compression_data"] - if FileData.is_empty(compression.fastq_first): - LOG.warning(f"Fastq files are empty for {sample_id}: {compression.fastq_first}") - self.delete_fastq_housekeeper( - hk_fastq_first=sample_fastq[run_name]["hk_first"], - hk_fastq_second=sample_fastq[run_name]["hk_second"], - ) - all_ok = False - continue - - if not self.crunchy_api.is_fastq_compression_possible(compression_obj=compression): + is_compression_possible: bool = self._is_fastq_compression_possible( + compression=compression, + sample_id=sample_id, + ) + if not is_compression_possible: LOG.warning(f"FASTQ to SPRING not possible for {sample_id}, run {run_name}") all_ok = False continue @@ -93,6 +88,20 @@ def compress_fastq(self, sample_id: str) -> bool: self.crunchy_api.fastq_to_spring(compression_obj=compression, sample_id=sample_id) return all_ok + def _is_fastq_compression_possible(self, compression: CompressionData, sample_id: str) -> bool: + if self._is_spring_archived(compression): + LOG.debug(f"Found archived Spring file for {sample_id} - compression not possible") + return False + return self.crunchy_api.is_fastq_compression_possible(compression_obj=compression) + + def _is_spring_archived(self, compression_data: CompressionData) -> bool: + spring_file: File | None = self.hk_api.get_file_insensitive_path( + path=compression_data.spring_path + ) + if (not spring_file) or (not spring_file.archive): + return False + return bool(spring_file.archive.archived_at) + def decompress_spring(self, sample_id: str) -> bool: """Decompress SPRING archive for a sample. diff --git a/cg/meta/orders/case_submitter.py b/cg/meta/orders/case_submitter.py index 3b43dd4963..551edf7201 100644 --- a/cg/meta/orders/case_submitter.py +++ b/cg/meta/orders/case_submitter.py @@ -9,7 +9,7 @@ from cg.meta.orders.submitter import Submitter from cg.models.orders.order import OrderIn from cg.models.orders.samples import Of1508Sample, OrderInSample -from cg.store.models import ApplicationVersion, Customer, Case, CaseSample, Sample +from cg.store.models import ApplicationVersion, Case, CaseSample, Customer, Sample LOG = logging.getLogger(__name__) diff --git a/cg/meta/report/balsamic.py b/cg/meta/report/balsamic.py index 25b496c512..e88f531b1f 100644 --- a/cg/meta/report/balsamic.py +++ b/cg/meta/report/balsamic.py @@ -156,10 +156,10 @@ def get_variant_caller_version(var_caller_name: str, var_caller_versions: dict) return versions[0] return None - def get_report_accreditation( + def is_report_accredited( self, samples: list[SampleModel], analysis_metadata: BalsamicAnalysis ) -> bool: - """Checks if the report is accredited or not.""" + """Check if the Balsamic report is accredited.""" if analysis_metadata.config.analysis.sequencing_type == "targeted" and next( ( panel diff --git a/cg/meta/report/mip_dna.py b/cg/meta/report/mip_dna.py index a3763e420c..215daf9b48 100644 --- a/cg/meta/report/mip_dna.py +++ b/cg/meta/report/mip_dna.py @@ -79,10 +79,10 @@ def get_genome_build(self, analysis_metadata: MipAnalysis) -> str: """Return build version of the genome reference of a specific case.""" return analysis_metadata.genome_build - def get_report_accreditation( + def is_report_accredited( self, samples: list[SampleModel], analysis_metadata: MipAnalysis = None ) -> bool: - """Checks if the report is accredited or not by evaluating each of the sample process accreditations.""" + """Check if the MIP-DNA report is accredited by evaluating each of the sample process accreditations.""" for sample in samples: if not sample.application.accredited: return False diff --git a/cg/meta/report/report_api.py b/cg/meta/report/report_api.py index 9ca2057b10..974c282625 100644 --- a/cg/meta/report/report_api.py +++ b/cg/meta/report/report_api.py @@ -174,7 +174,7 @@ def get_report_data(self, case_id: str, analysis_date: datetime) -> ReportModel: version=self.get_report_version(analysis=analysis), date=datetime.today(), case=case_model, - accredited=self.get_report_accreditation( + accredited=self.is_report_accredited( samples=case_model.samples, analysis_metadata=analysis_metadata ), ) @@ -407,10 +407,10 @@ def get_variant_callers(self, _analysis_metadata: AnalysisModel) -> list: """Return list of variant-calling filters used during analysis.""" return [] - def get_report_accreditation( + def is_report_accredited( self, samples: list[SampleModel], analysis_metadata: AnalysisModel ) -> bool: - """Checks if the report is accredited or not.""" + """Check if the report is accredited.""" raise NotImplementedError def get_required_fields(self, case: CaseModel) -> dict: diff --git a/cg/meta/report/rnafusion.py b/cg/meta/report/rnafusion.py index 91a86e700b..4f2eeaaa99 100644 --- a/cg/meta/report/rnafusion.py +++ b/cg/meta/report/rnafusion.py @@ -79,10 +79,10 @@ def get_genome_build(self, analysis_metadata: AnalysisModel) -> str: """Return build version of the genome reference of a specific case.""" return GenomeVersion.hg38.value - def get_report_accreditation( + def is_report_accredited( self, samples: list[SampleModel], analysis_metadata: AnalysisModel ) -> bool: - """Checks if the report is accredited or not. Rnafusion is an accredited workflow.""" + """Check if the report is accredited. Rnafusion is an accredited workflow.""" return True def get_template_name(self) -> str: diff --git a/cg/meta/upload/scout/balsamic_config_builder.py b/cg/meta/upload/scout/balsamic_config_builder.py index b85e25ff5e..9906292db8 100644 --- a/cg/meta/upload/scout/balsamic_config_builder.py +++ b/cg/meta/upload/scout/balsamic_config_builder.py @@ -4,7 +4,8 @@ from cg.apps.lims import LimsAPI from cg.constants.constants import SampleType -from cg.constants.scout import BALSAMIC_CASE_TAGS, BALSAMIC_SAMPLE_TAGS +from cg.constants.housekeeper_tags import HK_DELIVERY_REPORT_TAG +from cg.constants.scout import BALSAMIC_CASE_TAGS, BALSAMIC_SAMPLE_TAGS, UploadTrack from cg.constants.subject import PhenotypeStatus from cg.meta.upload.scout.hk_tags import CaseTags, SampleTags from cg.meta.upload.scout.scout_config_builder import ScoutConfigBuilder @@ -22,7 +23,10 @@ def __init__(self, hk_version_obj: Version, analysis_obj: Analysis, lims_api: Li ) self.case_tags: CaseTags = CaseTags(**BALSAMIC_CASE_TAGS) self.sample_tags: SampleTags = SampleTags(**BALSAMIC_SAMPLE_TAGS) - self.load_config: BalsamicLoadConfig = BalsamicLoadConfig(track="cancer") + self.load_config: BalsamicLoadConfig = BalsamicLoadConfig( + track=UploadTrack.CANCER.value, + delivery_report=self.get_file_from_hk({HK_DELIVERY_REPORT_TAG}), + ) def include_case_files(self): LOG.info("Including BALSAMIC specific case level files") @@ -34,7 +38,6 @@ def include_case_files(self): ) self.include_cnv_report() self.include_multiqc_report() - self.include_delivery_report() def include_sample_files(self, config_sample: ScoutCancerIndividual) -> None: LOG.info("Including BALSAMIC specific sample level files.") diff --git a/cg/meta/upload/scout/balsamic_umi_config_builder.py b/cg/meta/upload/scout/balsamic_umi_config_builder.py index a816eede6f..b288417330 100644 --- a/cg/meta/upload/scout/balsamic_umi_config_builder.py +++ b/cg/meta/upload/scout/balsamic_umi_config_builder.py @@ -3,13 +3,11 @@ from housekeeper.store.models import Version from cg.apps.lims import LimsAPI -from cg.constants.scout import BALSAMIC_UMI_CASE_TAGS, BALSAMIC_UMI_SAMPLE_TAGS +from cg.constants.housekeeper_tags import HK_DELIVERY_REPORT_TAG +from cg.constants.scout import BALSAMIC_UMI_CASE_TAGS, BALSAMIC_UMI_SAMPLE_TAGS, UploadTrack from cg.meta.upload.scout.balsamic_config_builder import BalsamicConfigBuilder from cg.meta.upload.scout.hk_tags import CaseTags, SampleTags -from cg.models.scout.scout_load_config import ( - BalsamicUmiLoadConfig, - ScoutCancerIndividual, -) +from cg.models.scout.scout_load_config import BalsamicUmiLoadConfig, ScoutCancerIndividual from cg.store.models import Analysis, Sample LOG = logging.getLogger(__name__) @@ -22,12 +20,14 @@ def __init__(self, hk_version_obj: Version, analysis_obj: Analysis, lims_api: Li ) self.case_tags: CaseTags = CaseTags(**BALSAMIC_UMI_CASE_TAGS) self.sample_tags: SampleTags = SampleTags(**BALSAMIC_UMI_SAMPLE_TAGS) - self.load_config: BalsamicUmiLoadConfig = BalsamicUmiLoadConfig(track="cancer") + self.load_config: BalsamicUmiLoadConfig = BalsamicUmiLoadConfig( + track=UploadTrack.CANCER.value, + delivery_report=self.get_file_from_hk({HK_DELIVERY_REPORT_TAG}), + ) def include_sample_files(self, config_sample: ScoutCancerIndividual) -> None: LOG.info("Including BALSAMIC specific sample level files") def get_balsamic_analysis_type(self, sample: Sample) -> str: - """Returns a formatted balsamic analysis type""" - + """Returns a formatted balsamic analysis type.""" return "panel-umi" diff --git a/cg/meta/upload/scout/mip_config_builder.py b/cg/meta/upload/scout/mip_config_builder.py index 071abaee25..1e0d9bcaaa 100644 --- a/cg/meta/upload/scout/mip_config_builder.py +++ b/cg/meta/upload/scout/mip_config_builder.py @@ -6,17 +6,14 @@ from cg.apps.lims import LimsAPI from cg.apps.madeline.api import MadelineAPI -from cg.constants.scout import MIP_CASE_TAGS, MIP_SAMPLE_TAGS +from cg.constants.housekeeper_tags import HK_DELIVERY_REPORT_TAG +from cg.constants.scout import MIP_CASE_TAGS, MIP_SAMPLE_TAGS, UploadTrack from cg.constants.subject import RelationshipStatus from cg.meta.upload.scout.hk_tags import CaseTags, SampleTags from cg.meta.upload.scout.scout_config_builder import ScoutConfigBuilder from cg.meta.workflow.mip import MipAnalysisAPI from cg.models.mip.mip_analysis import MipAnalysis -from cg.models.scout.scout_load_config import ( - MipLoadConfig, - ScoutLoadConfig, - ScoutMipIndividual, -) +from cg.models.scout.scout_load_config import MipLoadConfig, ScoutLoadConfig, ScoutMipIndividual from cg.store.models import Analysis, Case, CaseSample LOG = logging.getLogger(__name__) @@ -36,7 +33,10 @@ def __init__( ) self.case_tags: CaseTags = CaseTags(**MIP_CASE_TAGS) self.sample_tags: SampleTags = SampleTags(**MIP_SAMPLE_TAGS) - self.load_config: MipLoadConfig = MipLoadConfig(track="rare") + self.load_config: MipLoadConfig = MipLoadConfig( + track=UploadTrack.RARE_DISEASE.value, + delivery_report=self.get_file_from_hk({HK_DELIVERY_REPORT_TAG}), + ) self.mip_analysis_api: MipAnalysisAPI = mip_analysis_api self.lims_api: LimsAPI = lims_api self.madeline_api: MadelineAPI = madeline_api @@ -116,7 +116,6 @@ def include_case_files(self): self.load_config.vcf_sv = self.get_file_from_hk(self.case_tags.sv_vcf) self.load_config.vcf_sv_research = self.get_file_from_hk(self.case_tags.sv_research_vcf) self.include_multiqc_report() - self.include_delivery_report() def include_sample_files(self, config_sample: ScoutMipIndividual) -> None: """Include sample level files that are optional for mip samples""" diff --git a/cg/meta/upload/scout/rnafusion_config_builder.py b/cg/meta/upload/scout/rnafusion_config_builder.py index b536d5bec1..0bd369ea78 100644 --- a/cg/meta/upload/scout/rnafusion_config_builder.py +++ b/cg/meta/upload/scout/rnafusion_config_builder.py @@ -4,7 +4,8 @@ from cg.apps.lims import LimsAPI from cg.constants.constants import PrepCategory -from cg.constants.scout import RNAFUSION_CASE_TAGS, RNAFUSION_SAMPLE_TAGS, GenomeBuild +from cg.constants.housekeeper_tags import HK_DELIVERY_REPORT_TAG +from cg.constants.scout import RNAFUSION_CASE_TAGS, RNAFUSION_SAMPLE_TAGS, GenomeBuild, UploadTrack from cg.meta.upload.scout.hk_tags import CaseTags, SampleTags from cg.meta.upload.scout.scout_config_builder import ScoutConfigBuilder from cg.models.scout.scout_load_config import ( @@ -26,7 +27,10 @@ def __init__(self, hk_version_obj: Version, analysis_obj: Analysis, lims_api: Li ) self.case_tags: CaseTags = CaseTags(**RNAFUSION_CASE_TAGS) self.sample_tags: SampleTags = SampleTags(**RNAFUSION_SAMPLE_TAGS) - self.load_config: RnafusionLoadConfig = RnafusionLoadConfig(track="cancer") + self.load_config: RnafusionLoadConfig = RnafusionLoadConfig( + track=UploadTrack.CANCER.value, + delivery_report=self.get_file_from_hk({HK_DELIVERY_REPORT_TAG}), + ) def build_load_config(self) -> None: """Build a rnafusion-specific load config for uploading a case to scout.""" diff --git a/cg/meta/upload/scout/scout_config_builder.py b/cg/meta/upload/scout/scout_config_builder.py index 1b095889a4..a50d14c205 100644 --- a/cg/meta/upload/scout/scout_config_builder.py +++ b/cg/meta/upload/scout/scout_config_builder.py @@ -6,12 +6,14 @@ from cg.apps.housekeeper.hk import HousekeeperAPI from cg.apps.lims import LimsAPI +from cg.constants.housekeeper_tags import HK_DELIVERY_REPORT_TAG from cg.meta.upload.scout.hk_tags import CaseTags, SampleTags from cg.models.scout.scout_load_config import ScoutIndividual, ScoutLoadConfig from cg.store.models import Analysis, CaseSample, Sample LOG = logging.getLogger(__name__) + # Maps keys that are used in scout load config on tags that are used in scout @@ -24,7 +26,9 @@ def __init__(self, hk_version_obj: Version, analysis_obj: Analysis, lims_api: Li self.lims_api: LimsAPI = lims_api self.case_tags: CaseTags self.sample_tags: SampleTags - self.load_config: ScoutLoadConfig = ScoutLoadConfig() + self.load_config: ScoutLoadConfig = ScoutLoadConfig( + delivery_report=self.get_file_from_hk({HK_DELIVERY_REPORT_TAG}) + ) def add_common_info_to_load_config(self) -> None: """Add the mandatory common information to a scout load config object""" @@ -137,12 +141,6 @@ def include_multiqc_report(self) -> None: hk_tags=self.case_tags.multiqc_report, latest=True ) - def include_delivery_report(self) -> None: - LOG.info("Include delivery report to case") - self.load_config.delivery_report = self.get_file_from_hk( - hk_tags=self.case_tags.delivery_report, latest=True - ) - def include_sample_alignment_file(self, config_sample: ScoutIndividual) -> None: """Include the alignment file for a sample diff --git a/cg/meta/workflow/analysis.py b/cg/meta/workflow/analysis.py index cd82344249..84b10c0839 100644 --- a/cg/meta/workflow/analysis.py +++ b/cg/meta/workflow/analysis.py @@ -118,8 +118,8 @@ def get_case_config_path(self, case_id) -> Path: """Path to case config file""" raise NotImplementedError - def get_trailblazer_config_path(self, case_id: str) -> Path: - """Path to Trailblazer job id file""" + def get_job_ids_path(self, case_id: str) -> Path: + """Path to file containing slurm/tower job ids for the case.""" raise NotImplementedError def get_sample_name_from_lims_id(self, lims_id: str) -> str: @@ -206,17 +206,16 @@ def get_analysis_finish_path(self, case_id: str) -> Path: raise NotImplementedError def add_pending_trailblazer_analysis(self, case_id: str) -> None: - self.check_analysis_ongoing(case_id=case_id) - self.trailblazer_api.mark_analyses_deleted(case_id=case_id) + self.check_analysis_ongoing(case_id) self.trailblazer_api.add_pending_analysis( case_id=case_id, email=environ_email(), analysis_type=self.get_application_type( - self.status_db.get_case_by_internal_id(internal_id=case_id).links[0].sample + self.status_db.get_case_by_internal_id(case_id).links[0].sample ), - out_dir=self.get_trailblazer_config_path(case_id=case_id).parent.as_posix(), - config_path=self.get_trailblazer_config_path(case_id=case_id).as_posix(), - slurm_quality_of_service=self.get_slurm_qos_for_case(case_id=case_id), + out_dir=self.get_job_ids_path(case_id).parent.as_posix(), + config_path=self.get_job_ids_path(case_id).as_posix(), + slurm_quality_of_service=self.get_slurm_qos_for_case(case_id), data_analysis=str(self.pipeline), ticket=self.status_db.get_latest_ticket_from_case(case_id), workflow_manager=self.get_workflow_manager(), diff --git a/cg/meta/workflow/balsamic.py b/cg/meta/workflow/balsamic.py index 1bcb90e805..bf70942aed 100644 --- a/cg/meta/workflow/balsamic.py +++ b/cg/meta/workflow/balsamic.py @@ -117,7 +117,7 @@ def get_case_config_path(self, case_id: str) -> Path: """ return Path(self.root_dir, case_id, case_id + ".json") - def get_trailblazer_config_path(self, case_id: str) -> Path: + def get_job_ids_path(self, case_id: str) -> Path: return Path(self.root_dir, case_id, "analysis", "slurm_jobids.yaml") def get_bundle_deliverables_type(self, case_id: str) -> str: diff --git a/cg/meta/workflow/fluffy.py b/cg/meta/workflow/fluffy.py index f0c3ff310f..f805c32140 100644 --- a/cg/meta/workflow/fluffy.py +++ b/cg/meta/workflow/fluffy.py @@ -130,7 +130,7 @@ def get_deliverables_file_path(self, case_id: str) -> Path: """ return Path(self.get_output_path(case_id), "deliverables.yaml") - def get_trailblazer_config_path(self, case_id: str) -> Path: + def get_job_ids_path(self, case_id: str) -> Path: """ Location in working directory where SLURM job id file is to be stored. This file contains SLURM ID of jobs associated with current analysis , diff --git a/cg/meta/workflow/microsalt/microsalt.py b/cg/meta/workflow/microsalt/microsalt.py index bbcc1ed6d2..e670bed4e1 100644 --- a/cg/meta/workflow/microsalt/microsalt.py +++ b/cg/meta/workflow/microsalt/microsalt.py @@ -1,4 +1,3 @@ -import glob import logging import os import re @@ -88,14 +87,19 @@ def get_case_fastq_path(self, case_id: str) -> Path: def get_config_path(self, filename: str) -> Path: return Path(self.queries_path, filename).with_suffix(".json") - def get_trailblazer_config_path(self, case_id: str) -> Path: - """Get trailblazer config path.""" - case_obj: Case = self.status_db.get_case_by_internal_id(internal_id=case_id) - sample_obj: Sample = case_obj.links[0].sample - project_id: str = self.get_project(sample_obj.internal_id) - return Path( - self.root_dir, "results", "reports", "trailblazer", f"{project_id}_slurm_ids.yaml" - ) + def get_job_ids_path(self, case_id: str) -> Path: + case_path: Path = self.get_case_path(case_id) + job_ids_file_name: str = self.get_job_ids_file_name(case_id) + return Path(case_path, job_ids_file_name) + + def get_job_ids_file_name(self, case_id: str) -> str: + project_id: str = self.get_lims_project_id(case_id) + return f"{project_id}_slurm_ids.yaml" + + def get_lims_project_id(self, case_id: str): + case: Case = self.status_db.get_case_by_internal_id(case_id) + sample: Sample = case.links[0].sample + return self.get_project(sample.internal_id) def get_deliverables_file_path(self, case_id: str) -> Path: """Returns a path where the microSALT deliverables file for the order_id should be @@ -221,7 +225,7 @@ def get_case_id_from_ticket(self, unique_id: str) -> tuple[str, None]: Since sample_id is not specified, nothing is returned as sample_id""" case: Case = self.status_db.get_case_by_name(name=unique_id) if not case: - LOG.error("No case found for ticket number: %s", unique_id) + LOG.error(f"No case found for ticket number: {unique_id}") raise click.Abort case_id = case.internal_id return case_id, None @@ -230,9 +234,9 @@ def get_case_id_from_sample(self, unique_id: str) -> tuple[str, str]: """If sample is specified, finds the corresponding case_id to which this sample belongs. The case_id is to be used for identifying the appropriate path to link fastq files and store the analysis output """ - sample: Sample = self.status_db.get_sample_by_internal_id(internal_id=unique_id) + sample: Sample = self.status_db.get_sample_by_internal_id(unique_id) if not sample: - LOG.error("No sample found with id: %s", unique_id) + LOG.error(f"No sample found with id: {unique_id}") raise click.Abort case_id = sample.links[0].case.internal_id sample_id = sample.internal_id @@ -240,9 +244,9 @@ def get_case_id_from_sample(self, unique_id: str) -> tuple[str, str]: def get_case_id_from_case(self, unique_id: str) -> tuple[str, None]: """If case_id is specified, validates the presence of case_id in database and returns it""" - case_obj: Case = self.status_db.get_case_by_internal_id(internal_id=unique_id) + case_obj: Case = self.status_db.get_case_by_internal_id(unique_id) if not case_obj: - LOG.error("No case found with the id: %s", unique_id) + LOG.error(f"No case found with the id: {unique_id}") raise click.Abort case_id = case_obj.internal_id return case_id, None diff --git a/cg/meta/workflow/mip.py b/cg/meta/workflow/mip.py index e6dc4993dc..50aee67d54 100644 --- a/cg/meta/workflow/mip.py +++ b/cg/meta/workflow/mip.py @@ -312,7 +312,7 @@ def run_analysis(self, case_id: str, command_args: dict, dry_run: bool) -> None: def get_case_path(self, case_id: str) -> Path: return Path(self.root, case_id) - def get_trailblazer_config_path(self, case_id: str) -> Path: + def get_job_ids_path(self, case_id: str) -> Path: return Path(self.get_case_path(case_id=case_id), "analysis", "slurm_job_ids.yaml") def config_sample(self, link_obj: CaseSample, panel_bed: str) -> dict: diff --git a/cg/meta/workflow/mutant.py b/cg/meta/workflow/mutant.py index 7c99c30082..74ac98e14c 100644 --- a/cg/meta/workflow/mutant.py +++ b/cg/meta/workflow/mutant.py @@ -55,7 +55,7 @@ def get_case_output_path(self, case_id: str) -> Path: def get_case_fastq_dir(self, case_id: str) -> Path: return Path(self.get_case_path(case_id=case_id), "fastq") - def get_trailblazer_config_path(self, case_id: str) -> Path: + def get_job_ids_path(self, case_id: str) -> Path: return Path(self.get_case_output_path(case_id=case_id), "trailblazer_config.yaml") def _is_nanopore(self, application: Application) -> bool: diff --git a/cg/meta/workflow/nf_analysis.py b/cg/meta/workflow/nf_analysis.py index e8642d0e4a..644328b920 100644 --- a/cg/meta/workflow/nf_analysis.py +++ b/cg/meta/workflow/nf_analysis.py @@ -89,7 +89,7 @@ def get_nextflow_config_path(nextflow_config: str | None = None) -> Path | None: if nextflow_config: return Path(nextflow_config).absolute() - def get_trailblazer_config_path(self, case_id: str) -> Path: + def get_job_ids_path(self, case_id: str) -> Path: """Return the path to a Trailblazer config file containing Tower IDs.""" return Path(self.root_dir, case_id, "tower_ids").with_suffix(FileExtensions.YAML) @@ -197,7 +197,7 @@ def write_deliverables_file( def write_trailblazer_config(self, case_id: str, tower_id: str) -> None: """Write Tower IDs to a file used as the Trailblazer config.""" - config_path: Path = self.get_trailblazer_config_path(case_id=case_id) + config_path: Path = self.get_job_ids_path(case_id=case_id) LOG.info(f"Writing Tower ID to {config_path.as_posix()}") WriteFile.write_file_from_content( content={case_id: [tower_id]}, @@ -246,7 +246,7 @@ def _run_analysis_with_tower( if command_args.resume: from_tower_id: int = command_args.id or NfTowerHandler.get_last_tower_id( case_id=case_id, - trailblazer_config=self.get_trailblazer_config_path(case_id=case_id), + trailblazer_config=self.get_job_ids_path(case_id=case_id), ) LOG.info(f"Pipeline will be resumed from run with Tower id: {from_tower_id}.") parameters: list[str] = NfTowerHandler.get_tower_relaunch_parameters( diff --git a/cg/models/flow_cell/flow_cell.py b/cg/models/flow_cell/flow_cell.py index 22b3b81ad0..a28b8da320 100644 --- a/cg/models/flow_cell/flow_cell.py +++ b/cg/models/flow_cell/flow_cell.py @@ -8,8 +8,12 @@ from pydantic import ValidationError from typing_extensions import Literal -from cg.apps.demultiplex.sample_sheet.read_sample_sheet import get_sample_sheet_from_file +from cg.apps.demultiplex.sample_sheet.read_sample_sheet import ( + get_sample_sheet_from_file, + get_sample_type, +) from cg.apps.demultiplex.sample_sheet.sample_models import ( + FlowCellSample, FlowCellSampleBcl2Fastq, FlowCellSampleBCLConvert, ) @@ -36,6 +40,10 @@ Sequencers.NOVASEQ: RunParametersNovaSeq6000, Sequencers.NOVASEQX: RunParametersNovaSeqX, } +SAMPLE_MODEL_TO_BCL_CONVERTER: dict[Type[FlowCellSample], str] = { + FlowCellSampleBCLConvert: BclConverter.DRAGEN, + FlowCellSampleBcl2Fastq: BclConverter.BCL2FASTQ, +} class FlowCellDirectoryData: @@ -213,6 +221,16 @@ def sample_sheet_exists(self) -> bool: def validate_sample_sheet(self) -> bool: """Validate if sample sheet is on correct format.""" + sample_type_from_sample_sheet: Type[FlowCellSample] = get_sample_type( + self.sample_sheet_path + ) + if SAMPLE_MODEL_TO_BCL_CONVERTER[sample_type_from_sample_sheet] != self.bcl_converter: + LOG.warning( + f"Detected {SAMPLE_MODEL_TO_BCL_CONVERTER[sample_type_from_sample_sheet]} sample " + f"sheet for {self.bcl_converter} flow cell. " + "Generate the correct sample sheet or use the correct bcl converter." + ) + return False try: get_sample_sheet_from_file(self.sample_sheet_path) except (SampleSheetError, ValidationError) as error: diff --git a/cg/models/lims/sample.py b/cg/models/lims/sample.py index 474fc746fb..3fbd784837 100644 --- a/cg/models/lims/sample.py +++ b/cg/models/lims/sample.py @@ -13,6 +13,7 @@ class Udf(BaseModel): comment: str | None concentration: str | None concentration_sample: str | None + concentration_ng_ul: str | None customer: str control: str | None data_analysis: str | None diff --git a/cg/models/orders/samples.py b/cg/models/orders/samples.py index e835be055e..bae0d7aa70 100644 --- a/cg/models/orders/samples.py +++ b/cg/models/orders/samples.py @@ -43,6 +43,7 @@ class OrderInSample(BaseModel): priority: PriorityEnum = PriorityEnum.standard require_qc_ok: bool = False volume: str + concentration_ng_ul: str | None @classmethod def is_sample_for(cls, project: OrderType): diff --git a/cg/models/scout/scout_load_config.py b/cg/models/scout/scout_load_config.py index 3e275aad02..de6415cdfa 100644 --- a/cg/models/scout/scout_load_config.py +++ b/cg/models/scout/scout_load_config.py @@ -5,6 +5,7 @@ from pydantic import BaseModel, BeforeValidator, ConfigDict from typing_extensions import Annotated, Literal +from cg.constants.scout import UploadTrack from cg.models.scout.validators import field_not_none @@ -91,12 +92,13 @@ class ScoutLoadConfig(BaseModel): sv_rank_model_version: str | None = None analysis_date: datetime | None = None samples: list[ScoutIndividual] = [] - - delivery_report: str | None = None + delivery_report: str coverage_qc_report: str | None = None cnv_report: str | None = None multiqc: str | None = None - track: Literal["rare", "cancer"] = "rare" + track: Literal[ + UploadTrack.RARE_DISEASE.value, UploadTrack.CANCER.value + ] = UploadTrack.RARE_DISEASE.value model_config = ConfigDict(validate_assignment=True) diff --git a/cg/store/api/core.py b/cg/store/api/core.py index ce5d5bf174..9ae72f4809 100644 --- a/cg/store/api/core.py +++ b/cg/store/api/core.py @@ -1,18 +1,18 @@ import logging -from cg.store.api.add import AddHandler from cg.store.api.delete import DeleteDataHandler from cg.store.api.find_basic_data import FindBasicDataHandler from cg.store.api.find_business_data import FindBusinessDataHandler from cg.store.api.status import StatusHandler from cg.store.api.update import UpdateHandler +from cg.store.crud.create import CreateHandler from cg.store.database import get_session LOG = logging.getLogger(__name__) class CoreHandler( - AddHandler, + CreateHandler, DeleteDataHandler, FindBasicDataHandler, FindBusinessDataHandler, diff --git a/cg/store/crud/__init__.py b/cg/store/crud/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cg/store/api/add.py b/cg/store/crud/create.py similarity index 99% rename from cg/store/api/add.py rename to cg/store/crud/create.py index 5a7473ab25..b08b6f7527 100644 --- a/cg/store/api/add.py +++ b/cg/store/crud/create.py @@ -31,7 +31,7 @@ LOG = logging.getLogger(__name__) -class AddHandler(BaseHandler): +class CreateHandler(BaseHandler): """Methods related to adding new data to the store.""" def generate_readable_sample_id(self) -> str: diff --git a/cg/store/models.py b/cg/store/models.py index a5adf0dbfb..227e3d68c7 100644 --- a/cg/store/models.py +++ b/cg/store/models.py @@ -218,7 +218,6 @@ class Analysis(Model): created_at = Column(types.DateTime, default=dt.datetime.now, nullable=False) case_id = Column(ForeignKey("case.id", ondelete="CASCADE"), nullable=False) - uploaded_to_vogue_at = Column(types.DateTime, nullable=True) case = orm.relationship("Case", back_populates="analyses") diff --git a/poetry.lock b/poetry.lock index b67e2c2c90..e354ae5c7d 100644 --- a/poetry.lock +++ b/poetry.lock @@ -858,13 +858,13 @@ files = [ [[package]] name = "housekeeper" -version = "4.10.16" +version = "4.11.0" description = "Housekeeper takes care of files" optional = false python-versions = "*" files = [ - {file = "housekeeper-4.10.16-py2.py3-none-any.whl", hash = "sha256:ef85d34f61df05065b0fa5a67ee8505ef5b84dd5f655cfe44909898fafeef5ee"}, - {file = "housekeeper-4.10.16.tar.gz", hash = "sha256:cbd7bced914408e8a557f7817ead45bc61d3cf63c8a402aafac0982643f8e55d"}, + {file = "housekeeper-4.11.0-py2.py3-none-any.whl", hash = "sha256:c309d32d12d26bd57389a21f36cc6ec93745570707b26e5a2f09686b6e95b1c8"}, + {file = "housekeeper-4.11.0.tar.gz", hash = "sha256:05d8644bd4d6aa2765de4d6207049a82a7f12494b71923f3ef4fed9e23e844d3"}, ] [package.dependencies] @@ -1470,13 +1470,13 @@ xml = ["lxml (>=4.8.0)"] [[package]] name = "paramiko" -version = "3.3.1" +version = "3.4.0" description = "SSH2 protocol library" optional = false python-versions = ">=3.6" files = [ - {file = "paramiko-3.3.1-py3-none-any.whl", hash = "sha256:b7bc5340a43de4287bbe22fe6de728aa2c22468b2a849615498dd944c2f275eb"}, - {file = "paramiko-3.3.1.tar.gz", hash = "sha256:6a3777a961ac86dbef375c5f5b8d50014a1a96d0fd7f054a43bc880134b0ff77"}, + {file = "paramiko-3.4.0-py3-none-any.whl", hash = "sha256:43f0b51115a896f9c00f59618023484cb3a14b98bbceab43394a39c6739b7ee7"}, + {file = "paramiko-3.4.0.tar.gz", hash = "sha256:aac08f26a31dc4dffd92821527d1682d99d52f9ef6851968114a8728f3c274d3"}, ] [package.dependencies] @@ -2416,4 +2416,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p [metadata] lock-version = "2.0" python-versions = ">=3.9,<3.13" -content-hash = "5c0c1e334201d01f0f75f6e550bf3056dadec69449beefb642568ed7d16ccba6" +content-hash = "0e80e415fa553084eebcf19e0a13b173db121e7fe77aef7240f5dee85b894e04" diff --git a/pyproject.toml b/pyproject.toml index a687b929df..03c6563115 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "cg" -version = "54.9.1" +version = "55.2.0" description = "Clinical Genomics command center" authors = ["Clinical Genomics "] readme = "README.md" @@ -69,7 +69,7 @@ urllib3 = "*" # Apps genologics = "*" -housekeeper = ">=4.10.16" +housekeeper = "*" [tool.poetry.dev-dependencies] diff --git a/tests/apps/crunchy/test_crunchy.py b/tests/apps/crunchy/test_crunchy.py index c644a3d2bc..4a4c6e6d1d 100644 --- a/tests/apps/crunchy/test_crunchy.py +++ b/tests/apps/crunchy/test_crunchy.py @@ -56,7 +56,7 @@ def test_is_fastq_compression_possible( assert not spring_file.exists() # WHEN checking if SPRING compression is done - result = crunchy_api.is_fastq_compression_possible(compression_object) + result = crunchy_api.is_fastq_compression_possible(compression_obj=compression_object) # THEN result should be True assert result is True @@ -83,7 +83,7 @@ def test_is_fastq_compression_possible_compression_pending( assert not spring_file.exists() # WHEN checking if SPRING compression is done - result = crunchy_api.is_fastq_compression_possible(compression_object) + result = crunchy_api.is_fastq_compression_possible(compression_obj=compression_object) # THEN result should be False since the compression flag exists assert result is False @@ -107,7 +107,7 @@ def test_is_fastq_compression_possible_spring_exists( assert spring_file.exists() # WHEN checking if SPRING compression is done - result = crunchy_api.is_fastq_compression_possible(compression_object) + result = crunchy_api.is_fastq_compression_possible(compression_obj=compression_object) # THEN result should be False since the compression flag exists assert result is False @@ -116,7 +116,7 @@ def test_is_fastq_compression_possible_spring_exists( def test_is_compression_done( - crunchy_config: dict[str, dict[str, Any]], + real_crunchy_api, spring_metadata_file: Path, compression_object: CompressionData, caplog, @@ -124,14 +124,13 @@ def test_is_compression_done( """Test if compression is done when everything is correct""" caplog.set_level(logging.DEBUG) # GIVEN a crunchy-api, and FASTQ paths - crunchy_api = CrunchyAPI(crunchy_config) # GIVEN no SPRING file exists compression_object.spring_path.touch() assert spring_metadata_file == compression_object.spring_metadata_path assert spring_metadata_file.exists() # WHEN checking if SPRING compression is done - result = crunchy_api.is_fastq_compression_done(compression_object) + result = real_crunchy_api.is_fastq_compression_done(compression_object) # THEN result should be True assert result is True @@ -140,18 +139,17 @@ def test_is_compression_done( def test_is_compression_done_no_spring( - crunchy_config: dict[str, dict[str, Any]], compression_object: CompressionData, caplog + real_crunchy_api: CrunchyAPI, compression_object: CompressionData, caplog ): """Test if compression is done when no SPRING archive""" caplog.set_level(logging.DEBUG) # GIVEN a crunchy-api, and FASTQ paths - crunchy_api = CrunchyAPI(crunchy_config) # GIVEN no SPRING file exists spring_file = compression_object.spring_path assert not spring_file.exists() # WHEN checking if SPRING compression is done - result = crunchy_api.is_fastq_compression_done(compression_object) + result = real_crunchy_api.is_fastq_compression_done(compression_object) # THEN result should be false assert not result @@ -160,20 +158,19 @@ def test_is_compression_done_no_spring( def test_is_compression_done_no_flag_spring( - crunchy_config: dict[str, dict[str, Any]], compression_object: CompressionData, caplog + real_crunchy_api: CrunchyAPI, compression_object: CompressionData, caplog ): """Test if SPRING compression is done when no metadata file""" caplog.set_level(logging.DEBUG) # GIVEN a crunchy-api, and FASTQ paths - crunchy_api = CrunchyAPI(crunchy_config) # GIVEN a existing SPRING file compression_object.spring_path.touch() assert compression_object.spring_path.exists() - # GIVEN a non existing flag file + # GIVEN a non-existing flag file assert not compression_object.spring_metadata_path.exists() # WHEN checking if SPRING compression is done - result = crunchy_api.is_fastq_compression_done(compression_object) + result = real_crunchy_api.is_fastq_compression_done(compression_object) # THEN result should be false assert not result @@ -182,7 +179,7 @@ def test_is_compression_done_no_flag_spring( def test_is_compression_done_spring( - crunchy_config: dict[str, dict[str, Any]], + real_crunchy_api: CrunchyAPI, compression_object: CompressionData, spring_metadata_file: Path, caplog, @@ -190,7 +187,6 @@ def test_is_compression_done_spring( """Test if compression is done when SPRING files exists""" caplog.set_level(logging.DEBUG) # GIVEN a crunchy-api, and FASTQ paths - crunchy_api = CrunchyAPI(crunchy_config) # GIVEN a existing SPRING file compression_object.spring_path.touch() assert compression_object.spring_path.exists() @@ -199,7 +195,7 @@ def test_is_compression_done_spring( assert compression_object.spring_metadata_path.exists() # WHEN checking if SPRING compression is done - result = crunchy_api.is_fastq_compression_done(compression_object) + result = real_crunchy_api.is_fastq_compression_done(compression_object) # THEN result should be True assert result @@ -208,7 +204,7 @@ def test_is_compression_done_spring( def test_is_compression_done_spring_new_files( - crunchy_config: dict[str, dict[str, Any]], + real_crunchy_api: CrunchyAPI, compression_object: CompressionData, spring_metadata_file: Path, caplog, @@ -219,7 +215,6 @@ def test_is_compression_done_spring_new_files( """ caplog.set_level(logging.DEBUG) # GIVEN a crunchy-api, and FASTQ paths - crunchy_api = CrunchyAPI(crunchy_config) # GIVEN a existing SPRING file compression_object.spring_path.touch() assert compression_object.spring_path.exists() @@ -237,7 +232,7 @@ def test_is_compression_done_spring_new_files( assert "updated" in file_info # WHEN checking if SPRING compression is done - result = crunchy_api.is_fastq_compression_done(compression_object) + result = real_crunchy_api.is_fastq_compression_done(compression_object) # THEN result should be False since the updated date < 3 weeks assert result is False @@ -246,7 +241,7 @@ def test_is_compression_done_spring_new_files( def test_is_compression_done_spring_old_files( - crunchy_config: dict[str, dict[str, Any]], + real_crunchy_api: CrunchyAPI, compression_object: CompressionData, spring_metadata_file: Path, caplog, @@ -257,7 +252,6 @@ def test_is_compression_done_spring_old_files( """ caplog.set_level(logging.DEBUG) # GIVEN a crunchy-api, and FASTQ paths - crunchy_api = CrunchyAPI(crunchy_config) # GIVEN a existing SPRING file compression_object.spring_path.touch() assert compression_object.spring_path.exists() @@ -277,7 +271,7 @@ def test_is_compression_done_spring_old_files( ) # WHEN checking if SPRING compression is done - result = crunchy_api.is_fastq_compression_done(compression_object) + result = real_crunchy_api.is_fastq_compression_done(compression_object) # THEN result should be True since the updated date > 3 weeks assert result is True @@ -286,7 +280,7 @@ def test_is_compression_done_spring_old_files( def test_is_spring_decompression_possible_no_fastq( - crunchy_config: dict[str, dict[str, Any]], compression_object: CompressionData, caplog + real_crunchy_api: CrunchyAPI, compression_object: CompressionData, caplog ): """Test if decompression is possible when there are no FASTQ files @@ -294,7 +288,6 @@ def test_is_spring_decompression_possible_no_fastq( """ caplog.set_level(logging.DEBUG) # GIVEN a crunchy-api - crunchy_api = CrunchyAPI(crunchy_config) # GIVEN a existing SPRING file compression_object.spring_path.touch() assert compression_object.spring_path.exists() @@ -305,7 +298,7 @@ def test_is_spring_decompression_possible_no_fastq( assert not compression_object.fastq_second.exists() # WHEN checking if SPRING compression is done - result = crunchy_api.is_spring_decompression_possible(compression_object) + result = real_crunchy_api.is_spring_decompression_possible(compression_object) # THEN result should be True since there are no fastq files assert result is True @@ -314,7 +307,7 @@ def test_is_spring_decompression_possible_no_fastq( def test_is_spring_decompression_possible_no_spring( - crunchy_config: dict[str, dict[str, Any]], compression_object: CompressionData, caplog + real_crunchy_api: CrunchyAPI, compression_object: CompressionData, caplog ): """Test if decompression is possible when there are no SPRING archive @@ -322,10 +315,9 @@ def test_is_spring_decompression_possible_no_spring( """ caplog.set_level(logging.DEBUG) # GIVEN a crunchy-api, and FASTQ paths - crunchy_api = CrunchyAPI(crunchy_config) # WHEN checking if SPRING compression is done - result = crunchy_api.is_spring_decompression_possible(compression_object) + result = real_crunchy_api.is_spring_decompression_possible(compression_object) # THEN result should be False since there is no SPRING archive assert result is False @@ -334,7 +326,7 @@ def test_is_spring_decompression_possible_no_spring( def test_is_spring_decompression_possible_fastq( - crunchy_config: dict[str, dict[str, Any]], compression_object: CompressionData, caplog + real_crunchy_api: CrunchyAPI, compression_object: CompressionData, caplog ): """Test if decompression is possible when there are existing FASTQ files @@ -342,7 +334,6 @@ def test_is_spring_decompression_possible_fastq( """ caplog.set_level(logging.DEBUG) # GIVEN a crunchy-api, and FASTQ paths - crunchy_api = CrunchyAPI(crunchy_config) # GIVEN a existing SPRING file compression_object.spring_path.touch() # GIVEN that the FASTQ files exists @@ -350,7 +341,7 @@ def test_is_spring_decompression_possible_fastq( compression_object.fastq_second.touch() # WHEN checking if SPRING decompression is possible - result = crunchy_api.is_spring_decompression_possible(compression_object) + result = real_crunchy_api.is_spring_decompression_possible(compression_object) # THEN result should be False since the FASTQ files already exists assert result is False @@ -359,31 +350,29 @@ def test_is_spring_decompression_possible_fastq( def test_is_not_pending_when_no_flag_file( - crunchy_config: dict[str, dict[str, Any]], compression_object: CompressionData + real_crunchy_api: CrunchyAPI, compression_object: CompressionData ): """Test if SPRING compression is pending when no flag file""" # GIVEN a crunchy-api, and a FASTQ file - crunchy_api = CrunchyAPI(crunchy_config) - # GIVEN a non existing pending flag + # GIVEN a non-existing pending flag assert not compression_object.pending_path.exists() # WHEN checking if SPRING compression is ongoing - result = crunchy_api.is_compression_pending(compression_object) + result = real_crunchy_api.is_compression_pending(compression_object) # THEN result should be False since the pending flag is not there assert result is False -def test_is_pending(crunchy_config: dict[str, dict[str, Any]], compression_object: CompressionData): +def test_is_pending(real_crunchy_api: CrunchyAPI, compression_object: CompressionData): """Test if SPRING compression is pending when pending file exists""" # GIVEN a crunchy-api, and FASTQ files - crunchy_api = CrunchyAPI(crunchy_config) # GIVEN a existing pending flag compression_object.pending_path.touch() assert compression_object.pending_path.exists() # WHEN checking if SPRING compression is pending - result = crunchy_api.is_compression_pending(compression_object) + result = real_crunchy_api.is_compression_pending(compression_object) # THEN result should be True since the pending_path exists assert result is True diff --git a/tests/apps/scout/test_scout_load_config.py b/tests/apps/scout/test_scout_load_config.py index b97a613233..f732dc0193 100644 --- a/tests/apps/scout/test_scout_load_config.py +++ b/tests/apps/scout/test_scout_load_config.py @@ -1,4 +1,5 @@ -"""Tests for the models in scout load config""" +"""Tests for the models in Scout load config.""" +from pathlib import Path from typing import Any import pytest @@ -11,8 +12,10 @@ @pytest.mark.parametrize("key, value", list(SCOUT_INDIVIDUAL.items())) def test_validate_scout_individual_attributes(scout_individual: dict, key: str, value: Any): - """Test to validate that all attributes of a ScoutMipIndividual are correctly set.""" + """Test that all attributes of a ScoutMipIndividual are correctly set.""" + # GIVEN some sample information + # WHEN instantiating a ScoutMipIndividual ind_obj: ScoutMipIndividual = scout_load_config.ScoutMipIndividual(**scout_individual) @@ -20,26 +23,29 @@ def test_validate_scout_individual_attributes(scout_individual: dict, key: str, assert getattr(ind_obj, key) == value -def test_instantiate_empty_mip_config(): - """Tests whether a MipLoadConfig can be instantiate without arguments.""" - # GIVEN nothing +def test_instantiate_empty_mip_config(delivery_report_html: Path): + """Tests whether a MipLoadConfig can be instantiated only with mandatory arguments.""" + + # GIVEN a delivery report file - # WHEN instantiating a empty mip load config - config: MipLoadConfig = scout_load_config.MipLoadConfig() + # WHEN instantiating an empty MIP load config + config: MipLoadConfig = scout_load_config.MipLoadConfig( + delivery_report=delivery_report_html.as_posix() + ) - # THEN assert it is possible to instantiate without any information + # THEN assert it is possible to instantiate without any not mandatory information assert isinstance(config, scout_load_config.ScoutLoadConfig) -def test_set_mandatory_to_none(): - """The scout load config object should validate fields as they are set. +def test_set_mandatory_to_none(delivery_report_html: Path): + """Test that a value error is raised when a mandatory field is set to None.""" - This test will check that a value error is raised when a mandatory field is set to None. - """ # GIVEN a load config object - config: MipLoadConfig = scout_load_config.MipLoadConfig() + config: MipLoadConfig = scout_load_config.MipLoadConfig( + delivery_report=delivery_report_html.as_posix() + ) # WHEN setting a mandatory field to None with pytest.raises(ValidationError): - # THEN assert a validation error was raised + # THEN a validation error should be raised config.vcf_snv = None diff --git a/tests/cli/demultiplex/test_demultiplex_flowcell.py b/tests/cli/demultiplex/test_demultiplex_flowcell.py index 2032cb814c..9d72e529cf 100644 --- a/tests/cli/demultiplex/test_demultiplex_flowcell.py +++ b/tests/cli/demultiplex/test_demultiplex_flowcell.py @@ -10,13 +10,9 @@ from cg.meta.demultiplex.housekeeper_storage_functions import add_sample_sheet_path_to_housekeeper from cg.models.cg_config import CGConfig from cg.models.flow_cell.flow_cell import FlowCellDirectoryData -from tests.meta.demultiplex.conftest import ( - tmp_flow_cell_demux_base_path, - tmp_flow_cell_run_base_path, -) -def test_demultiplex_flow_cell_dry_run( +def test_demultiplex_bcl2fastq_flow_cell_dry_run( cli_runner: testing.CliRunner, tmp_flow_cells_directory_ready_for_demultiplexing_bcl2fastq: Path, demultiplexing_context_for_demux: CGConfig, @@ -26,7 +22,8 @@ def test_demultiplex_flow_cell_dry_run( # GIVEN that all files are present for demultiplexing flow_cell: FlowCellDirectoryData = FlowCellDirectoryData( - tmp_flow_cells_directory_ready_for_demultiplexing_bcl2fastq + tmp_flow_cells_directory_ready_for_demultiplexing_bcl2fastq, + bcl_converter=BclConverter.BCL2FASTQ, ) add_sample_sheet_path_to_housekeeper( flow_cell_directory=tmp_flow_cells_directory_ready_for_demultiplexing_bcl2fastq, @@ -45,7 +42,12 @@ def test_demultiplex_flow_cell_dry_run( # WHEN starting demultiplexing from the CLI with dry run flag result: testing.Result = cli_runner.invoke( demultiplex_flow_cell, - [str(tmp_flow_cells_directory_ready_for_demultiplexing_bcl2fastq), "--dry-run"], + [ + str(tmp_flow_cells_directory_ready_for_demultiplexing_bcl2fastq), + "--dry-run", + "-b", + "bcl2fastq", + ], obj=demultiplexing_context_for_demux, ) diff --git a/tests/cli/upload/conftest.py b/tests/cli/upload/conftest.py index 3bac71741b..d3ad48b1dd 100644 --- a/tests/cli/upload/conftest.py +++ b/tests/cli/upload/conftest.py @@ -28,12 +28,6 @@ from cg.models.scout.scout_load_config import ScoutLoadConfig from cg.store import Store from cg.store.models import Analysis -from tests.cli.workflow.mip.conftest import ( - mip_case_id, - mip_case_ids, - mip_dna_context, - mip_rna_context, -) from tests.meta.upload.scout.conftest import mip_load_config from tests.mocks.hk_mock import MockHousekeeperAPI from tests.mocks.madeline import MockMadelineAPI @@ -248,7 +242,9 @@ def __init__(self, **kwargs): self.housekeeper = None self.madeline_api = MockMadelineAPI() self.analysis = MockAnalysisApi() - self.config = ScoutLoadConfig() + self.config = ScoutLoadConfig( + delivery_report=Path("path", "to", "delivery-report.html").as_posix() + ) self.file_exists = False self.lims = MockLims() self.missing_mandatory_field = False diff --git a/tests/cli/workflow/conftest.py b/tests/cli/workflow/conftest.py index 8f676c960f..2d3c07014a 100644 --- a/tests/cli/workflow/conftest.py +++ b/tests/cli/workflow/conftest.py @@ -135,7 +135,6 @@ class MockTB: def __init__(self): self._link_was_called = False - self._mark_analyses_deleted_called = False self._add_pending_was_called = False self._add_pending_analysis_was_called = False self._family = None @@ -187,11 +186,6 @@ def first_was_called(self): return Row() - def mark_analyses_deleted(self, case_id: str): - """Mock this function""" - self._case_id = case_id - self._mark_analyses_deleted_called = True - def add_pending(self, case_id: str, email: str): """Mock this function""" self._case_id = case_id @@ -204,10 +198,6 @@ def add_pending_analysis(self, case_id: str, email: str): self._email = email self._add_pending_analysis_was_called = True - def mark_analyses_deleted_called(self): - """check if mark_analyses_deleted was called""" - return self._mark_analyses_deleted_called - def add_pending_was_called(self): """check if add_pending was called""" return self._add_pending_was_called diff --git a/tests/conftest.py b/tests/conftest.py index bc3f593a77..12b16a06cb 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -5,7 +5,7 @@ import os import shutil from copy import deepcopy -from datetime import MAXYEAR, datetime, timedelta +from datetime import datetime from pathlib import Path from subprocess import CompletedProcess from typing import Any, Generator @@ -14,27 +14,23 @@ from housekeeper.store.models import File, Version from requests import Response +from cg.apps.crunchy import CrunchyAPI from cg.apps.demultiplex.demultiplex_api import DemultiplexingAPI -from cg.apps.demultiplex.sample_sheet.sample_models import ( - FlowCellSampleBcl2Fastq, - FlowCellSampleBCLConvert, -) -from cg.apps.demultiplex.sample_sheet.sample_sheet_creator import SampleSheetCreatorBCLConvert from cg.apps.downsample.downsample import DownsampleAPI from cg.apps.gens import GensAPI from cg.apps.gt import GenotypeAPI from cg.apps.hermes.hermes_api import HermesApi from cg.apps.housekeeper.hk import HousekeeperAPI -from cg.apps.lims.api import LimsAPI +from cg.apps.lims import LimsAPI from cg.apps.slurm.slurm_api import SlurmAPI from cg.constants import FileExtensions, Pipeline, SequencingFileTag from cg.constants.constants import CaseActions, FileFormat, Strandedness -from cg.constants.demultiplexing import BclConverter, DemultiplexingDirsAndFiles -from cg.constants.nanopore_files import NanoporeDirsAndFiles +from cg.constants.demultiplexing import DemultiplexingDirsAndFiles +from cg.constants.housekeeper_tags import HK_DELIVERY_REPORT_TAG from cg.constants.priority import SlurmQos from cg.constants.sequencing import SequencingPlatform from cg.constants.subject import Sex -from cg.io.controller import ReadFile, WriteFile +from cg.io.controller import WriteFile from cg.io.json import read_json, write_json from cg.io.yaml import write_yaml from cg.meta.encryption.encryption import FlowCellEncryptionAPI @@ -46,11 +42,6 @@ from cg.meta.workflow.taxprofiler import TaxprofilerAnalysisAPI from cg.models import CompressionData from cg.models.cg_config import CGConfig, PDCArchivingDirectory -from cg.models.demultiplex.run_parameters import ( - RunParametersHiSeq, - RunParametersNovaSeq6000, - RunParametersNovaSeqX, -) from cg.models.downsample.downsample_data import DownsampleData from cg.models.flow_cell.flow_cell import FlowCellDirectoryData from cg.models.rnafusion.rnafusion import RnafusionParameters @@ -72,51 +63,14 @@ LOG = logging.getLogger(__name__) - -# Timestamp fixture - - -@pytest.fixture(scope="session") -def old_timestamp() -> datetime: - """Return a time stamp in date time format.""" - return datetime(1900, 1, 1) - - -@pytest.fixture(scope="session") -def timestamp() -> datetime: - """Return a time stamp in date time format.""" - return datetime(2020, 5, 1) - - -@pytest.fixture(scope="session") -def later_timestamp() -> datetime: - """Return a time stamp in date time format.""" - return datetime(2020, 6, 1) - - -@pytest.fixture(scope="session") -def future_date() -> datetime: - """Return a distant date in the future for which no events happen later.""" - return datetime(MAXYEAR, 1, 1, 1, 1, 1) - - -@pytest.fixture(scope="session") -def timestamp_now() -> datetime: - """Return a time stamp of today's date in date time format.""" - return datetime.now() - - -@pytest.fixture(scope="session") -def timestamp_yesterday(timestamp_now: datetime) -> datetime: - """Return a time stamp of yesterday's date in date time format.""" - return timestamp_now - timedelta(days=1) - - -@pytest.fixture(scope="session") -def timestamp_in_2_weeks(timestamp_now: datetime) -> datetime: - """Return a time stamp 14 days ahead in time.""" - return timestamp_now + timedelta(days=14) - +pytest_plugins = [ + "tests.fixture_plugins.timestamp_fixtures", + "tests.fixture_plugins.demultiplex_fixtures.flow_cell_fixtures", + "tests.fixture_plugins.demultiplex_fixtures.name_fixtures", + "tests.fixture_plugins.demultiplex_fixtures.path_fixtures", + "tests.fixture_plugins.demultiplex_fixtures.run_parameters_fixtures", + "tests.fixture_plugins.demultiplex_fixtures.sample_fixtures", +] # Case fixtures @@ -380,6 +334,64 @@ def crunchy_config() -> dict[str, dict[str, Any]]: } +@pytest.fixture +def demultiplexing_context_for_demux( + demultiplexing_api_for_demux: DemultiplexingAPI, + cg_context: CGConfig, + store_with_demultiplexed_samples: Store, +) -> CGConfig: + """Return cg context with a demultiplex context.""" + cg_context.demultiplex_api_ = demultiplexing_api_for_demux + cg_context.housekeeper_api_ = demultiplexing_api_for_demux.hk_api + cg_context.status_db_ = store_with_demultiplexed_samples + return cg_context + + +@pytest.fixture(name="demultiplex_context") +def demultiplex_context( + demultiplexing_api: DemultiplexingAPI, + real_housekeeper_api: HousekeeperAPI, + cg_context: CGConfig, + store_with_demultiplexed_samples: Store, +) -> CGConfig: + """Return cg context with a demultiplex context.""" + cg_context.demultiplex_api_ = demultiplexing_api + cg_context.housekeeper_api_ = real_housekeeper_api + cg_context.status_db_ = store_with_demultiplexed_samples + return cg_context + + +@pytest.fixture(name="demultiplex_configs_for_demux") +def demultiplex_configs_for_demux( + tmp_flow_cells_demux_all_directory: Path, + tmp_empty_demultiplexed_runs_directory: Path, +) -> dict: + """Return demultiplex configs.""" + return { + "flow_cells_dir": tmp_flow_cells_demux_all_directory.as_posix(), + "demultiplexed_flow_cells_dir": tmp_empty_demultiplexed_runs_directory.as_posix(), + "demultiplex": {"slurm": {"account": "test", "mail_user": "testuser@github.se"}}, + } + + +@pytest.fixture(name="demultiplex_configs") +def demultiplex_configs( + tmp_flow_cells_directory: Path, + tmp_demultiplexed_runs_directory: Path, +) -> dict: + """Return demultiplex configs.""" + return { + "flow_cells_dir": tmp_flow_cells_directory.as_posix(), + "demultiplexed_flow_cells_dir": tmp_demultiplexed_runs_directory.as_posix(), + "demultiplex": {"slurm": {"account": "test", "mail_user": "testuser@github.se"}}, + } + + +@pytest.fixture +def real_crunchy_api(crunchy_config) -> CrunchyAPI: + return CrunchyAPI(crunchy_config) + + @pytest.fixture def hk_config_dict(root_path: Path): """Housekeeper configs.""" @@ -414,9 +426,46 @@ def gens_config() -> dict[str, dict[str, str]]: } +@pytest.fixture(name="sample_sheet_context") +def sample_sheet_context( + cg_context: CGConfig, lims_api: LimsAPI, populated_housekeeper_api: HousekeeperAPI +) -> CGConfig: + """Return cg context with added Lims and Housekeeper API.""" + cg_context.lims_api_ = lims_api + cg_context.housekeeper_api_ = populated_housekeeper_api + return cg_context + + # Api fixtures +@pytest.fixture(name="demultiplexing_api_for_demux") +def demultiplexing_api_for_demux( + demultiplex_configs_for_demux: dict, + sbatch_process: Process, + populated_housekeeper_api: HousekeeperAPI, +) -> DemultiplexingAPI: + """Return demultiplex API.""" + demux_api = DemultiplexingAPI( + config=demultiplex_configs_for_demux, + housekeeper_api=populated_housekeeper_api, + ) + demux_api.slurm_api.process = sbatch_process + return demux_api + + +@pytest.fixture +def demultiplexing_api( + demultiplex_configs: dict, sbatch_process: Process, populated_housekeeper_api: HousekeeperAPI +) -> DemultiplexingAPI: + """Return demultiplex API.""" + demux_api = DemultiplexingAPI( + config=demultiplex_configs, housekeeper_api=populated_housekeeper_api + ) + demux_api.slurm_api.process = sbatch_process + return demux_api + + @pytest.fixture def rsync_api(cg_context: CGConfig) -> RsyncAPI: """RsyncAPI fixture.""" @@ -510,1320 +559,267 @@ def fixtures_dir() -> Path: return Path("tests", "fixtures") -@pytest.fixture(scope="session") -def analysis_dir(fixtures_dir: Path) -> Path: - """Return the path to the analysis dir.""" - return Path(fixtures_dir, "analysis") - - -@pytest.fixture(scope="session") -def microsalt_analysis_dir(analysis_dir: Path) -> Path: - """Return the path to the analysis dir.""" - return Path(analysis_dir, "microsalt") - - -@pytest.fixture(scope="session") -def apps_dir(fixtures_dir: Path) -> Path: - """Return the path to the apps dir.""" - return Path(fixtures_dir, "apps") - - -@pytest.fixture(scope="session") -def cgweb_orders_dir(fixtures_dir: Path) -> Path: - """Return the path to the cgweb_orders dir.""" - return Path(fixtures_dir, "cgweb_orders") - - -@pytest.fixture(scope="session") -def data_dir(fixtures_dir: Path) -> Path: - """Return the path to the data dir.""" - return Path(fixtures_dir, "data") - - -@pytest.fixture -def fastq_dir(demultiplex_fixtures: Path) -> Path: - """Return the path to the fastq files dir.""" - return Path(demultiplex_fixtures, "fastq") - - -@pytest.fixture -def spring_dir(demultiplex_fixtures: Path) -> Path: - """Return the path to the fastq files dir.""" - return Path(demultiplex_fixtures, "spring") - - -@pytest.fixture -def project_dir(tmpdir_factory) -> Generator[Path, None, None]: - """Path to a temporary directory where intermediate files can be stored.""" - yield Path(tmpdir_factory.mktemp("data")) - - -@pytest.fixture -def tmp_file(project_dir) -> Path: - """Return a temp file path.""" - return Path(project_dir, "test") - - -@pytest.fixture -def non_existing_file_path(project_dir: Path) -> Path: - """Return the path to a non-existing file.""" - return Path(project_dir, "a_file.txt") - - -@pytest.fixture(scope="session") -def content() -> str: - """Return some content for a file.""" - return ( - "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt" - " ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ull" - "amco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehende" - "rit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaec" - "at cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum." - ) - - -@pytest.fixture -def filled_file(non_existing_file_path: Path, content: str) -> Path: - """Return the path to a existing file with some content.""" - with open(non_existing_file_path, "w") as outfile: - outfile.write(content) - return non_existing_file_path - - -@pytest.fixture(scope="session") -def orderforms(fixtures_dir: Path) -> Path: - """Return the path to the directory with order forms.""" - return Path(fixtures_dir, "orderforms") - - -@pytest.fixture -def hk_file(filled_file: Path, case_id: str) -> File: - """Return a housekeeper File object.""" - return File(id=case_id, path=filled_file.as_posix()) - - -@pytest.fixture -def mip_dna_store_files(apps_dir: Path) -> Path: - """Return the path to the directory with mip dna store files.""" - return Path(apps_dir, "mip", "dna", "store") - - -@pytest.fixture -def case_qc_sample_info_path(mip_dna_store_files: Path) -> Path: - """Return path to case_qc_sample_info.yaml.""" - return Path(mip_dna_store_files, "case_qc_sample_info.yaml") - - -@pytest.fixture -def delivery_report_html(mip_dna_store_files: Path) -> Path: - """Return the path to a qc metrics deliverables file with case data.""" - return Path(mip_dna_store_files, "empty_delivery_report.html") - - -@pytest.fixture -def mip_deliverables_file(mip_dna_store_files: Path) -> Path: - """Fixture for general deliverables file in mip.""" - return Path(mip_dna_store_files, "case_id_deliverables.yaml") - - -@pytest.fixture -def case_qc_metrics_deliverables(apps_dir: Path) -> Path: - """Return the path to a qc metrics deliverables file with case data.""" - return Path(apps_dir, "mip", "case_metrics_deliverables.yaml") - - -@pytest.fixture -def mip_analysis_dir(analysis_dir: Path) -> Path: - """Return the path to the directory with mip analysis files.""" - return Path(analysis_dir, "mip") - - -@pytest.fixture -def balsamic_analysis_dir(analysis_dir: Path) -> Path: - """Return the path to the directory with balsamic analysis files.""" - return Path(analysis_dir, "balsamic") - - -@pytest.fixture -def balsamic_wgs_analysis_dir(balsamic_analysis_dir: Path) -> Path: - """Return the path to the directory with balsamic analysis files.""" - return Path(balsamic_analysis_dir, "tn_wgs") - - -@pytest.fixture -def mip_dna_analysis_dir(mip_analysis_dir: Path) -> Path: - """Return the path to the directory with mip dna analysis files.""" - return Path(mip_analysis_dir, "dna") - - -@pytest.fixture -def rnafusion_analysis_dir(analysis_dir: Path) -> Path: - """Return the path to the directory with rnafusion analysis files.""" - return Path(analysis_dir, "rnafusion") - - -@pytest.fixture -def sample_cram(mip_dna_analysis_dir: Path) -> Path: - """Return the path to the cram file for a sample.""" - return Path(mip_dna_analysis_dir, "adm1.cram") - - -@pytest.fixture(name="father_sample_cram") -def father_sample_cram( - mip_dna_analysis_dir: Path, - father_sample_id: str, -) -> Path: - """Return the path to the cram file for the father sample.""" - return Path(mip_dna_analysis_dir, father_sample_id + FileExtensions.CRAM) - - -@pytest.fixture(name="mother_sample_cram") -def mother_sample_cram(mip_dna_analysis_dir: Path, mother_sample_id: str) -> Path: - """Return the path to the cram file for the mother sample.""" - return Path(mip_dna_analysis_dir, mother_sample_id + FileExtensions.CRAM) - - -@pytest.fixture(name="sample_cram_files") -def sample_crams( - sample_cram: Path, father_sample_cram: Path, mother_sample_cram: Path -) -> list[Path]: - """Return a list of cram paths for three samples.""" - return [sample_cram, father_sample_cram, mother_sample_cram] - - -@pytest.fixture(name="vcf_file") -def vcf_file(mip_dna_store_files: Path) -> Path: - """Return the path to a VCF file.""" - return Path(mip_dna_store_files, "yellowhog_clinical_selected.vcf") - - -@pytest.fixture(name="fastq_file") -def fastq_file(fastq_dir: Path) -> Path: - """Return the path to a FASTQ file.""" - return Path(fastq_dir, "dummy_run_R1_001.fastq.gz") - - -@pytest.fixture(name="fastq_file_father") -def fastq_file_father(fastq_dir: Path) -> Path: - """Return the path to a FASTQ file.""" - return Path(fastq_dir, "fastq_run_R1_001.fastq.gz") - - -@pytest.fixture(name="spring_file") -def spring_file(spring_dir: Path) -> Path: - """Return the path to an existing spring file.""" - return Path(spring_dir, "dummy_run_001.spring") - - -@pytest.fixture(name="spring_meta_data_file") -def spring_meta_data_file(spring_dir: Path) -> Path: - """Return the path to an existing spring file.""" - return Path(spring_dir, "dummy_spring_meta_data.json") - - -@pytest.fixture(name="spring_file_father") -def spring_file_father(spring_dir: Path) -> Path: - """Return the path to a second existing spring file.""" - return Path(spring_dir, "dummy_run_002.spring") - - -@pytest.fixture(name="madeline_output") -def madeline_output(apps_dir: Path) -> Path: - """Return str of path for file with Madeline output.""" - return Path(apps_dir, "madeline", "madeline.xml") - - -@pytest.fixture(name="file_does_not_exist") -def file_does_not_exist() -> Path: - """Return a file path that does not exist.""" - return Path("file", "does", "not", "exist") - - -# Compression fixtures - - -@pytest.fixture(name="run_name") -def run_name() -> str: - """Return the name of a fastq run.""" - return "fastq_run" - - -@pytest.fixture(name="original_fastq_data") -def original_fastq_data(fastq_dir: Path, run_name) -> CompressionData: - """Return a compression object with a path to the original fastq files.""" - return CompressionData(Path(fastq_dir, run_name)) - - -@pytest.fixture(name="fastq_stub") -def fastq_stub(project_dir: Path, run_name: str) -> Path: - """Creates a path to the base format of a fastq run.""" - return Path(project_dir, run_name) - - -@pytest.fixture(name="compression_object") -def compression_object(fastq_stub: Path, original_fastq_data: CompressionData) -> CompressionData: - """Creates compression data object with information about files used in fastq compression.""" - working_files: CompressionData = CompressionData(fastq_stub) - working_file_map: dict[str, str] = { - original_fastq_data.fastq_first.as_posix(): working_files.fastq_first.as_posix(), - original_fastq_data.fastq_second.as_posix(): working_files.fastq_second.as_posix(), - } - for original_file, working_file in working_file_map.items(): - shutil.copy(original_file, working_file) - return working_files - - -# Demultiplex fixtures - - -@pytest.fixture -def lims_novaseq_bcl_convert_samples( - lims_novaseq_samples_raw: list[dict], -) -> list[FlowCellSampleBCLConvert]: - """Return a list of parsed flow cell samples demultiplexed with BCL convert.""" - return [FlowCellSampleBCLConvert.model_validate(sample) for sample in lims_novaseq_samples_raw] - - -@pytest.fixture -def lims_novaseq_bcl2fastq_samples( - lims_novaseq_samples_raw: list[dict], -) -> list[FlowCellSampleBcl2Fastq]: - """Return a list of parsed Bcl2fastq flow cell samples""" - return [FlowCellSampleBcl2Fastq.model_validate(sample) for sample in lims_novaseq_samples_raw] - - -@pytest.fixture -def lims_novaseq_6000_bcl2fastq_samples( - lims_novaseq_6000_sample_raw: list[dict], -) -> list[FlowCellSampleBcl2Fastq]: - """Return a list of parsed Bcl2fastq flow cell samples""" - return [ - FlowCellSampleBcl2Fastq.model_validate(sample) for sample in lims_novaseq_6000_sample_raw - ] - - -@pytest.fixture(name="tmp_flow_cells_directory") -def tmp_flow_cells_directory(tmp_path: Path, flow_cells_dir: Path) -> Path: - """ - Return the path to a temporary flow cells directory with flow cells ready for demultiplexing. - Generates a copy of the original flow cells directory - """ - original_dir = flow_cells_dir - tmp_dir = Path(tmp_path, "flow_cells") - - return Path(shutil.copytree(original_dir, tmp_dir)) - - -@pytest.fixture(name="tmp_flow_cells_demux_all_directory") -def tmp_flow_cells_demux_all_directory(tmp_path: Path, flow_cells_demux_all_dir: Path) -> Path: - """ - Return the path to a temporary flow cells directory with flow cells ready for demultiplexing. - Generates a copy of the original flow cells directory. - This fixture is used for testing of the cg demutliplex all cmd. - """ - original_dir = flow_cells_demux_all_dir - tmp_dir = Path(tmp_path, "flow_cells_demux_all") - - return Path(shutil.copytree(original_dir, tmp_dir)) - - -@pytest.fixture(name="tmp_flow_cell_directory_bcl2fastq") -def flow_cell_working_directory_bcl2fastq( - bcl2fastq_flow_cell_dir: Path, tmp_flow_cells_directory: Path -) -> Path: - """Return the path to a working directory that will be deleted after test is run. - - This is a path to a flow cell directory with the run parameters present. - """ - return Path(tmp_flow_cells_directory, bcl2fastq_flow_cell_dir.name) - - -@pytest.fixture(name="tmp_flow_cell_directory_bclconvert") -def flow_cell_working_directory_bclconvert( - bcl_convert_flow_cell_dir: Path, tmp_flow_cells_directory: Path -) -> Path: - """Return the path to a working directory that will be deleted after test is run. - This is a path to a flow cell directory with the run parameters present. - """ - return Path(tmp_flow_cells_directory, bcl_convert_flow_cell_dir.name) - - -@pytest.fixture -def tmp_flow_cell_name_no_run_parameters() -> str: - """This is the name of a flow cell directory with the run parameters missing.""" - return "180522_A00689_0200_BHLCKNCCXY" - - -@pytest.fixture -def tmp_flow_cell_name_malformed_sample_sheet() -> str: - """ "Returns the name of a flow cell directory ready for demultiplexing with BCL convert. - Contains a sample sheet with malformed headers. - """ - return "201203_A00689_0200_AHVKJCDRXY" - - -@pytest.fixture -def tmp_flow_cell_name_no_sample_sheet() -> str: - """Return the name of a flow cell directory with the run parameters and sample sheet missing.""" - return "170407_A00689_0209_BHHKVCALXX" - - -@pytest.fixture(name="tmp_flow_cell_name_ready_for_demultiplexing_bcl2fastq") -def tmp_flow_cell_name_ready_for_demultiplexing_bcl2fastq() -> str: - """Returns the name of a flow cell directory ready for demultiplexing with bcl2fastq.""" - return "211101_D00483_0615_AHLG5GDRXY" - - -@pytest.fixture -def tmp_flow_cells_directory_no_run_parameters( - tmp_flow_cell_name_no_run_parameters: str, tmp_flow_cells_directory: Path -) -> Path: - """This is a path to a flow cell directory with the run parameters missing.""" - return Path(tmp_flow_cells_directory, tmp_flow_cell_name_no_run_parameters) - - -@pytest.fixture(name="tmp_flow_cells_directory_no_sample_sheet") -def tmp_flow_cells_directory_no_sample_sheet( - tmp_flow_cell_name_no_sample_sheet: str, tmp_flow_cells_directory: Path -) -> Path: - """This is a path to a flow cell directory with the sample sheet and run parameters missing.""" - return Path(tmp_flow_cells_directory, tmp_flow_cell_name_no_sample_sheet) - - -@pytest.fixture -def tmp_flow_cells_directory_malformed_sample_sheet( - tmp_flow_cell_name_malformed_sample_sheet: str, tmp_flow_cells_directory: Path -) -> Path: - """This is a path to a flow cell directory with a sample sheet with malformed headers.""" - return Path(tmp_flow_cells_directory, tmp_flow_cell_name_malformed_sample_sheet) - - -@pytest.fixture -def tmp_flow_cells_directory_ready_for_demultiplexing_bcl_convert( - bcl_convert_flow_cell_full_name: str, tmp_flow_cells_directory: Path -) -> Path: - """This is a path to a flow cell directory with the run parameters missing.""" - return Path(tmp_flow_cells_directory, bcl_convert_flow_cell_full_name) - - -@pytest.fixture -def tmp_flow_cells_directory_ready_for_demultiplexing_bcl2fastq( - tmp_flow_cell_name_ready_for_demultiplexing_bcl2fastq: str, tmp_flow_cells_directory: Path -) -> Path: - """This is a path to a flow cell directory with the run parameters missing.""" - return Path(tmp_flow_cells_directory, tmp_flow_cell_name_ready_for_demultiplexing_bcl2fastq) - - -# Temporary demultiplexed runs fixtures -@pytest.fixture(name="tmp_demultiplexed_runs_directory") -def tmp_demultiplexed_flow_cells_directory(tmp_path: Path, demultiplexed_runs: Path) -> Path: - """Return the path to a temporary demultiplex-runs directory. - Generates a copy of the original demultiplexed-runs - """ - original_dir = demultiplexed_runs - tmp_dir = Path(tmp_path, "demultiplexed-runs") - return Path(shutil.copytree(original_dir, tmp_dir)) - - -@pytest.fixture(name="tmp_demultiplexed_runs_bcl2fastq_directory") -def tmp_demultiplexed_runs_bcl2fastq_directory( - tmp_demultiplexed_runs_directory: Path, bcl2fastq_flow_cell_dir: Path -) -> Path: - """Return the path to a temporary demultiplex-runs bcl2fastq flow cell directory.""" - return Path(tmp_demultiplexed_runs_directory, bcl2fastq_flow_cell_dir.name) - - -@pytest.fixture(name="tmp_bcl2fastq_flow_cell") -def tmp_bcl2fastq_flow_cell( - tmp_demultiplexed_runs_bcl2fastq_directory: Path, -) -> FlowCellDirectoryData: - """Create a flow cell object with flow cell that is demultiplexed.""" - return FlowCellDirectoryData( - flow_cell_path=tmp_demultiplexed_runs_bcl2fastq_directory, - bcl_converter=BclConverter.BCL2FASTQ, - ) - - -@pytest.fixture -def novaseq6000_flow_cell( - tmp_flow_cells_directory_malformed_sample_sheet: Path, -) -> FlowCellDirectoryData: - """Return a NovaSeq6000 flow cell.""" - return FlowCellDirectoryData( - flow_cell_path=tmp_flow_cells_directory_malformed_sample_sheet, - bcl_converter=BclConverter.BCLCONVERT, - ) - - -@pytest.fixture(name="tmp_bcl_convert_flow_cell") -def tmp_bcl_convert_flow_cell( - tmp_flow_cell_directory_bclconvert: Path, -) -> FlowCellDirectoryData: - """Create a flow cell object with flow cell that is demultiplexed.""" - return FlowCellDirectoryData( - flow_cell_path=tmp_flow_cell_directory_bclconvert, - bcl_converter=BclConverter.DRAGEN, - ) - - -@pytest.fixture(name="tmp_demultiplexed_runs_not_finished_directory") -def tmp_demultiplexed_runs_not_finished_flow_cells_directory( - tmp_path: Path, demux_results_not_finished_dir: Path -) -> Path: - """ - Return a temporary demultiplex-runs-unfinished path with an unfinished flow cell directory. - Generates a copy of the original demultiplexed-runs-unfinished directory. - """ - original_dir = demux_results_not_finished_dir - tmp_dir = Path(tmp_path, "demultiplexed-runs-unfinished") - return Path(shutil.copytree(original_dir, tmp_dir)) - - -@pytest.fixture(name="demultiplexed_runs_unfinished_bcl2fastq_flow_cell_directory") -def demultiplexed_runs_bcl2fastq_flow_cell_directory( - tmp_demultiplexed_runs_not_finished_directory: Path, - bcl2fastq_flow_cell_full_name: str, -) -> Path: - """Copy the content of a demultiplexed but not finished directory to a temporary location.""" - return Path(tmp_demultiplexed_runs_not_finished_directory, bcl2fastq_flow_cell_full_name) - - -@pytest.fixture(name="tmp_unfinished_bcl2fastq_flow_cell") -def unfinished_bcl2fastq_flow_cell( - demultiplexed_runs_unfinished_bcl2fastq_flow_cell_directory: Path, - bcl2fastq_flow_cell_full_name: str, -) -> FlowCellDirectoryData: - """Copy the content of a demultiplexed but not finished directory to a temporary location.""" - return FlowCellDirectoryData( - flow_cell_path=demultiplexed_runs_unfinished_bcl2fastq_flow_cell_directory, - bcl_converter=BclConverter.BCL2FASTQ, - ) - - -@pytest.fixture(name="sample_sheet_context") -def sample_sheet_context( - cg_context: CGConfig, lims_api: LimsAPI, populated_housekeeper_api: HousekeeperAPI -) -> CGConfig: - """Return cg context with added Lims and Housekeeper API.""" - cg_context.lims_api_ = lims_api - cg_context.housekeeper_api_ = populated_housekeeper_api - return cg_context - - -@pytest.fixture -def bcl_convert_sample_sheet_creator( - bcl_convert_flow_cell: FlowCellDirectoryData, - lims_novaseq_bcl_convert_samples: list[FlowCellSampleBCLConvert], -) -> SampleSheetCreatorBCLConvert: - """Returns a sample sheet creator for version 2 sample sheets with dragen format.""" - return SampleSheetCreatorBCLConvert( - flow_cell=bcl_convert_flow_cell, - lims_samples=lims_novaseq_bcl_convert_samples, - ) - - -@pytest.fixture(scope="session") -def bcl_convert_demultiplexed_flow_cell_sample_internal_ids() -> list[str]: - """ - Sample id:s present in sample sheet for dummy flow cell demultiplexed with BCL Convert in - cg/tests/fixtures/apps/demultiplexing/demultiplexed-runs/230504_A00689_0804_BHY7FFDRX2. - """ - return ["ACC11927A2", "ACC11927A5"] - - -@pytest.fixture(scope="session") -def bcl2fastq_demultiplexed_flow_cell_sample_internal_ids() -> list[str]: - """ - Sample id:s present in sample sheet for dummy flow cell demultiplexed with BCL Convert in - cg/tests/fixtures/apps/demultiplexing/demultiplexed-runs/170407_A00689_0209_BHHKVCALXX. - """ - return ["SVE2528A1"] - - -@pytest.fixture(scope="session") -def flow_cell_name_demultiplexed_with_bcl2fastq() -> str: - """Return the name of a flow cell that has been demultiplexed with BCL2Fastq.""" - return "HHKVCALXX" - - -@pytest.fixture(scope="session") -def flow_cell_directory_name_demultiplexed_with_bcl2fastq( - flow_cell_name_demultiplexed_with_bcl2fastq: str, -): - """Return the name of a flow cell directory that has been demultiplexed with BCL2Fastq.""" - return f"170407_ST-E00198_0209_B{flow_cell_name_demultiplexed_with_bcl2fastq}" - - -@pytest.fixture(scope="session") -def flow_cell_name_demultiplexed_with_bcl_convert() -> str: - return "HY7FFDRX2" - - -@pytest.fixture(scope="session") -def flow_cell_directory_name_demultiplexed_with_bcl_convert( - flow_cell_name_demultiplexed_with_bcl_convert: str, -): - return f"230504_A00689_0804_B{flow_cell_name_demultiplexed_with_bcl_convert}" - - -# Fixtures for test demultiplex flow cell -@pytest.fixture -def tmp_empty_demultiplexed_runs_directory(tmp_demultiplexed_runs_directory) -> Path: - return Path(tmp_demultiplexed_runs_directory, "empty") - - -@pytest.fixture -def store_with_demultiplexed_samples( - store: Store, - helpers: StoreHelpers, - bcl_convert_demultiplexed_flow_cell_sample_internal_ids: list[str], - bcl2fastq_demultiplexed_flow_cell_sample_internal_ids: list[str], - flow_cell_name_demultiplexed_with_bcl2fastq: str, - flow_cell_name_demultiplexed_with_bcl_convert: str, -) -> Store: - """Return a store with samples that have been demultiplexed with BCL Convert and BCL2Fastq.""" - helpers.add_flow_cell( - store, flow_cell_name_demultiplexed_with_bcl_convert, sequencer_type="novaseq" - ) - helpers.add_flow_cell( - store, flow_cell_name_demultiplexed_with_bcl2fastq, sequencer_type="hiseqx" - ) - for i, sample_internal_id in enumerate(bcl_convert_demultiplexed_flow_cell_sample_internal_ids): - helpers.add_sample(store, internal_id=sample_internal_id, name=f"sample_bcl_convert_{i}") - helpers.add_sample_lane_sequencing_metrics( - store, - sample_internal_id=sample_internal_id, - flow_cell_name=flow_cell_name_demultiplexed_with_bcl_convert, - ) - - for i, sample_internal_id in enumerate(bcl2fastq_demultiplexed_flow_cell_sample_internal_ids): - helpers.add_sample(store, internal_id=sample_internal_id, name=f"sample_bcl2fastq_{i}") - helpers.add_sample_lane_sequencing_metrics( - store, - sample_internal_id=sample_internal_id, - flow_cell_name=flow_cell_name_demultiplexed_with_bcl2fastq, - ) - return store - - -@pytest.fixture -def demultiplexing_context_for_demux( - demultiplexing_api_for_demux: DemultiplexingAPI, - cg_context: CGConfig, - store_with_demultiplexed_samples: Store, -) -> CGConfig: - """Return cg context with a demultiplex context.""" - cg_context.demultiplex_api_ = demultiplexing_api_for_demux - cg_context.housekeeper_api_ = demultiplexing_api_for_demux.hk_api - cg_context.status_db_ = store_with_demultiplexed_samples - return cg_context - - -@pytest.fixture(name="demultiplex_context") -def demultiplex_context( - demultiplexing_api: DemultiplexingAPI, - real_housekeeper_api: HousekeeperAPI, - cg_context: CGConfig, - store_with_demultiplexed_samples: Store, -) -> CGConfig: - """Return cg context with a demultiplex context.""" - cg_context.demultiplex_api_ = demultiplexing_api - cg_context.housekeeper_api_ = real_housekeeper_api - cg_context.status_db_ = store_with_demultiplexed_samples - return cg_context - - -@pytest.fixture(name="demultiplex_configs_for_demux") -def demultiplex_configs_for_demux( - tmp_flow_cells_demux_all_directory: Path, - tmp_empty_demultiplexed_runs_directory: Path, -) -> dict: - """Return demultiplex configs.""" - return { - "flow_cells_dir": tmp_flow_cells_demux_all_directory.as_posix(), - "demultiplexed_flow_cells_dir": tmp_empty_demultiplexed_runs_directory.as_posix(), - "demultiplex": {"slurm": {"account": "test", "mail_user": "testuser@github.se"}}, - } - - -@pytest.fixture(name="demultiplex_configs") -def demultiplex_configs( - tmp_flow_cells_directory: Path, - tmp_demultiplexed_runs_directory: Path, -) -> dict: - """Return demultiplex configs.""" - return { - "flow_cells_dir": tmp_flow_cells_directory.as_posix(), - "demultiplexed_flow_cells_dir": tmp_demultiplexed_runs_directory.as_posix(), - "demultiplex": {"slurm": {"account": "test", "mail_user": "testuser@github.se"}}, - } - - -@pytest.fixture(name="demultiplexing_api_for_demux") -def demultiplexing_api_for_demux( - demultiplex_configs_for_demux: dict, - sbatch_process: Process, - populated_housekeeper_api: HousekeeperAPI, -) -> DemultiplexingAPI: - """Return demultiplex API.""" - demux_api = DemultiplexingAPI( - config=demultiplex_configs_for_demux, - housekeeper_api=populated_housekeeper_api, - ) - demux_api.slurm_api.process = sbatch_process - return demux_api - - -@pytest.fixture -def demultiplexing_api( - demultiplex_configs: dict, sbatch_process: Process, populated_housekeeper_api: HousekeeperAPI -) -> DemultiplexingAPI: - """Return demultiplex API.""" - demux_api = DemultiplexingAPI( - config=demultiplex_configs, housekeeper_api=populated_housekeeper_api - ) - demux_api.slurm_api.process = sbatch_process - return demux_api - - -@pytest.fixture(name="novaseq6000_bcl_convert_sample_sheet_path") -def novaseq6000_sample_sheet_path() -> Path: - """Return the path to a NovaSeq 6000 BCL convert sample sheet.""" - return Path( - "tests", - "fixtures", - "apps", - "sequencing_metrics_parser", - "230622_A00621_0864_AHY7FFDRX2", - "Unaligned", - "Reports", - "SampleSheet.csv", - ) - - -@pytest.fixture(scope="session") -def demultiplex_fixtures(apps_dir: Path) -> Path: - """Return the path to the demultiplex fixture directory.""" - return Path(apps_dir, "demultiplexing") - - -@pytest.fixture(scope="session") -def raw_lims_sample_dir(demultiplex_fixtures: Path) -> Path: - """Return the path to the raw samples fixture directory.""" - return Path(demultiplex_fixtures, "raw_lims_samples") - - -@pytest.fixture(scope="session") -def run_parameters_dir(demultiplex_fixtures: Path) -> Path: - """Return the path to the run parameters fixture directory.""" - return Path(demultiplex_fixtures, "run_parameters") - - -@pytest.fixture(scope="session") -def demultiplexed_runs(demultiplex_fixtures: Path) -> Path: - """Return the path to the demultiplexed flow cells fixture directory.""" - return Path(demultiplex_fixtures, "demultiplexed-runs") - - -@pytest.fixture(scope="session") -def flow_cells_dir(demultiplex_fixtures: Path) -> Path: - """Return the path to the sequenced flow cells fixture directory.""" - return Path(demultiplex_fixtures, DemultiplexingDirsAndFiles.FLOW_CELLS_DIRECTORY_NAME) - - -@pytest.fixture(scope="session") -def nanopore_flow_cells_dir(demultiplex_fixtures: Path) -> Path: - """Return the path to the sequenced flow cells fixture directory.""" - return Path(demultiplex_fixtures, NanoporeDirsAndFiles.DATA_DIRECTORY) - - -@pytest.fixture(scope="session") -def flow_cells_demux_all_dir(demultiplex_fixtures: Path) -> Path: - """Return the path to the sequenced flow cells fixture directory.""" - return Path(demultiplex_fixtures, "flow_cells_demux_all") - - -@pytest.fixture(scope="session") -def demux_results_not_finished_dir(demultiplex_fixtures: Path) -> Path: - """Return the path to a dir with demultiplexing results where nothing has been cleaned.""" - return Path(demultiplex_fixtures, "demultiplexed-runs-unfinished") - - -@pytest.fixture -def novaseq_6000_post_1_5_kits_flow_cell(tmp_flow_cells_directory: Path) -> Path: - return Path(tmp_flow_cells_directory, "230912_A00187_1009_AHK33MDRX3") - - -@pytest.fixture() -def novaseq_6000_post_1_5_kits_flow_cell_data(flow_cells_dir: Path) -> FlowCellDirectoryData: - return FlowCellDirectoryData(Path(flow_cells_dir, "230912_A00187_1009_AHK33MDRX3")) - - -@pytest.fixture -def novaseq_6000_post_1_5_kits_correct_sample_sheet( - novaseq_6000_post_1_5_kits_flow_cell: Path, -) -> Path: - return Path(novaseq_6000_post_1_5_kits_flow_cell, "CorrectSampleSheet.csv") - - -@pytest.fixture -def novaseq_6000_post_1_5_kits_raw_lims_samples( - novaseq_6000_post_1_5_kits_flow_cell: Path, -) -> Path: - return Path(novaseq_6000_post_1_5_kits_flow_cell, "HK33MDRX3_raw.json") - - -@pytest.fixture -def novaseq_6000_post_1_5_kits_lims_samples( - novaseq_6000_post_1_5_kits_raw_lims_samples: Path, -) -> list[FlowCellSampleBCLConvert]: - return [ - FlowCellSampleBCLConvert.model_validate(sample) - for sample in read_json(novaseq_6000_post_1_5_kits_raw_lims_samples) - ] - - -@pytest.fixture() -def novaseq_6000_pre_1_5_kits_flow_cell_data(flow_cells_dir: Path) -> FlowCellDirectoryData: - return FlowCellDirectoryData(Path(flow_cells_dir, "190927_A00689_0069_BHLYWYDSXX")) - - -@pytest.fixture -def novaseq_6000_pre_1_5_kits_flow_cell(tmp_flow_cells_directory: Path) -> Path: - return Path(tmp_flow_cells_directory, "190927_A00689_0069_BHLYWYDSXX") - - -@pytest.fixture -def novaseq_6000_pre_1_5_kits_correct_sample_sheet( - novaseq_6000_pre_1_5_kits_flow_cell: Path, -) -> Path: - return Path(novaseq_6000_pre_1_5_kits_flow_cell, "CorrectSampleSheet.csv") - - -@pytest.fixture -def novaseq_6000_pre_1_5_kits_raw_lims_samples(novaseq_6000_pre_1_5_kits_flow_cell: Path) -> Path: - return Path(novaseq_6000_pre_1_5_kits_flow_cell, "HLYWYDSXX_raw.json") - - -@pytest.fixture -def novaseq_6000_pre_1_5_kits_lims_samples( - novaseq_6000_pre_1_5_kits_raw_lims_samples: Path, -) -> list[FlowCellSampleBCLConvert]: - return [ - FlowCellSampleBCLConvert.model_validate(sample) - for sample in read_json(novaseq_6000_pre_1_5_kits_raw_lims_samples) - ] - - -@pytest.fixture -def novaseq_x_flow_cell_directory(tmp_flow_cells_directory: Path) -> Path: - return Path(tmp_flow_cells_directory, "20231108_LH00188_0028_B22F52TLT3") - - -@pytest.fixture() -def novaseq_x_flow_cell_data(flow_cells_dir: Path) -> FlowCellDirectoryData: - return FlowCellDirectoryData(Path(flow_cells_dir, "20231108_LH00188_0028_B22F52TLT3")) - - -@pytest.fixture -def novaseq_x_correct_sample_sheet(novaseq_x_flow_cell_directory: Path) -> Path: - return Path(novaseq_x_flow_cell_directory, "CorrectSampleSheet.csv") - - -@pytest.fixture -def novaseq_x_raw_lims_samples(novaseq_x_flow_cell_directory: Path) -> Path: - return Path(novaseq_x_flow_cell_directory, "22F52TLT3_raw.json") - - -@pytest.fixture -def novaseq_x_lims_samples(novaseq_x_raw_lims_samples: Path) -> list[FlowCellSampleBCLConvert]: - return [ - FlowCellSampleBCLConvert.model_validate(sample) - for sample in read_json(novaseq_x_raw_lims_samples) - ] - - -@pytest.fixture(scope="session") -def hiseq_x_single_index_flow_cell_name() -> str: - """Return the full name of a HiSeqX flow cell with only one index.""" - return "170517_ST-E00266_0210_BHJCFFALXX" - - -@pytest.fixture(scope="session") -def hiseq_x_dual_index_flow_cell_name() -> str: - """Return the full name of a HiSeqX flow cell with two indexes.""" - return "180508_ST-E00269_0269_AHL32LCCXY" - - -@pytest.fixture(scope="session") -def hiseq_2500_dual_index_flow_cell_name() -> str: - """Return the full name of a HiSeq2500 flow cell with double indexes.""" - return "181005_D00410_0735_BHM2LNBCX2" - - -@pytest.fixture(scope="session") -def hiseq_2500_custom_index_flow_cell_name() -> str: - """Return the full name of a HiSeq2500 flow cell with double indexes.""" - return "180509_D00450_0598_BHGYFNBCX2" - - -@pytest.fixture(scope="session") -def bcl2fastq_flow_cell_full_name() -> str: - """Return full flow cell name.""" - return "201203_D00483_0200_AHVKJCDRXX" - - -@pytest.fixture(scope="session") -def bcl_convert_flow_cell_full_name() -> str: - """Return the full name of a bcl_convert flow cell.""" - return "211101_A00187_0615_AHLG5GDRZZ" - - -@pytest.fixture(scope="session") -def novaseq_x_flow_cell_full_name() -> str: - """Return the full name of a NovaSeqX flow cell.""" - return "20230508_LH00188_0003_A22522YLT3" - - -@pytest.fixture(scope="session") -def novaseq_x_manifest_file(novaseq_x_flow_cell_dir: Path) -> Path: - """Return the path to a NovaSeqX manifest file.""" - return Path(novaseq_x_flow_cell_dir, "Manifest.tsv") - - -@pytest.fixture(scope="session") -def hiseq_x_single_index_flow_cell_dir( - flow_cells_dir: Path, hiseq_x_single_index_flow_cell_name: str -) -> Path: - """Return the path to a HiSeqX flow cell.""" - return Path(flow_cells_dir, hiseq_x_single_index_flow_cell_name) - - -@pytest.fixture(scope="session") -def hiseq_x_dual_index_flow_cell_dir( - flow_cells_dir: Path, hiseq_x_dual_index_flow_cell_name: str -) -> Path: - """Return the path to a HiSeqX flow cell.""" - return Path(flow_cells_dir, hiseq_x_dual_index_flow_cell_name) - - -@pytest.fixture(scope="session") -def hiseq_2500_dual_index_flow_cell_dir( - flow_cells_dir: Path, hiseq_2500_dual_index_flow_cell_name: str -) -> Path: - """Return the path to a HiSeq2500 flow cell.""" - return Path(flow_cells_dir, hiseq_2500_dual_index_flow_cell_name) - - -@pytest.fixture(scope="session") -def hiseq_2500_custom_index_flow_cell_dir( - flow_cells_dir: Path, hiseq_2500_custom_index_flow_cell_name: str -) -> Path: - """Return the path to a HiSeq2500 flow cell.""" - return Path(flow_cells_dir, hiseq_2500_custom_index_flow_cell_name) - - -@pytest.fixture(scope="session") -def bcl2fastq_flow_cell_dir(flow_cells_dir: Path, bcl2fastq_flow_cell_full_name: str) -> Path: - """Return the path to the bcl2fastq flow cell demultiplex fixture directory.""" - return Path(flow_cells_dir, bcl2fastq_flow_cell_full_name) - - -@pytest.fixture(scope="session") -def bcl_convert_flow_cell_dir(flow_cells_dir: Path, bcl_convert_flow_cell_full_name: str) -> Path: - """Return the path to the bcl_convert flow cell demultiplex fixture directory.""" - return Path(flow_cells_dir, bcl_convert_flow_cell_full_name) - - -@pytest.fixture(scope="session") -def novaseq_x_flow_cell_dir(flow_cells_dir: Path, novaseq_x_flow_cell_full_name: str) -> Path: - """Return the path to the NovaSeqX flow cell demultiplex fixture directory.""" - return Path(flow_cells_dir, novaseq_x_flow_cell_full_name) - - -@pytest.fixture -def hiseq_x_single_index_bcl_convert_lims_samples( - hiseq_x_single_index_flow_cell_dir: Path, -) -> list[FlowCellSampleBCLConvert]: - """Return a list of BCLConvert samples from a HiSeqX single index flow cell.""" - path = Path( - hiseq_x_single_index_flow_cell_dir, f"HJCFFALXX_bcl_convert_raw{FileExtensions.JSON}" - ) - return [FlowCellSampleBCLConvert.model_validate(sample) for sample in read_json(path)] - - -@pytest.fixture -def hiseq_x_dual_index_bcl_convert_lims_samples( - hiseq_x_dual_index_flow_cell_dir: Path, -) -> list[FlowCellSampleBCLConvert]: - """Return a list of BCLConvert samples from a HiSeqX dual index flow cell.""" - path = Path(hiseq_x_dual_index_flow_cell_dir, f"HL32LCCXY_bcl_convert_raw{FileExtensions.JSON}") - return [FlowCellSampleBCLConvert.model_validate(sample) for sample in read_json(path)] - - -@pytest.fixture -def hiseq_2500_dual_index_bcl_convert_lims_samples( - hiseq_2500_dual_index_flow_cell_dir: Path, -) -> list[FlowCellSampleBCLConvert]: - """Return a list of BCLConvert samples from a HiSeq2500 dual index flow cell.""" - path = Path(hiseq_2500_dual_index_flow_cell_dir, "HM2LNBCX2_bcl_convert_raw.json") - return [FlowCellSampleBCLConvert.model_validate(sample) for sample in read_json(path)] - - -@pytest.fixture -def hiseq_2500_custom_index_bcl_convert_lims_samples( - hiseq_2500_custom_index_flow_cell_dir: Path, -) -> list[FlowCellSampleBCLConvert]: - """Return a list of BCLConvert samples from a HiSeq2500 custom index flow cell.""" - path = Path(hiseq_2500_custom_index_flow_cell_dir, "HGYFNBCX2_bcl_convert_raw.json") - return [FlowCellSampleBCLConvert.model_validate(sample) for sample in read_json(path)] - - -@pytest.fixture(scope="session") -def novaseq_bcl2fastq_sample_sheet_path(bcl2fastq_flow_cell_dir: Path) -> Path: - """Return the path to a NovaSeq6000 Bcl2fastq sample sheet.""" - return Path(bcl2fastq_flow_cell_dir, DemultiplexingDirsAndFiles.SAMPLE_SHEET_FILE_NAME) - - -@pytest.fixture(scope="session") -def novaseq_bcl_convert_sample_sheet_path(bcl_convert_flow_cell_dir: Path) -> Path: - """Return the path to a NovaSeq6000 bcl_convert sample sheet.""" - return Path(bcl_convert_flow_cell_dir, DemultiplexingDirsAndFiles.SAMPLE_SHEET_FILE_NAME) - - -@pytest.fixture(scope="session") -def run_parameters_wrong_instrument(run_parameters_dir: Path) -> Path: - """Return a NovaSeqX run parameters file path with a wrong instrument value.""" - return Path(run_parameters_dir, "RunParameters_novaseq_X_wrong_instrument.xml") - - -@pytest.fixture(scope="session") -def hiseq_x_single_index_run_parameters_path( - hiseq_x_single_index_flow_cell_dir: Path, -) -> Path: - """Return the path to a HiSeqX run parameters file with single index.""" - return Path( - hiseq_x_single_index_flow_cell_dir, DemultiplexingDirsAndFiles.RUN_PARAMETERS_CAMEL_CASE - ) +@pytest.fixture(scope="session") +def analysis_dir(fixtures_dir: Path) -> Path: + """Return the path to the analysis dir.""" + return Path(fixtures_dir, "analysis") @pytest.fixture(scope="session") -def hiseq_x_dual_index_run_parameters_path( - hiseq_x_dual_index_flow_cell_dir: Path, -) -> Path: - """Return the path to a HiSeqX run parameters file with dual index.""" - return Path( - hiseq_x_dual_index_flow_cell_dir, DemultiplexingDirsAndFiles.RUN_PARAMETERS_CAMEL_CASE - ) +def microsalt_analysis_dir(analysis_dir: Path) -> Path: + """Return the path to the analysis dir.""" + return Path(analysis_dir, "microsalt") @pytest.fixture(scope="session") -def hiseq_2500_dual_index_run_parameters_path( - hiseq_2500_dual_index_flow_cell_dir: Path, -) -> Path: - """Return the path to a HiSeq2500 run parameters file with dual index.""" - return Path( - hiseq_2500_dual_index_flow_cell_dir, DemultiplexingDirsAndFiles.RUN_PARAMETERS_CAMEL_CASE - ) +def apps_dir(fixtures_dir: Path) -> Path: + """Return the path to the apps dir.""" + return Path(fixtures_dir, "apps") @pytest.fixture(scope="session") -def hiseq_2500_custom_index_run_parameters_path( - hiseq_2500_custom_index_flow_cell_dir: Path, -) -> Path: - """Return the path to a HiSeq2500 run parameters file with custom index.""" - return Path( - hiseq_2500_custom_index_flow_cell_dir, DemultiplexingDirsAndFiles.RUN_PARAMETERS_CAMEL_CASE - ) +def cgweb_orders_dir(fixtures_dir: Path) -> Path: + """Return the path to the cgweb_orders dir.""" + return Path(fixtures_dir, "cgweb_orders") @pytest.fixture(scope="session") -def novaseq_6000_run_parameters_path(bcl2fastq_flow_cell_dir: Path) -> Path: - """Return the path to a NovaSeq6000 run parameters file.""" - return Path(bcl2fastq_flow_cell_dir, DemultiplexingDirsAndFiles.RUN_PARAMETERS_PASCAL_CASE) +def data_dir(fixtures_dir: Path) -> Path: + """Return the path to the data dir.""" + return Path(fixtures_dir, "data") @pytest.fixture -def novaseq_6000_run_parameters_pre_1_5_kits_path( - novaseq_6000_pre_1_5_kits_flow_cell: Path, -) -> Path: - """Return the path to a NovaSeq6000 pre 1.5 kit run parameters file.""" - return Path( - novaseq_6000_pre_1_5_kits_flow_cell, - DemultiplexingDirsAndFiles.RUN_PARAMETERS_PASCAL_CASE, - ) +def fastq_dir(demultiplex_fixtures: Path) -> Path: + """Return the path to the fastq files dir.""" + return Path(demultiplex_fixtures, "fastq") @pytest.fixture -def novaseq_6000_run_parameters_post_1_5_kits_path( - novaseq_6000_post_1_5_kits_flow_cell: Path, -) -> Path: - """Return the path to a NovaSeq6000 post 1.5 kit run parameters file.""" - return Path( - novaseq_6000_post_1_5_kits_flow_cell, - DemultiplexingDirsAndFiles.RUN_PARAMETERS_PASCAL_CASE, - ) +def spring_dir(demultiplex_fixtures: Path) -> Path: + """Return the path to the fastq files dir.""" + return Path(demultiplex_fixtures, "spring") -@pytest.fixture(scope="session") -def novaseq_x_run_parameters_path(novaseq_x_flow_cell_dir: Path) -> Path: - """Return the path to a NovaSeqX run parameters file.""" - return Path(novaseq_x_flow_cell_dir, DemultiplexingDirsAndFiles.RUN_PARAMETERS_PASCAL_CASE) +@pytest.fixture +def project_dir(tmpdir_factory) -> Generator[Path, None, None]: + """Path to a temporary directory where intermediate files can be stored.""" + yield Path(tmpdir_factory.mktemp("data")) -@pytest.fixture(scope="function") -def run_parameters_hiseq_different_index(run_parameters_dir: Path) -> RunParametersHiSeq: - """Return a HiSeq RunParameters object with different index cycles.""" - path = Path(run_parameters_dir, "RunParameters_hiseq_2500_different_index_cycles.xml") - return RunParametersHiSeq(run_parameters_path=path) +@pytest.fixture +def tmp_file(project_dir) -> Path: + """Return a temp file path.""" + return Path(project_dir, "test") -@pytest.fixture(scope="function") -def run_parameters_novaseq_6000_different_index( - run_parameters_dir: Path, -) -> RunParametersNovaSeq6000: - """Return a NovaSeq6000 RunParameters object with different index cycles.""" - path = Path(run_parameters_dir, "RunParameters_novaseq_6000_different_index_cycles.xml") - return RunParametersNovaSeq6000(run_parameters_path=path) +@pytest.fixture +def non_existing_file_path(project_dir: Path) -> Path: + """Return the path to a non-existing file.""" + return Path(project_dir, "a_file.txt") -@pytest.fixture(scope="function") -def run_parameters_novaseq_x_different_index(run_parameters_dir: Path) -> RunParametersNovaSeqX: - """Return a NovaSeqX RunParameters object with different index cycles.""" - path = Path(run_parameters_dir, "RunParameters_novaseq_X_different_index_cycles.xml") - return RunParametersNovaSeqX(run_parameters_path=path) +@pytest.fixture(scope="session") +def content() -> str: + """Return some content for a file.""" + return ( + "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt" + " ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ull" + "amco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehende" + "rit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaec" + "at cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum." + ) -@pytest.fixture(scope="module") -def run_parameters_missing_versions_path( - run_parameters_dir: Path, -) -> Path: - """Return a NovaSeq6000 run parameters path without software and reagent kit versions.""" - return Path(run_parameters_dir, "RunParameters_novaseq_no_software_nor_reagent_version.xml") +@pytest.fixture +def filled_file(non_existing_file_path: Path, content: str) -> Path: + """Return the path to a existing file with some content.""" + with open(non_existing_file_path, "w") as outfile: + outfile.write(content) + return non_existing_file_path @pytest.fixture(scope="session") -def hiseq_x_single_index_run_parameters( - hiseq_x_single_index_run_parameters_path: Path, -) -> RunParametersHiSeq: - """Return a HiSeqX run parameters object with single index.""" - return RunParametersHiSeq(run_parameters_path=hiseq_x_single_index_run_parameters_path) +def orderforms(fixtures_dir: Path) -> Path: + """Return the path to the directory with order forms.""" + return Path(fixtures_dir, "orderforms") -@pytest.fixture(scope="session") -def hiseq_x_dual_index_run_parameters( - hiseq_x_dual_index_run_parameters_path: Path, -) -> RunParametersHiSeq: - """Return a HiSeqX run parameters object with dual index.""" - return RunParametersHiSeq(run_parameters_path=hiseq_x_dual_index_run_parameters_path) +@pytest.fixture +def hk_file(filled_file: Path, case_id: str) -> File: + """Return a housekeeper File object.""" + return File(id=case_id, path=filled_file.as_posix()) -@pytest.fixture(scope="session") -def hiseq_2500_dual_index_run_parameters( - hiseq_2500_dual_index_run_parameters_path: Path, -) -> RunParametersHiSeq: - """Return a HiSeq2500 run parameters object with dual index.""" - return RunParametersHiSeq(run_parameters_path=hiseq_2500_dual_index_run_parameters_path) +@pytest.fixture +def mip_dna_store_files(apps_dir: Path) -> Path: + """Return the path to the directory with mip dna store files.""" + return Path(apps_dir, "mip", "dna", "store") -@pytest.fixture(scope="session") -def hiseq_2500_custom_index_run_parameters( - hiseq_2500_custom_index_run_parameters_path: Path, -) -> RunParametersHiSeq: - """Return a HiSeq2500 run parameters object with custom index.""" - return RunParametersHiSeq(run_parameters_path=hiseq_2500_custom_index_run_parameters_path) +@pytest.fixture +def case_qc_sample_info_path(mip_dna_store_files: Path) -> Path: + """Return path to case_qc_sample_info.yaml.""" + return Path(mip_dna_store_files, "case_qc_sample_info.yaml") -@pytest.fixture(scope="session") -def novaseq_6000_run_parameters( - novaseq_6000_run_parameters_path: Path, -) -> RunParametersNovaSeq6000: - """Return a NovaSeq6000 run parameters object.""" - return RunParametersNovaSeq6000(run_parameters_path=novaseq_6000_run_parameters_path) +@pytest.fixture +def delivery_report_html(mip_dna_store_files: Path) -> Path: + """Return the path to a qc metrics deliverables file with case data.""" + return Path(mip_dna_store_files, "empty_delivery_report.html") @pytest.fixture -def novaseq_6000_run_parameters_pre_1_5_kits( - novaseq_6000_run_parameters_pre_1_5_kits_path: Path, -) -> RunParametersNovaSeq6000: - """Return a NovaSeq6000 run parameters pre 1.5 kit object.""" - return RunParametersNovaSeq6000( - run_parameters_path=novaseq_6000_run_parameters_pre_1_5_kits_path - ) +def mip_deliverables_file(mip_dna_store_files: Path) -> Path: + """Fixture for general deliverables file in mip.""" + return Path(mip_dna_store_files, "case_id_deliverables.yaml") @pytest.fixture -def novaseq_6000_run_parameters_post_1_5_kits(novaseq_6000_run_parameters_post_1_5_kits_path: Path): - """Return a NovaSeq6000 run parameters post 1.5 kit object.""" - return RunParametersNovaSeq6000( - run_parameters_path=novaseq_6000_run_parameters_post_1_5_kits_path - ) +def case_qc_metrics_deliverables(apps_dir: Path) -> Path: + """Return the path to a qc metrics deliverables file with case data.""" + return Path(apps_dir, "mip", "case_metrics_deliverables.yaml") -@pytest.fixture(scope="session") -def novaseq_x_run_parameters( - novaseq_x_run_parameters_path: Path, -) -> RunParametersNovaSeqX: - """Return a NovaSeqX run parameters object.""" - return RunParametersNovaSeqX(run_parameters_path=novaseq_x_run_parameters_path) +@pytest.fixture +def mip_analysis_dir(analysis_dir: Path) -> Path: + """Return the path to the directory with mip analysis files.""" + return Path(analysis_dir, "mip") -@pytest.fixture(scope="module") -def hiseq_x_single_index_flow_cell( - hiseq_x_single_index_flow_cell_dir: Path, -) -> FlowCellDirectoryData: - """Return a single-index HiSeqX flow cell.""" - return FlowCellDirectoryData(flow_cell_path=hiseq_x_single_index_flow_cell_dir) +@pytest.fixture +def balsamic_analysis_dir(analysis_dir: Path) -> Path: + """Return the path to the directory with balsamic analysis files.""" + return Path(analysis_dir, "balsamic") -@pytest.fixture(scope="module") -def hiseq_x_dual_index_flow_cell( - hiseq_x_dual_index_flow_cell_dir: Path, -) -> FlowCellDirectoryData: - """Return a dual-index HiSeqX flow cell.""" - return FlowCellDirectoryData(flow_cell_path=hiseq_x_dual_index_flow_cell_dir) +@pytest.fixture +def balsamic_wgs_analysis_dir(balsamic_analysis_dir: Path) -> Path: + """Return the path to the directory with balsamic analysis files.""" + return Path(balsamic_analysis_dir, "tn_wgs") -@pytest.fixture(scope="module") -def hiseq_2500_dual_index_flow_cell( - hiseq_2500_dual_index_flow_cell_dir: Path, -) -> FlowCellDirectoryData: - """Return a dual-index HiSeq2500 flow cell.""" - return FlowCellDirectoryData(flow_cell_path=hiseq_2500_dual_index_flow_cell_dir) +@pytest.fixture +def mip_dna_analysis_dir(mip_analysis_dir: Path) -> Path: + """Return the path to the directory with mip dna analysis files.""" + return Path(mip_analysis_dir, "dna") -@pytest.fixture(scope="module") -def hiseq_2500_custom_index_flow_cell( - hiseq_2500_custom_index_flow_cell_dir: Path, -) -> FlowCellDirectoryData: - """Return a custom-index HiSeq2500 flow cell.""" - return FlowCellDirectoryData(flow_cell_path=hiseq_2500_custom_index_flow_cell_dir) +@pytest.fixture +def rnafusion_analysis_dir(analysis_dir: Path) -> Path: + """Return the path to the directory with rnafusion analysis files.""" + return Path(analysis_dir, "rnafusion") -@pytest.fixture(scope="session") -def bcl2fastq_flow_cell(bcl2fastq_flow_cell_dir: Path) -> FlowCellDirectoryData: - """Create a flow cell object with flow cell that is demultiplexed.""" - return FlowCellDirectoryData( - flow_cell_path=bcl2fastq_flow_cell_dir, bcl_converter=BclConverter.BCL2FASTQ - ) +@pytest.fixture +def sample_cram(mip_dna_analysis_dir: Path) -> Path: + """Return the path to the cram file for a sample.""" + return Path(mip_dna_analysis_dir, "adm1.cram") -@pytest.fixture(scope="session") -def novaseq_flow_cell_demultiplexed_with_bcl2fastq( - bcl_convert_flow_cell_dir: Path, -) -> FlowCellDirectoryData: - """Return a Novaseq6000 flow cell object demultiplexed using Bcl2fastq.""" - return FlowCellDirectoryData( - flow_cell_path=bcl_convert_flow_cell_dir, bcl_converter=BclConverter.BCL2FASTQ - ) +@pytest.fixture(name="father_sample_cram") +def father_sample_cram( + mip_dna_analysis_dir: Path, + father_sample_id: str, +) -> Path: + """Return the path to the cram file for the father sample.""" + return Path(mip_dna_analysis_dir, father_sample_id + FileExtensions.CRAM) -@pytest.fixture(scope="module") -def bcl_convert_flow_cell(bcl_convert_flow_cell_dir: Path) -> FlowCellDirectoryData: - """Create a bcl_convert flow cell object with flow cell that is demultiplexed.""" - return FlowCellDirectoryData( - flow_cell_path=bcl_convert_flow_cell_dir, bcl_converter=BclConverter.DRAGEN - ) +@pytest.fixture(name="mother_sample_cram") +def mother_sample_cram(mip_dna_analysis_dir: Path, mother_sample_id: str) -> Path: + """Return the path to the cram file for the mother sample.""" + return Path(mip_dna_analysis_dir, mother_sample_id + FileExtensions.CRAM) -@pytest.fixture(scope="function") -def novaseq_6000_flow_cell(bcl_convert_flow_cell: FlowCellDirectoryData) -> FlowCellDirectoryData: - """Return a NovaSeq6000 flow cell object.""" - return bcl_convert_flow_cell +@pytest.fixture(name="sample_cram_files") +def sample_crams( + sample_cram: Path, father_sample_cram: Path, mother_sample_cram: Path +) -> list[Path]: + """Return a list of cram paths for three samples.""" + return [sample_cram, father_sample_cram, mother_sample_cram] -@pytest.fixture(scope="function") -def novaseq_x_flow_cell(novaseq_x_flow_cell_dir: Path) -> FlowCellDirectoryData: - """Create a NovaSeqX flow cell object with flow cell that is demultiplexed.""" - return FlowCellDirectoryData( - flow_cell_path=novaseq_x_flow_cell_dir, bcl_converter=BclConverter.DRAGEN - ) +@pytest.fixture(name="vcf_file") +def vcf_file(mip_dna_store_files: Path) -> Path: + """Return the path to a VCF file.""" + return Path(mip_dna_store_files, "yellowhog_clinical_selected.vcf") -@pytest.fixture(scope="session") -def bcl2fastq_flow_cell_id(bcl2fastq_flow_cell: FlowCellDirectoryData) -> str: - """Return flow cell id from bcl2fastq flow cell object.""" - return bcl2fastq_flow_cell.id +@pytest.fixture(name="fastq_file") +def fastq_file(fastq_dir: Path) -> Path: + """Return the path to a FASTQ file.""" + return Path(fastq_dir, "dummy_run_R1_001.fastq.gz") -@pytest.fixture(scope="module") -def bcl_convert_flow_cell_id(bcl_convert_flow_cell: FlowCellDirectoryData) -> str: - """Return flow cell id from bcl_convert flow cell object.""" - return bcl_convert_flow_cell.id +@pytest.fixture(name="fastq_file_father") +def fastq_file_father(fastq_dir: Path) -> Path: + """Return the path to a FASTQ file.""" + return Path(fastq_dir, "fastq_run_R1_001.fastq.gz") -@pytest.fixture(name="demultiplexing_delivery_file") -def demultiplexing_delivery_file(bcl2fastq_flow_cell: FlowCellDirectoryData) -> Path: - """Return demultiplexing delivery started file.""" - return Path(bcl2fastq_flow_cell.path, DemultiplexingDirsAndFiles.DELIVERY) +@pytest.fixture(name="spring_file") +def spring_file(spring_dir: Path) -> Path: + """Return the path to an existing spring file.""" + return Path(spring_dir, "dummy_run_001.spring") -@pytest.fixture(name="hiseq_x_tile_dir") -def hiseq_x_tile_dir(bcl2fastq_flow_cell: FlowCellDirectoryData) -> Path: - """Return HiSeqX tile dir.""" - return Path(bcl2fastq_flow_cell.path, DemultiplexingDirsAndFiles.HISEQ_X_TILE_DIR) +@pytest.fixture(name="spring_meta_data_file") +def spring_meta_data_file(spring_dir: Path) -> Path: + """Return the path to an existing spring file.""" + return Path(spring_dir, "dummy_spring_meta_data.json") -@pytest.fixture(name="lims_novaseq_samples_file") -def lims_novaseq_samples_file(raw_lims_sample_dir: Path) -> Path: - """Return the path to a file with sample info in lims format.""" - return Path(raw_lims_sample_dir, "raw_samplesheet_novaseq.json") +@pytest.fixture(name="spring_file_father") +def spring_file_father(spring_dir: Path) -> Path: + """Return the path to a second existing spring file.""" + return Path(spring_dir, "dummy_run_002.spring") -@pytest.fixture -def lims_novaseq_6000_samples_file(bcl2fastq_flow_cell_dir: Path) -> Path: - """Return the path to the file with the raw samples of HVKJCDRXX flow cell in lims format.""" - return Path(bcl2fastq_flow_cell_dir, "HVKJCDRXX_raw.json") +@pytest.fixture(name="madeline_output") +def madeline_output(apps_dir: Path) -> Path: + """Return str of path for file with Madeline output.""" + return Path(apps_dir, "madeline", "madeline.xml") -@pytest.fixture -def lims_novaseq_samples_raw(lims_novaseq_samples_file: Path) -> list[dict]: - """Return a list of raw flow cell samples.""" - return ReadFile.get_content_from_file( - file_format=FileFormat.JSON, file_path=lims_novaseq_samples_file - ) +@pytest.fixture(name="file_does_not_exist") +def file_does_not_exist() -> Path: + """Return a file path that does not exist.""" + return Path("file", "does", "not", "exist") -@pytest.fixture -def lims_novaseq_6000_sample_raw(lims_novaseq_6000_samples_file: Path) -> list[dict]: - """Return the list of raw samples from flow cell HVKJCDRXX.""" - return ReadFile.get_content_from_file( - file_format=FileFormat.JSON, file_path=lims_novaseq_6000_samples_file - ) +# Compression fixtures -@pytest.fixture(name="demultiplexed_flow_cell") -def demultiplexed_flow_cell(demultiplexed_runs: Path, bcl2fastq_flow_cell_full_name: str) -> Path: - """Return the path to a demultiplexed flow cell with bcl2fastq.""" - return Path(demultiplexed_runs, bcl2fastq_flow_cell_full_name) +@pytest.fixture(name="run_name") +def run_name() -> str: + """Return the name of a fastq run.""" + return "fastq_run" -@pytest.fixture(name="bcl_convert_demultiplexed_flow_cell") -def bcl_convert_demultiplexed_flow_cell( - demultiplexed_runs: Path, bcl_convert_flow_cell_full_name: str -) -> Path: - """Return the path to a demultiplexed flow cell with BCLConvert.""" - return Path(demultiplexed_runs, bcl_convert_flow_cell_full_name) +@pytest.fixture(name="original_fastq_data") +def original_fastq_data(fastq_dir: Path, run_name) -> CompressionData: + """Return a compression object with a path to the original fastq files.""" + return CompressionData(Path(fastq_dir, run_name)) -@pytest.fixture(name="novaseqx_demultiplexed_flow_cell") -def novaseqx_demultiplexed_flow_cell(demultiplexed_runs: Path, novaseq_x_flow_cell_full_name: str): - """Return the path to a demultiplexed NovaSeqX flow cell.""" - return Path(demultiplexed_runs, novaseq_x_flow_cell_full_name) +@pytest.fixture(name="fastq_stub") +def fastq_stub(project_dir: Path, run_name: str) -> Path: + """Creates a path to the base format of a fastq run.""" + return Path(project_dir, run_name) -@pytest.fixture() -def novaseqx_flow_cell_with_sample_sheet_no_fastq( - novaseqx_flow_cell_directory: Path, novaseqx_demultiplexed_flow_cell: Path -) -> FlowCellDirectoryData: - """Return a flow cell from a tmp dir with a sample sheet and no sample fastq files.""" - novaseqx_flow_cell_directory.mkdir(parents=True, exist_ok=True) - flow_cell = FlowCellDirectoryData(novaseqx_flow_cell_directory) - sample_sheet_path = Path( - novaseqx_demultiplexed_flow_cell, DemultiplexingDirsAndFiles.SAMPLE_SHEET_FILE_NAME - ) - flow_cell._sample_sheet_path_hk = sample_sheet_path - return flow_cell +@pytest.fixture(name="compression_object") +def compression_object(fastq_stub: Path, original_fastq_data: CompressionData) -> CompressionData: + """Creates compression data object with information about files used in fastq compression.""" + working_files: CompressionData = CompressionData(fastq_stub) + working_file_map: dict[str, str] = { + original_fastq_data.fastq_first.as_posix(): working_files.fastq_first.as_posix(), + original_fastq_data.fastq_second.as_posix(): working_files.fastq_second.as_posix(), + } + for original_file, working_file in working_file_map.items(): + shutil.copy(original_file, working_file) + return working_files # Genotype file fixture @@ -1895,6 +891,7 @@ def hk_bundle_sample_path(sample_id: str, timestamp: datetime) -> Path: def hk_bundle_data( case_id: str, bed_file: Path, + delivery_report_html: Path, timestamp_yesterday: datetime, sample_id: str, father_sample_id: str, @@ -1910,7 +907,12 @@ def hk_bundle_data( "path": bed_file.as_posix(), "archive": False, "tags": ["bed", sample_id, father_sample_id, mother_sample_id, "coverage"], - } + }, + { + "path": delivery_report_html.as_posix(), + "archive": False, + "tags": [HK_DELIVERY_REPORT_TAG], + }, ], } @@ -2128,6 +1130,40 @@ def analysis_store_single_case( yield base_store +@pytest.fixture +def store_with_demultiplexed_samples( + store: Store, + helpers: StoreHelpers, + bcl_convert_demultiplexed_flow_cell_sample_internal_ids: list[str], + bcl2fastq_demultiplexed_flow_cell_sample_internal_ids: list[str], + flow_cell_name_demultiplexed_with_bcl2fastq: str, + flow_cell_name_demultiplexed_with_bcl_convert: str, +) -> Store: + """Return a store with samples that have been demultiplexed with BCL Convert and BCL2Fastq.""" + helpers.add_flow_cell( + store, flow_cell_name_demultiplexed_with_bcl_convert, sequencer_type="novaseq" + ) + helpers.add_flow_cell( + store, flow_cell_name_demultiplexed_with_bcl2fastq, sequencer_type="hiseqx" + ) + for i, sample_internal_id in enumerate(bcl_convert_demultiplexed_flow_cell_sample_internal_ids): + helpers.add_sample(store, internal_id=sample_internal_id, name=f"sample_bcl_convert_{i}") + helpers.add_sample_lane_sequencing_metrics( + store, + sample_internal_id=sample_internal_id, + flow_cell_name=flow_cell_name_demultiplexed_with_bcl_convert, + ) + + for i, sample_internal_id in enumerate(bcl2fastq_demultiplexed_flow_cell_sample_internal_ids): + helpers.add_sample(store, internal_id=sample_internal_id, name=f"sample_bcl2fastq_{i}") + helpers.add_sample_lane_sequencing_metrics( + store, + sample_internal_id=sample_internal_id, + flow_cell_name=flow_cell_name_demultiplexed_with_bcl2fastq, + ) + return store + + @pytest.fixture(name="collaboration_id") def collaboration_id() -> str: """Return a default customer group.""" @@ -3245,7 +2281,6 @@ def rnafusion_context( case_id_not_enough_reads: str, sample_id_not_enough_reads: str, total_sequenced_reads_not_pass: int, - timestamp_yesterday: datetime, ) -> CGConfig: """context to use in cli""" cg_context.housekeeper_api_ = nf_analysis_housekeeper @@ -3469,6 +2504,7 @@ def nf_analysis_housekeeper( helpers: StoreHelpers, mock_fastq_files: list[Path], sample_id: str, + timestamp_now: datetime, ): """Create populated Housekeeper sample bundle mock.""" diff --git a/tests/fixture_plugins/__init__.py b/tests/fixture_plugins/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/fixture_plugins/demultiplex_fixtures/__init__.py b/tests/fixture_plugins/demultiplex_fixtures/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/fixture_plugins/demultiplex_fixtures/flow_cell_fixtures.py b/tests/fixture_plugins/demultiplex_fixtures/flow_cell_fixtures.py new file mode 100644 index 0000000000..09213170bd --- /dev/null +++ b/tests/fixture_plugins/demultiplex_fixtures/flow_cell_fixtures.py @@ -0,0 +1,172 @@ +"""Fixtures for flow cell objects.""" +from pathlib import Path + +import pytest + +from cg.constants.demultiplexing import BclConverter, DemultiplexingDirsAndFiles +from cg.models.flow_cell.flow_cell import FlowCellDirectoryData + +# Functional flow cells + + +@pytest.fixture(scope="module") +def hiseq_x_single_index_flow_cell( + hiseq_x_single_index_flow_cell_dir: Path, +) -> FlowCellDirectoryData: + """Return a single-index HiSeqX flow cell.""" + return FlowCellDirectoryData(flow_cell_path=hiseq_x_single_index_flow_cell_dir) + + +@pytest.fixture(scope="module") +def hiseq_x_dual_index_flow_cell( + hiseq_x_dual_index_flow_cell_dir: Path, +) -> FlowCellDirectoryData: + """Return a dual-index HiSeqX flow cell.""" + return FlowCellDirectoryData(flow_cell_path=hiseq_x_dual_index_flow_cell_dir) + + +@pytest.fixture(scope="module") +def hiseq_2500_dual_index_flow_cell( + hiseq_2500_dual_index_flow_cell_dir: Path, +) -> FlowCellDirectoryData: + """Return a dual-index HiSeq2500 flow cell.""" + return FlowCellDirectoryData(flow_cell_path=hiseq_2500_dual_index_flow_cell_dir) + + +@pytest.fixture(scope="module") +def hiseq_2500_custom_index_flow_cell( + hiseq_2500_custom_index_flow_cell_dir: Path, +) -> FlowCellDirectoryData: + """Return a custom-index HiSeq2500 flow cell.""" + return FlowCellDirectoryData(flow_cell_path=hiseq_2500_custom_index_flow_cell_dir) + + +@pytest.fixture() +def novaseq_6000_post_1_5_kits_flow_cell_data(flow_cells_dir: Path) -> FlowCellDirectoryData: + return FlowCellDirectoryData(Path(flow_cells_dir, "230912_A00187_1009_AHK33MDRX3")) + + +@pytest.fixture() +def novaseq_6000_pre_1_5_kits_flow_cell_data(flow_cells_dir: Path) -> FlowCellDirectoryData: + return FlowCellDirectoryData(Path(flow_cells_dir, "190927_A00689_0069_BHLYWYDSXX")) + + +@pytest.fixture() +def novaseq_x_flow_cell_data(flow_cells_dir: Path) -> FlowCellDirectoryData: + return FlowCellDirectoryData(Path(flow_cells_dir, "20231108_LH00188_0028_B22F52TLT3")) + + +# Broken flow cells + + +@pytest.fixture(scope="session") +def bcl2fastq_flow_cell(bcl2fastq_flow_cell_dir: Path) -> FlowCellDirectoryData: + """Create a flow cell object with flow cell that is demultiplexed.""" + return FlowCellDirectoryData( + flow_cell_path=bcl2fastq_flow_cell_dir, bcl_converter=BclConverter.BCL2FASTQ + ) + + +@pytest.fixture(scope="session") +def novaseq_flow_cell_demultiplexed_with_bcl2fastq( + bcl_convert_flow_cell_dir: Path, +) -> FlowCellDirectoryData: + """Return a Novaseq6000 flow cell object demultiplexed using Bcl2fastq.""" + return FlowCellDirectoryData( + flow_cell_path=bcl_convert_flow_cell_dir, bcl_converter=BclConverter.BCL2FASTQ + ) + + +@pytest.fixture(scope="module") +def bcl_convert_flow_cell(bcl_convert_flow_cell_dir: Path) -> FlowCellDirectoryData: + """Create a bcl_convert flow cell object with flow cell that is demultiplexed.""" + return FlowCellDirectoryData( + flow_cell_path=bcl_convert_flow_cell_dir, bcl_converter=BclConverter.DRAGEN + ) + + +@pytest.fixture(scope="function") +def novaseq_6000_flow_cell(bcl_convert_flow_cell: FlowCellDirectoryData) -> FlowCellDirectoryData: + """Return a NovaSeq6000 flow cell object.""" + return bcl_convert_flow_cell + + +@pytest.fixture(scope="function") +def novaseq_x_flow_cell(novaseq_x_flow_cell_dir: Path) -> FlowCellDirectoryData: + """Create a NovaSeqX flow cell object with flow cell that is demultiplexed.""" + return FlowCellDirectoryData( + flow_cell_path=novaseq_x_flow_cell_dir, bcl_converter=BclConverter.DRAGEN + ) + + +@pytest.fixture() +def novaseqx_flow_cell_with_sample_sheet_no_fastq( + novaseqx_flow_cell_directory: Path, novaseqx_demultiplexed_flow_cell: Path +) -> FlowCellDirectoryData: + """Return a flow cell from a tmp dir with a sample sheet and no sample fastq files.""" + novaseqx_flow_cell_directory.mkdir(parents=True, exist_ok=True) + flow_cell = FlowCellDirectoryData(novaseqx_flow_cell_directory) + sample_sheet_path = Path( + novaseqx_demultiplexed_flow_cell, DemultiplexingDirsAndFiles.SAMPLE_SHEET_FILE_NAME + ) + flow_cell._sample_sheet_path_hk = sample_sheet_path + return flow_cell + + +@pytest.fixture(name="tmp_bcl2fastq_flow_cell") +def tmp_bcl2fastq_flow_cell( + tmp_demultiplexed_runs_bcl2fastq_directory: Path, +) -> FlowCellDirectoryData: + """Create a flow cell object with flow cell that is demultiplexed.""" + return FlowCellDirectoryData( + flow_cell_path=tmp_demultiplexed_runs_bcl2fastq_directory, + bcl_converter=BclConverter.BCL2FASTQ, + ) + + +@pytest.fixture +def novaseq6000_flow_cell( + tmp_flow_cells_directory_malformed_sample_sheet: Path, +) -> FlowCellDirectoryData: + """Return a NovaSeq6000 flow cell.""" + return FlowCellDirectoryData( + flow_cell_path=tmp_flow_cells_directory_malformed_sample_sheet, + bcl_converter=BclConverter.BCLCONVERT, + ) + + +@pytest.fixture(name="tmp_bcl_convert_flow_cell") +def tmp_bcl_convert_flow_cell( + tmp_flow_cell_directory_bclconvert: Path, +) -> FlowCellDirectoryData: + """Create a flow cell object with flow cell that is demultiplexed.""" + return FlowCellDirectoryData( + flow_cell_path=tmp_flow_cell_directory_bclconvert, + bcl_converter=BclConverter.DRAGEN, + ) + + +@pytest.fixture(name="tmp_unfinished_bcl2fastq_flow_cell") +def unfinished_bcl2fastq_flow_cell( + demultiplexed_runs_unfinished_bcl2fastq_flow_cell_directory: Path, +) -> FlowCellDirectoryData: + """Copy the content of a demultiplexed but not finished directory to a temporary location.""" + return FlowCellDirectoryData( + flow_cell_path=demultiplexed_runs_unfinished_bcl2fastq_flow_cell_directory, + bcl_converter=BclConverter.BCL2FASTQ, + ) + + +# Flow cell attributes + + +@pytest.fixture(scope="session") +def bcl2fastq_flow_cell_id(bcl2fastq_flow_cell: FlowCellDirectoryData) -> str: + """Return flow cell id from bcl2fastq flow cell object.""" + return bcl2fastq_flow_cell.id + + +@pytest.fixture(scope="module") +def bcl_convert_flow_cell_id(bcl_convert_flow_cell: FlowCellDirectoryData) -> str: + """Return flow cell id from bcl_convert flow cell object.""" + return bcl_convert_flow_cell.id diff --git a/tests/fixture_plugins/demultiplex_fixtures/name_fixtures.py b/tests/fixture_plugins/demultiplex_fixtures/name_fixtures.py new file mode 100644 index 0000000000..c4bbb1d3b4 --- /dev/null +++ b/tests/fixture_plugins/demultiplex_fixtures/name_fixtures.py @@ -0,0 +1,116 @@ +import pytest + + +@pytest.fixture +def tmp_flow_cell_name_no_run_parameters() -> str: + """This is the name of a flow cell directory with the run parameters missing.""" + return "180522_A00689_0200_BHLCKNCCXY" + + +@pytest.fixture +def tmp_flow_cell_name_malformed_sample_sheet() -> str: + """ "Returns the name of a flow cell directory ready for demultiplexing with BCL convert. + Contains a sample sheet with malformed headers. + """ + return "201203_A00689_0200_AHVKJCDRXY" + + +@pytest.fixture +def tmp_flow_cell_name_no_sample_sheet() -> str: + """Return the name of a flow cell directory with the run parameters and sample sheet missing.""" + return "170407_A00689_0209_BHHKVCALXX" + + +@pytest.fixture(name="tmp_flow_cell_name_ready_for_demultiplexing_bcl2fastq") +def tmp_flow_cell_name_ready_for_demultiplexing_bcl2fastq() -> str: + """Returns the name of a flow cell directory ready for demultiplexing with bcl2fastq.""" + return "211101_D00483_0615_AHLG5GDRXY" + + +@pytest.fixture(scope="session") +def flow_cell_name_demultiplexed_with_bcl2fastq() -> str: + """Return the name of a flow cell that has been demultiplexed with BCL2Fastq.""" + return "HHKVCALXX" + + +@pytest.fixture(scope="session") +def flow_cell_directory_name_demultiplexed_with_bcl2fastq( + flow_cell_name_demultiplexed_with_bcl2fastq: str, +) -> str: + """Return the name of a flow cell directory that has been demultiplexed with BCL2Fastq.""" + return f"170407_ST-E00198_0209_B{flow_cell_name_demultiplexed_with_bcl2fastq}" + + +@pytest.fixture(scope="session") +def flow_cell_name_demultiplexed_with_bcl_convert() -> str: + return "HY7FFDRX2" + + +@pytest.fixture(scope="session") +def flow_cell_directory_name_demultiplexed_with_bcl_convert( + flow_cell_name_demultiplexed_with_bcl_convert: str, +) -> str: + return f"230504_A00689_0804_B{flow_cell_name_demultiplexed_with_bcl_convert}" + + +@pytest.fixture(scope="session") +def hiseq_x_single_index_flow_cell_name() -> str: + """Return the full name of a HiSeqX flow cell with only one index.""" + return "170517_ST-E00266_0210_BHJCFFALXX" + + +@pytest.fixture(scope="session") +def hiseq_x_dual_index_flow_cell_name() -> str: + """Return the full name of a HiSeqX flow cell with two indexes.""" + return "180508_ST-E00269_0269_AHL32LCCXY" + + +@pytest.fixture(scope="session") +def hiseq_2500_dual_index_flow_cell_name() -> str: + """Return the full name of a HiSeq2500 flow cell with double indexes.""" + return "181005_D00410_0735_BHM2LNBCX2" + + +@pytest.fixture(scope="session") +def hiseq_2500_custom_index_flow_cell_name() -> str: + """Return the full name of a HiSeq2500 flow cell with double indexes.""" + return "180509_D00450_0598_BHGYFNBCX2" + + +@pytest.fixture(scope="session") +def bcl2fastq_flow_cell_full_name() -> str: + """Return full flow cell name.""" + return "201203_D00483_0200_AHVKJCDRXX" + + +@pytest.fixture(scope="session") +def bcl_convert_flow_cell_full_name() -> str: + """Return the full name of a bcl_convert flow cell.""" + return "211101_A00187_0615_AHLG5GDRZZ" + + +@pytest.fixture(scope="session") +def novaseq_x_flow_cell_full_name() -> str: + """Return the full name of a NovaSeqX flow cell.""" + return "20230508_LH00188_0003_A22522YLT3" + + +# Lists + + +@pytest.fixture(scope="session") +def bcl_convert_demultiplexed_flow_cell_sample_internal_ids() -> list[str]: + """ + Sample id:s present in sample sheet for dummy flow cell demultiplexed with BCL Convert in + cg/tests/fixtures/apps/demultiplexing/demultiplexed-runs/230504_A00689_0804_BHY7FFDRX2. + """ + return ["ACC11927A2", "ACC11927A5"] + + +@pytest.fixture(scope="session") +def bcl2fastq_demultiplexed_flow_cell_sample_internal_ids() -> list[str]: + """ + Sample id:s present in sample sheet for dummy flow cell demultiplexed with BCL Convert in + cg/tests/fixtures/apps/demultiplexing/demultiplexed-runs/170407_A00689_0209_BHHKVCALXX. + """ + return ["SVE2528A1"] diff --git a/tests/fixture_plugins/demultiplex_fixtures/path_fixtures.py b/tests/fixture_plugins/demultiplex_fixtures/path_fixtures.py new file mode 100644 index 0000000000..807c34ee41 --- /dev/null +++ b/tests/fixture_plugins/demultiplex_fixtures/path_fixtures.py @@ -0,0 +1,458 @@ +"""Path fixtures for demultiplex tests.""" +import shutil +from pathlib import Path + +import pytest + +from cg.constants.demultiplexing import DemultiplexingDirsAndFiles +from cg.constants.nanopore_files import NanoporeDirsAndFiles +from cg.models.flow_cell.flow_cell import FlowCellDirectoryData + +CORRECT_SAMPLE_SHEET: str = "CorrectSampleSheet.csv" + + +@pytest.fixture(name="tmp_flow_cells_directory") +def tmp_flow_cells_directory(tmp_path: Path, flow_cells_dir: Path) -> Path: + """ + Return the path to a temporary flow cells directory with flow cells ready for demultiplexing. + Generates a copy of the original flow cells directory + """ + original_dir = flow_cells_dir + tmp_dir = Path(tmp_path, "flow_cells") + + return Path(shutil.copytree(original_dir, tmp_dir)) + + +@pytest.fixture(name="tmp_flow_cells_demux_all_directory") +def tmp_flow_cells_demux_all_directory(tmp_path: Path, flow_cells_demux_all_dir: Path) -> Path: + """ + Return the path to a temporary flow cells directory with flow cells ready for demultiplexing. + Generates a copy of the original flow cells directory. + This fixture is used for testing of the cg demutliplex all cmd. + """ + original_dir = flow_cells_demux_all_dir + tmp_dir = Path(tmp_path, "flow_cells_demux_all") + + return Path(shutil.copytree(original_dir, tmp_dir)) + + +@pytest.fixture(name="tmp_flow_cell_directory_bcl2fastq") +def flow_cell_working_directory_bcl2fastq( + bcl2fastq_flow_cell_dir: Path, tmp_flow_cells_directory: Path +) -> Path: + """Return the path to a working directory that will be deleted after test is run. + + This is a path to a flow cell directory with the run parameters present. + """ + return Path(tmp_flow_cells_directory, bcl2fastq_flow_cell_dir.name) + + +@pytest.fixture(name="tmp_flow_cell_directory_bclconvert") +def flow_cell_working_directory_bclconvert( + bcl_convert_flow_cell_dir: Path, tmp_flow_cells_directory: Path +) -> Path: + """Return the path to a working directory that will be deleted after test is run. + This is a path to a flow cell directory with the run parameters present. + """ + return Path(tmp_flow_cells_directory, bcl_convert_flow_cell_dir.name) + + +@pytest.fixture +def tmp_flow_cells_directory_no_run_parameters( + tmp_flow_cell_name_no_run_parameters: str, tmp_flow_cells_directory: Path +) -> Path: + """This is a path to a flow cell directory with the run parameters missing.""" + return Path(tmp_flow_cells_directory, tmp_flow_cell_name_no_run_parameters) + + +@pytest.fixture(name="tmp_flow_cells_directory_no_sample_sheet") +def tmp_flow_cells_directory_no_sample_sheet( + tmp_flow_cell_name_no_sample_sheet: str, tmp_flow_cells_directory: Path +) -> Path: + """This is a path to a flow cell directory with the sample sheet and run parameters missing.""" + return Path(tmp_flow_cells_directory, tmp_flow_cell_name_no_sample_sheet) + + +@pytest.fixture +def tmp_flow_cells_directory_malformed_sample_sheet( + tmp_flow_cell_name_malformed_sample_sheet: str, tmp_flow_cells_directory: Path +) -> Path: + """This is a path to a flow cell directory with a sample sheet with malformed headers.""" + return Path(tmp_flow_cells_directory, tmp_flow_cell_name_malformed_sample_sheet) + + +@pytest.fixture +def tmp_flow_cells_directory_ready_for_demultiplexing_bcl_convert( + bcl_convert_flow_cell_full_name: str, tmp_flow_cells_directory: Path +) -> Path: + """This is a path to a flow cell directory with the run parameters missing.""" + return Path(tmp_flow_cells_directory, bcl_convert_flow_cell_full_name) + + +@pytest.fixture +def tmp_flow_cells_directory_ready_for_demultiplexing_bcl2fastq( + tmp_flow_cell_name_ready_for_demultiplexing_bcl2fastq: str, tmp_flow_cells_directory: Path +) -> Path: + """This is a path to a flow cell directory with the run parameters missing.""" + return Path(tmp_flow_cells_directory, tmp_flow_cell_name_ready_for_demultiplexing_bcl2fastq) + + +# Temporary demultiplexed runs fixtures +@pytest.fixture(name="tmp_demultiplexed_runs_directory") +def tmp_demultiplexed_flow_cells_directory(tmp_path: Path, demultiplexed_runs: Path) -> Path: + """Return the path to a temporary demultiplex-runs directory. + Generates a copy of the original demultiplexed-runs + """ + original_dir = demultiplexed_runs + tmp_dir = Path(tmp_path, "demultiplexed-runs") + return Path(shutil.copytree(original_dir, tmp_dir)) + + +@pytest.fixture(name="tmp_demultiplexed_runs_bcl2fastq_directory") +def tmp_demultiplexed_runs_bcl2fastq_directory( + tmp_demultiplexed_runs_directory: Path, bcl2fastq_flow_cell_dir: Path +) -> Path: + """Return the path to a temporary demultiplex-runs bcl2fastq flow cell directory.""" + return Path(tmp_demultiplexed_runs_directory, bcl2fastq_flow_cell_dir.name) + + +@pytest.fixture(name="tmp_demultiplexed_runs_not_finished_directory") +def tmp_demultiplexed_runs_not_finished_flow_cells_directory( + tmp_path: Path, demux_results_not_finished_dir: Path +) -> Path: + """ + Return a temporary demultiplex-runs-unfinished path with an unfinished flow cell directory. + Generates a copy of the original demultiplexed-runs-unfinished directory. + """ + original_dir = demux_results_not_finished_dir + tmp_dir = Path(tmp_path, "demultiplexed-runs-unfinished") + return Path(shutil.copytree(original_dir, tmp_dir)) + + +@pytest.fixture(name="demultiplexed_runs_unfinished_bcl2fastq_flow_cell_directory") +def demultiplexed_runs_bcl2fastq_flow_cell_directory( + tmp_demultiplexed_runs_not_finished_directory: Path, + bcl2fastq_flow_cell_full_name: str, +) -> Path: + """Copy the content of a demultiplexed but not finished directory to a temporary location.""" + return Path(tmp_demultiplexed_runs_not_finished_directory, bcl2fastq_flow_cell_full_name) + + +@pytest.fixture(name="novaseq6000_bcl_convert_sample_sheet_path") +def novaseq6000_sample_sheet_path() -> Path: + """Return the path to a NovaSeq 6000 BCL convert sample sheet.""" + return Path( + "tests", + "fixtures", + "apps", + "sequencing_metrics_parser", + "230622_A00621_0864_AHY7FFDRX2", + "Unaligned", + "Reports", + "SampleSheet.csv", + ) + + +@pytest.fixture(scope="session") +def demultiplex_fixtures(apps_dir: Path) -> Path: + """Return the path to the demultiplex fixture directory.""" + return Path(apps_dir, "demultiplexing") + + +@pytest.fixture(scope="session") +def raw_lims_sample_dir(demultiplex_fixtures: Path) -> Path: + """Return the path to the raw samples fixture directory.""" + return Path(demultiplex_fixtures, "raw_lims_samples") + + +@pytest.fixture(scope="session") +def run_parameters_dir(demultiplex_fixtures: Path) -> Path: + """Return the path to the run parameters fixture directory.""" + return Path(demultiplex_fixtures, "run_parameters") + + +@pytest.fixture(scope="session") +def demultiplexed_runs(demultiplex_fixtures: Path) -> Path: + """Return the path to the demultiplexed flow cells fixture directory.""" + return Path(demultiplex_fixtures, "demultiplexed-runs") + + +@pytest.fixture(scope="session") +def flow_cells_dir(demultiplex_fixtures: Path) -> Path: + """Return the path to the sequenced flow cells fixture directory.""" + return Path(demultiplex_fixtures, DemultiplexingDirsAndFiles.FLOW_CELLS_DIRECTORY_NAME) + + +@pytest.fixture(scope="session") +def nanopore_flow_cells_dir(demultiplex_fixtures: Path) -> Path: + """Return the path to the sequenced flow cells fixture directory.""" + return Path(demultiplex_fixtures, NanoporeDirsAndFiles.DATA_DIRECTORY) + + +@pytest.fixture(scope="session") +def flow_cells_demux_all_dir(demultiplex_fixtures: Path) -> Path: + """Return the path to the sequenced flow cells fixture directory.""" + return Path(demultiplex_fixtures, "flow_cells_demux_all") + + +@pytest.fixture(scope="session") +def demux_results_not_finished_dir(demultiplex_fixtures: Path) -> Path: + """Return the path to a dir with demultiplexing results where nothing has been cleaned.""" + return Path(demultiplex_fixtures, "demultiplexed-runs-unfinished") + + +@pytest.fixture +def novaseq_6000_post_1_5_kits_flow_cell(tmp_flow_cells_directory: Path) -> Path: + return Path(tmp_flow_cells_directory, "230912_A00187_1009_AHK33MDRX3") + + +@pytest.fixture +def novaseq_6000_post_1_5_kits_correct_sample_sheet( + novaseq_6000_post_1_5_kits_flow_cell: Path, +) -> Path: + return Path(novaseq_6000_post_1_5_kits_flow_cell, CORRECT_SAMPLE_SHEET) + + +@pytest.fixture +def novaseq_6000_post_1_5_kits_raw_lims_samples( + novaseq_6000_post_1_5_kits_flow_cell: Path, +) -> Path: + return Path(novaseq_6000_post_1_5_kits_flow_cell, "HK33MDRX3_raw.json") + + +@pytest.fixture +def novaseq_6000_pre_1_5_kits_flow_cell(tmp_flow_cells_directory: Path) -> Path: + return Path(tmp_flow_cells_directory, "190927_A00689_0069_BHLYWYDSXX") + + +@pytest.fixture +def novaseq_6000_pre_1_5_kits_correct_sample_sheet( + novaseq_6000_pre_1_5_kits_flow_cell: Path, +) -> Path: + return Path(novaseq_6000_pre_1_5_kits_flow_cell, CORRECT_SAMPLE_SHEET) + + +@pytest.fixture +def novaseq_6000_pre_1_5_kits_raw_lims_samples(novaseq_6000_pre_1_5_kits_flow_cell: Path) -> Path: + return Path(novaseq_6000_pre_1_5_kits_flow_cell, "HLYWYDSXX_raw.json") + + +@pytest.fixture +def novaseq_x_flow_cell_directory(tmp_flow_cells_directory: Path) -> Path: + return Path(tmp_flow_cells_directory, "20231108_LH00188_0028_B22F52TLT3") + + +@pytest.fixture +def novaseq_x_correct_sample_sheet(novaseq_x_flow_cell_directory: Path) -> Path: + return Path(novaseq_x_flow_cell_directory, CORRECT_SAMPLE_SHEET) + + +@pytest.fixture +def novaseq_x_raw_lims_samples(novaseq_x_flow_cell_directory: Path) -> Path: + return Path(novaseq_x_flow_cell_directory, "22F52TLT3_raw.json") + + +@pytest.fixture(scope="session") +def novaseq_x_manifest_file(novaseq_x_flow_cell_dir: Path) -> Path: + """Return the path to a NovaSeqX manifest file.""" + return Path(novaseq_x_flow_cell_dir, "Manifest.tsv") + + +@pytest.fixture(scope="session") +def hiseq_x_single_index_flow_cell_dir( + flow_cells_dir: Path, hiseq_x_single_index_flow_cell_name: str +) -> Path: + """Return the path to a HiSeqX flow cell.""" + return Path(flow_cells_dir, hiseq_x_single_index_flow_cell_name) + + +@pytest.fixture(scope="session") +def hiseq_x_dual_index_flow_cell_dir( + flow_cells_dir: Path, hiseq_x_dual_index_flow_cell_name: str +) -> Path: + """Return the path to a HiSeqX flow cell.""" + return Path(flow_cells_dir, hiseq_x_dual_index_flow_cell_name) + + +@pytest.fixture(scope="session") +def hiseq_2500_dual_index_flow_cell_dir( + flow_cells_dir: Path, hiseq_2500_dual_index_flow_cell_name: str +) -> Path: + """Return the path to a HiSeq2500 flow cell.""" + return Path(flow_cells_dir, hiseq_2500_dual_index_flow_cell_name) + + +@pytest.fixture(scope="session") +def hiseq_2500_custom_index_flow_cell_dir( + flow_cells_dir: Path, hiseq_2500_custom_index_flow_cell_name: str +) -> Path: + """Return the path to a HiSeq2500 flow cell.""" + return Path(flow_cells_dir, hiseq_2500_custom_index_flow_cell_name) + + +@pytest.fixture(scope="session") +def bcl2fastq_flow_cell_dir(flow_cells_dir: Path, bcl2fastq_flow_cell_full_name: str) -> Path: + """Return the path to the bcl2fastq flow cell demultiplex fixture directory.""" + return Path(flow_cells_dir, bcl2fastq_flow_cell_full_name) + + +@pytest.fixture(scope="session") +def bcl_convert_flow_cell_dir(flow_cells_dir: Path, bcl_convert_flow_cell_full_name: str) -> Path: + """Return the path to the bcl_convert flow cell demultiplex fixture directory.""" + return Path(flow_cells_dir, bcl_convert_flow_cell_full_name) + + +@pytest.fixture(scope="session") +def novaseq_x_flow_cell_dir(flow_cells_dir: Path, novaseq_x_flow_cell_full_name: str) -> Path: + """Return the path to the NovaSeqX flow cell demultiplex fixture directory.""" + return Path(flow_cells_dir, novaseq_x_flow_cell_full_name) + + +@pytest.fixture(scope="session") +def novaseq_bcl2fastq_sample_sheet_path(bcl2fastq_flow_cell_dir: Path) -> Path: + """Return the path to a NovaSeq6000 Bcl2fastq sample sheet.""" + return Path(bcl2fastq_flow_cell_dir, DemultiplexingDirsAndFiles.SAMPLE_SHEET_FILE_NAME) + + +@pytest.fixture(scope="session") +def novaseq_bcl_convert_sample_sheet_path(bcl_convert_flow_cell_dir: Path) -> Path: + """Return the path to a NovaSeq6000 bcl_convert sample sheet.""" + return Path(bcl_convert_flow_cell_dir, DemultiplexingDirsAndFiles.SAMPLE_SHEET_FILE_NAME) + + +@pytest.fixture(scope="session") +def run_parameters_wrong_instrument(run_parameters_dir: Path) -> Path: + """Return a NovaSeqX run parameters file path with a wrong instrument value.""" + return Path(run_parameters_dir, "RunParameters_novaseq_X_wrong_instrument.xml") + + +@pytest.fixture(scope="session") +def hiseq_x_single_index_run_parameters_path( + hiseq_x_single_index_flow_cell_dir: Path, +) -> Path: + """Return the path to a HiSeqX run parameters file with single index.""" + return Path( + hiseq_x_single_index_flow_cell_dir, DemultiplexingDirsAndFiles.RUN_PARAMETERS_CAMEL_CASE + ) + + +@pytest.fixture(scope="session") +def hiseq_x_dual_index_run_parameters_path( + hiseq_x_dual_index_flow_cell_dir: Path, +) -> Path: + """Return the path to a HiSeqX run parameters file with dual index.""" + return Path( + hiseq_x_dual_index_flow_cell_dir, DemultiplexingDirsAndFiles.RUN_PARAMETERS_CAMEL_CASE + ) + + +@pytest.fixture(scope="session") +def hiseq_2500_dual_index_run_parameters_path( + hiseq_2500_dual_index_flow_cell_dir: Path, +) -> Path: + """Return the path to a HiSeq2500 run parameters file with dual index.""" + return Path( + hiseq_2500_dual_index_flow_cell_dir, DemultiplexingDirsAndFiles.RUN_PARAMETERS_CAMEL_CASE + ) + + +@pytest.fixture(scope="session") +def hiseq_2500_custom_index_run_parameters_path( + hiseq_2500_custom_index_flow_cell_dir: Path, +) -> Path: + """Return the path to a HiSeq2500 run parameters file with custom index.""" + return Path( + hiseq_2500_custom_index_flow_cell_dir, DemultiplexingDirsAndFiles.RUN_PARAMETERS_CAMEL_CASE + ) + + +@pytest.fixture(scope="session") +def novaseq_6000_run_parameters_path(bcl2fastq_flow_cell_dir: Path) -> Path: + """Return the path to a NovaSeq6000 run parameters file.""" + return Path(bcl2fastq_flow_cell_dir, DemultiplexingDirsAndFiles.RUN_PARAMETERS_PASCAL_CASE) + + +@pytest.fixture +def novaseq_6000_run_parameters_pre_1_5_kits_path( + novaseq_6000_pre_1_5_kits_flow_cell: Path, +) -> Path: + """Return the path to a NovaSeq6000 pre 1.5 kit run parameters file.""" + return Path( + novaseq_6000_pre_1_5_kits_flow_cell, + DemultiplexingDirsAndFiles.RUN_PARAMETERS_PASCAL_CASE, + ) + + +@pytest.fixture +def novaseq_6000_run_parameters_post_1_5_kits_path( + novaseq_6000_post_1_5_kits_flow_cell: Path, +) -> Path: + """Return the path to a NovaSeq6000 post 1.5 kit run parameters file.""" + return Path( + novaseq_6000_post_1_5_kits_flow_cell, + DemultiplexingDirsAndFiles.RUN_PARAMETERS_PASCAL_CASE, + ) + + +@pytest.fixture(scope="session") +def novaseq_x_run_parameters_path(novaseq_x_flow_cell_dir: Path) -> Path: + """Return the path to a NovaSeqX run parameters file.""" + return Path(novaseq_x_flow_cell_dir, DemultiplexingDirsAndFiles.RUN_PARAMETERS_PASCAL_CASE) + + +@pytest.fixture(scope="module") +def run_parameters_missing_versions_path( + run_parameters_dir: Path, +) -> Path: + """Return a NovaSeq6000 run parameters path without software and reagent kit versions.""" + return Path(run_parameters_dir, "RunParameters_novaseq_no_software_nor_reagent_version.xml") + + +@pytest.fixture(name="demultiplexing_delivery_file") +def demultiplexing_delivery_file(bcl2fastq_flow_cell: FlowCellDirectoryData) -> Path: + """Return demultiplexing delivery started file.""" + return Path(bcl2fastq_flow_cell.path, DemultiplexingDirsAndFiles.DELIVERY) + + +@pytest.fixture(name="hiseq_x_tile_dir") +def hiseq_x_tile_dir(bcl2fastq_flow_cell: FlowCellDirectoryData) -> Path: + """Return HiSeqX tile dir.""" + return Path(bcl2fastq_flow_cell.path, DemultiplexingDirsAndFiles.HISEQ_X_TILE_DIR) + + +@pytest.fixture(name="lims_novaseq_samples_file") +def lims_novaseq_samples_file(raw_lims_sample_dir: Path) -> Path: + """Return the path to a file with sample info in lims format.""" + return Path(raw_lims_sample_dir, "raw_samplesheet_novaseq.json") + + +@pytest.fixture +def lims_novaseq_6000_samples_file(bcl2fastq_flow_cell_dir: Path) -> Path: + """Return the path to the file with the raw samples of HVKJCDRXX flow cell in lims format.""" + return Path(bcl2fastq_flow_cell_dir, "HVKJCDRXX_raw.json") + + +@pytest.fixture(name="demultiplexed_flow_cell") +def demultiplexed_flow_cell(demultiplexed_runs: Path, bcl2fastq_flow_cell_full_name: str) -> Path: + """Return the path to a demultiplexed flow cell with bcl2fastq.""" + return Path(demultiplexed_runs, bcl2fastq_flow_cell_full_name) + + +@pytest.fixture(name="bcl_convert_demultiplexed_flow_cell") +def bcl_convert_demultiplexed_flow_cell( + demultiplexed_runs: Path, bcl_convert_flow_cell_full_name: str +) -> Path: + """Return the path to a demultiplexed flow cell with BCLConvert.""" + return Path(demultiplexed_runs, bcl_convert_flow_cell_full_name) + + +# Fixtures for test demultiplex flow cell +@pytest.fixture +def tmp_empty_demultiplexed_runs_directory(tmp_demultiplexed_runs_directory) -> Path: + return Path(tmp_demultiplexed_runs_directory, "empty") + + +@pytest.fixture(name="novaseqx_demultiplexed_flow_cell") +def novaseqx_demultiplexed_flow_cell(demultiplexed_runs: Path, novaseq_x_flow_cell_full_name: str): + """Return the path to a demultiplexed NovaSeqX flow cell.""" + return Path(demultiplexed_runs, novaseq_x_flow_cell_full_name) diff --git a/tests/fixture_plugins/demultiplex_fixtures/run_parameters_fixtures.py b/tests/fixture_plugins/demultiplex_fixtures/run_parameters_fixtures.py new file mode 100644 index 0000000000..e8c7459cec --- /dev/null +++ b/tests/fixture_plugins/demultiplex_fixtures/run_parameters_fixtures.py @@ -0,0 +1,98 @@ +from pathlib import Path + +import pytest + +from cg.models.demultiplex.run_parameters import ( + RunParametersHiSeq, + RunParametersNovaSeq6000, + RunParametersNovaSeqX, +) + + +@pytest.fixture(scope="function") +def run_parameters_hiseq_different_index(run_parameters_dir: Path) -> RunParametersHiSeq: + """Return a HiSeq RunParameters object with different index cycles.""" + path = Path(run_parameters_dir, "RunParameters_hiseq_2500_different_index_cycles.xml") + return RunParametersHiSeq(run_parameters_path=path) + + +@pytest.fixture(scope="function") +def run_parameters_novaseq_6000_different_index( + run_parameters_dir: Path, +) -> RunParametersNovaSeq6000: + """Return a NovaSeq6000 RunParameters object with different index cycles.""" + path = Path(run_parameters_dir, "RunParameters_novaseq_6000_different_index_cycles.xml") + return RunParametersNovaSeq6000(run_parameters_path=path) + + +@pytest.fixture(scope="function") +def run_parameters_novaseq_x_different_index(run_parameters_dir: Path) -> RunParametersNovaSeqX: + """Return a NovaSeqX RunParameters object with different index cycles.""" + path = Path(run_parameters_dir, "RunParameters_novaseq_X_different_index_cycles.xml") + return RunParametersNovaSeqX(run_parameters_path=path) + + +@pytest.fixture(scope="session") +def hiseq_x_single_index_run_parameters( + hiseq_x_single_index_run_parameters_path: Path, +) -> RunParametersHiSeq: + """Return a HiSeqX run parameters object with single index.""" + return RunParametersHiSeq(run_parameters_path=hiseq_x_single_index_run_parameters_path) + + +@pytest.fixture(scope="session") +def hiseq_x_dual_index_run_parameters( + hiseq_x_dual_index_run_parameters_path: Path, +) -> RunParametersHiSeq: + """Return a HiSeqX run parameters object with dual index.""" + return RunParametersHiSeq(run_parameters_path=hiseq_x_dual_index_run_parameters_path) + + +@pytest.fixture(scope="session") +def hiseq_2500_dual_index_run_parameters( + hiseq_2500_dual_index_run_parameters_path: Path, +) -> RunParametersHiSeq: + """Return a HiSeq2500 run parameters object with dual index.""" + return RunParametersHiSeq(run_parameters_path=hiseq_2500_dual_index_run_parameters_path) + + +@pytest.fixture(scope="session") +def hiseq_2500_custom_index_run_parameters( + hiseq_2500_custom_index_run_parameters_path: Path, +) -> RunParametersHiSeq: + """Return a HiSeq2500 run parameters object with custom index.""" + return RunParametersHiSeq(run_parameters_path=hiseq_2500_custom_index_run_parameters_path) + + +@pytest.fixture(scope="session") +def novaseq_6000_run_parameters( + novaseq_6000_run_parameters_path: Path, +) -> RunParametersNovaSeq6000: + """Return a NovaSeq6000 run parameters object.""" + return RunParametersNovaSeq6000(run_parameters_path=novaseq_6000_run_parameters_path) + + +@pytest.fixture +def novaseq_6000_run_parameters_pre_1_5_kits( + novaseq_6000_run_parameters_pre_1_5_kits_path: Path, +) -> RunParametersNovaSeq6000: + """Return a NovaSeq6000 run parameters pre 1.5 kit object.""" + return RunParametersNovaSeq6000( + run_parameters_path=novaseq_6000_run_parameters_pre_1_5_kits_path + ) + + +@pytest.fixture +def novaseq_6000_run_parameters_post_1_5_kits(novaseq_6000_run_parameters_post_1_5_kits_path: Path): + """Return a NovaSeq6000 run parameters post 1.5 kit object.""" + return RunParametersNovaSeq6000( + run_parameters_path=novaseq_6000_run_parameters_post_1_5_kits_path + ) + + +@pytest.fixture(scope="session") +def novaseq_x_run_parameters( + novaseq_x_run_parameters_path: Path, +) -> RunParametersNovaSeqX: + """Return a NovaSeqX run parameters object.""" + return RunParametersNovaSeqX(run_parameters_path=novaseq_x_run_parameters_path) diff --git a/tests/fixture_plugins/demultiplex_fixtures/sample_fixtures.py b/tests/fixture_plugins/demultiplex_fixtures/sample_fixtures.py new file mode 100644 index 0000000000..9973ddf58e --- /dev/null +++ b/tests/fixture_plugins/demultiplex_fixtures/sample_fixtures.py @@ -0,0 +1,135 @@ +"""Demultiplex sample fixtures.""" +from pathlib import Path + +import pytest + +from cg.apps.demultiplex.sample_sheet.sample_models import ( + FlowCellSampleBcl2Fastq, + FlowCellSampleBCLConvert, +) +from cg.apps.demultiplex.sample_sheet.sample_sheet_creator import SampleSheetCreatorBCLConvert +from cg.constants import FileExtensions +from cg.constants.constants import FileFormat +from cg.io.controller import ReadFile +from cg.io.json import read_json +from cg.models.flow_cell.flow_cell import FlowCellDirectoryData + + +@pytest.fixture +def lims_novaseq_bcl_convert_samples( + lims_novaseq_samples_raw: list[dict], +) -> list[FlowCellSampleBCLConvert]: + """Return a list of parsed flow cell samples demultiplexed with BCL convert.""" + return [FlowCellSampleBCLConvert.model_validate(sample) for sample in lims_novaseq_samples_raw] + + +@pytest.fixture +def lims_novaseq_bcl2fastq_samples( + lims_novaseq_samples_raw: list[dict], +) -> list[FlowCellSampleBcl2Fastq]: + """Return a list of parsed Bcl2fastq flow cell samples""" + return [FlowCellSampleBcl2Fastq.model_validate(sample) for sample in lims_novaseq_samples_raw] + + +@pytest.fixture +def lims_novaseq_6000_bcl2fastq_samples( + lims_novaseq_6000_sample_raw: list[dict], +) -> list[FlowCellSampleBcl2Fastq]: + """Return a list of parsed Bcl2fastq flow cell samples""" + return [ + FlowCellSampleBcl2Fastq.model_validate(sample) for sample in lims_novaseq_6000_sample_raw + ] + + +@pytest.fixture +def bcl_convert_sample_sheet_creator( + bcl_convert_flow_cell: FlowCellDirectoryData, + lims_novaseq_bcl_convert_samples: list[FlowCellSampleBCLConvert], +) -> SampleSheetCreatorBCLConvert: + """Returns a sample sheet creator for version 2 sample sheets with dragen format.""" + return SampleSheetCreatorBCLConvert( + flow_cell=bcl_convert_flow_cell, + lims_samples=lims_novaseq_bcl_convert_samples, + ) + + +@pytest.fixture +def novaseq_6000_post_1_5_kits_lims_samples( + novaseq_6000_post_1_5_kits_raw_lims_samples: Path, +) -> list[FlowCellSampleBCLConvert]: + return [ + FlowCellSampleBCLConvert.model_validate(sample) + for sample in read_json(novaseq_6000_post_1_5_kits_raw_lims_samples) + ] + + +@pytest.fixture +def novaseq_6000_pre_1_5_kits_lims_samples( + novaseq_6000_pre_1_5_kits_raw_lims_samples: Path, +) -> list[FlowCellSampleBCLConvert]: + return [ + FlowCellSampleBCLConvert.model_validate(sample) + for sample in read_json(novaseq_6000_pre_1_5_kits_raw_lims_samples) + ] + + +@pytest.fixture +def novaseq_x_lims_samples(novaseq_x_raw_lims_samples: Path) -> list[FlowCellSampleBCLConvert]: + return [ + FlowCellSampleBCLConvert.model_validate(sample) + for sample in read_json(novaseq_x_raw_lims_samples) + ] + + +@pytest.fixture +def hiseq_x_single_index_bcl_convert_lims_samples( + hiseq_x_single_index_flow_cell_dir: Path, +) -> list[FlowCellSampleBCLConvert]: + """Return a list of BCLConvert samples from a HiSeqX single index flow cell.""" + path = Path( + hiseq_x_single_index_flow_cell_dir, f"HJCFFALXX_bcl_convert_raw{FileExtensions.JSON}" + ) + return [FlowCellSampleBCLConvert.model_validate(sample) for sample in read_json(path)] + + +@pytest.fixture +def hiseq_x_dual_index_bcl_convert_lims_samples( + hiseq_x_dual_index_flow_cell_dir: Path, +) -> list[FlowCellSampleBCLConvert]: + """Return a list of BCLConvert samples from a HiSeqX dual index flow cell.""" + path = Path(hiseq_x_dual_index_flow_cell_dir, f"HL32LCCXY_bcl_convert_raw{FileExtensions.JSON}") + return [FlowCellSampleBCLConvert.model_validate(sample) for sample in read_json(path)] + + +@pytest.fixture +def hiseq_2500_dual_index_bcl_convert_lims_samples( + hiseq_2500_dual_index_flow_cell_dir: Path, +) -> list[FlowCellSampleBCLConvert]: + """Return a list of BCLConvert samples from a HiSeq2500 dual index flow cell.""" + path = Path(hiseq_2500_dual_index_flow_cell_dir, "HM2LNBCX2_bcl_convert_raw.json") + return [FlowCellSampleBCLConvert.model_validate(sample) for sample in read_json(path)] + + +@pytest.fixture +def hiseq_2500_custom_index_bcl_convert_lims_samples( + hiseq_2500_custom_index_flow_cell_dir: Path, +) -> list[FlowCellSampleBCLConvert]: + """Return a list of BCLConvert samples from a HiSeq2500 custom index flow cell.""" + path = Path(hiseq_2500_custom_index_flow_cell_dir, "HGYFNBCX2_bcl_convert_raw.json") + return [FlowCellSampleBCLConvert.model_validate(sample) for sample in read_json(path)] + + +@pytest.fixture +def lims_novaseq_samples_raw(lims_novaseq_samples_file: Path) -> list[dict]: + """Return a list of raw flow cell samples.""" + return ReadFile.get_content_from_file( + file_format=FileFormat.JSON, file_path=lims_novaseq_samples_file + ) + + +@pytest.fixture +def lims_novaseq_6000_sample_raw(lims_novaseq_6000_samples_file: Path) -> list[dict]: + """Return the list of raw samples from flow cell HVKJCDRXX.""" + return ReadFile.get_content_from_file( + file_format=FileFormat.JSON, file_path=lims_novaseq_6000_samples_file + ) diff --git a/tests/fixture_plugins/timestamp_fixtures.py b/tests/fixture_plugins/timestamp_fixtures.py new file mode 100644 index 0000000000..6e77e289bb --- /dev/null +++ b/tests/fixture_plugins/timestamp_fixtures.py @@ -0,0 +1,46 @@ +"""Timestamp fixtures.""" +from datetime import MAXYEAR, datetime, timedelta + +import pytest + + +@pytest.fixture(scope="session") +def old_timestamp() -> datetime: + """Return a time stamp in date time format.""" + return datetime(1900, 1, 1) + + +@pytest.fixture(scope="session") +def timestamp() -> datetime: + """Return a time stamp in date time format.""" + return datetime(2020, 5, 1) + + +@pytest.fixture(scope="session") +def later_timestamp() -> datetime: + """Return a time stamp in date time format.""" + return datetime(2020, 6, 1) + + +@pytest.fixture(scope="session") +def future_date() -> datetime: + """Return a distant date in the future for which no events happen later.""" + return datetime(MAXYEAR, 1, 1, 1, 1, 1) + + +@pytest.fixture(scope="session") +def timestamp_now() -> datetime: + """Return a time stamp of today's date in date time format.""" + return datetime.now() + + +@pytest.fixture(scope="session") +def timestamp_yesterday(timestamp_now: datetime) -> datetime: + """Return a time stamp of yesterday's date in date time format.""" + return timestamp_now - timedelta(days=1) + + +@pytest.fixture(scope="session") +def timestamp_in_2_weeks(timestamp_now: datetime) -> datetime: + """Return a time stamp 14 days ahead in time.""" + return timestamp_now + timedelta(days=14) diff --git a/tests/fixtures/cgweb_orders/balsamic.json b/tests/fixtures/cgweb_orders/balsamic.json index b11371b00c..17179dde03 100644 --- a/tests/fixtures/cgweb_orders/balsamic.json +++ b/tests/fixtures/cgweb_orders/balsamic.json @@ -12,6 +12,7 @@ ], "comment": "other Elution buffer", "container": "96 well plate", + "concentration_ng_ul": "18", "container_name": "p1", "data_analysis": "balsamic", "data_delivery": "fastq-analysis-scout", diff --git a/tests/meta/archive/test_archive_api.py b/tests/meta/archive/test_archive_api.py index b884991b9e..fe30c2cb47 100644 --- a/tests/meta/archive/test_archive_api.py +++ b/tests/meta/archive/test_archive_api.py @@ -12,6 +12,7 @@ from cg.meta.archive.archive import ARCHIVE_HANDLERS, FileAndSample, SpringArchiveAPI from cg.meta.archive.ddn.constants import ( FAILED_JOB_STATUSES, + METADATA_LIST, ONGOING_JOB_STATUSES, JobStatus, ) @@ -22,6 +23,7 @@ GetJobStatusResponse, MiriaObject, ) +from cg.meta.archive.ddn.utils import get_metadata from cg.meta.archive.models import ArchiveHandler, FileTransferData from cg.models.cg_config import DataFlowConfig from cg.store.models import Sample @@ -141,16 +143,16 @@ def test_call_corresponding_archiving_method(spring_archive_api: SpringArchiveAP return_value=123, ), mock.patch.object( DDNDataFlowClient, - "archive_files", + "archive_file", return_value=123, ) as mock_request_submitter: # WHEN calling the corresponding archive method - spring_archive_api.archive_files_to_location( - files_and_samples=[file_and_sample], archive_location=ArchiveLocations.KAROLINSKA_BUCKET + spring_archive_api.archive_file_to_location( + file_and_sample=file_and_sample, archive_location=ArchiveLocations.KAROLINSKA_BUCKET ) # THEN the correct archive function should have been called once - mock_request_submitter.assert_called_once_with(files_and_samples=[file_and_sample]) + mock_request_submitter.assert_called_once_with(file_and_sample=file_and_sample) @pytest.mark.parametrize("limit", [None, -1, 0, 1]) @@ -184,6 +186,9 @@ def test_archive_all_non_archived_spring_files( # THEN the DDN archiving function should have been called with the correct destination and source if limit > 0 if limit not in [0, -1]: + sample: Sample = spring_archive_api.status_db.get_sample_by_internal_id(sample_id) + metadata: list[dict] = get_metadata(sample) + archive_request_json[METADATA_LIST] = metadata mock_request_submitter.assert_called_with( api_method=APIMethods.POST, url="some/api/files/archive", diff --git a/tests/meta/archive/test_archiving.py b/tests/meta/archive/test_archiving.py index 5447483bf2..0c8679d50d 100644 --- a/tests/meta/archive/test_archiving.py +++ b/tests/meta/archive/test_archiving.py @@ -18,6 +18,7 @@ ) from cg.meta.archive.ddn.ddn_data_flow_client import DDNDataFlowClient from cg.meta.archive.ddn.models import MiriaObject, TransferPayload +from cg.meta.archive.ddn.utils import get_metadata from cg.meta.archive.models import FileAndSample from cg.models.cg_config import DataFlowConfig from cg.store import Store @@ -264,7 +265,7 @@ def test__refresh_auth_token(ddn_dataflow_client: DDNDataFlowClient, ok_response assert ddn_dataflow_client.token_expiration.second == new_expiration.second -def test_archive_folders( +def test_archive_file( ddn_dataflow_client: DDNDataFlowClient, remote_storage_repository: str, local_storage_repository: str, @@ -281,7 +282,7 @@ def test_archive_folders( "api_request_from_content", return_value=ok_miria_response, ) as mock_request_submitter: - job_id: int = ddn_dataflow_client.archive_files([file_and_sample]) + job_id: int = ddn_dataflow_client.archive_file(file_and_sample) # THEN an integer should be returned assert isinstance(job_id, int) @@ -300,7 +301,7 @@ def test_archive_folders( ], "osType": OSTYPE, "createFolder": True, - "metadataList": [], + "metadataList": get_metadata(file_and_sample.sample), "settings": [], }, verify=False, diff --git a/tests/meta/compress/test_compress_meta_fastq.py b/tests/meta/compress/test_compress_meta_fastq.py index 1431fb6796..49fb08cf95 100644 --- a/tests/meta/compress/test_compress_meta_fastq.py +++ b/tests/meta/compress/test_compress_meta_fastq.py @@ -1,5 +1,8 @@ """Tests for FASTQ part of meta compress api""" import logging +from unittest import mock + +from cg.meta.compress import CompressAPI def test_compress_case_fastq_one_sample(populated_compress_fastq_api, sample, caplog): @@ -10,14 +13,15 @@ def test_compress_case_fastq_one_sample(populated_compress_fastq_api, sample, ca # GIVEN a populated compress api # WHEN Compressing the bam files for the case - res = compress_api.compress_fastq(sample) + with mock.patch.object(CompressAPI, "_is_spring_archived", return_value=False): + result = compress_api.compress_fastq(sample) - # THEN assert compression succeded - assert res is True - # THEN assert that the correct information is communicated - assert "Compressing" in caplog.text - # THEN assert that the correct information is communicated - assert "to SPRING format" in caplog.text + # THEN assert compression succeded + assert result is True + # THEN assert that the correct information is communicated + assert "Compressing" in caplog.text + # THEN assert that the correct information is communicated + assert "to SPRING format" in caplog.text def test_compress_fastq_compression_done( @@ -34,15 +38,16 @@ def test_compress_fastq_compression_done( compression_object.spring_path.touch() # WHEN Compressing the bam files for the case - res = compress_api.compress_fastq(sample) + with mock.patch.object(CompressAPI, "_is_spring_archived", return_value=False): + result = compress_api.compress_fastq(sample) - # THEN assert compression succeded - assert res is False - # THEN assert that the correct information is communicated - assert f"FASTQ to SPRING not possible for {sample}" in caplog.text + # THEN assert compression succeded + assert result is False + # THEN assert that the correct information is communicated + assert f"FASTQ to SPRING not possible for {sample}" in caplog.text -def test_compress_case_fastq_compression_pending( +def test_compress_sample_fastq_compression_pending( populated_compress_fastq_api, sample, compression_object, caplog ): """Test to compress all FASTQ files for a sample when compression is pending @@ -55,9 +60,32 @@ def test_compress_case_fastq_compression_pending( compression_object.pending_path.touch() # WHEN compressing the FASTQ files for the case - res = compress_api.compress_fastq(sample) + with mock.patch.object(CompressAPI, "_is_spring_archived", return_value=False): + result = compress_api.compress_fastq(sample) + + # THEN assert compression returns False + assert result is False + # THEN assert that the correct information is communicated + assert f"FASTQ to SPRING not possible for {sample}" in caplog.text + + +def test_compress_sample_fastq_archived_spring_file( + populated_compress_fastq_api, sample, compression_object, caplog +): + """Test to compress all FASTQ files for a sample when the Spring file is archived + + The program should not compress any files since the Spring file already exists + """ + caplog.set_level(logging.DEBUG) + compress_api = populated_compress_fastq_api + # GIVEN that the pending flag exists + compression_object.pending_path.touch() + + # WHEN compressing the FASTQ files for the case + with mock.patch.object(CompressAPI, "_is_spring_archived", return_value=True): + result = compress_api.compress_fastq(sample) - # THEN assert compression returns False - assert res is False - # THEN assert that the correct information is communicated - assert f"FASTQ to SPRING not possible for {sample}" in caplog.text + # THEN assert compression returns False + assert result is False + # THEN assert that the correct information is communicated + assert f"FASTQ to SPRING not possible for {sample}" in caplog.text diff --git a/tests/meta/report/test_balsamic_api.py b/tests/meta/report/test_balsamic_api.py index 2fc3954873..191b4e95ff 100644 --- a/tests/meta/report/test_balsamic_api.py +++ b/tests/meta/report/test_balsamic_api.py @@ -95,8 +95,8 @@ def test_get_variant_caller_version(report_api_balsamic, case_id): assert version == expected_version -def test_get_report_accreditation(report_api_balsamic, case_id): - """Tests report accreditation for a specific BALSAMIC analysis.""" +def test_is_report_accredited(report_api_balsamic, case_id): + """Test report accreditation for a specific BALSAMIC analysis.""" # GIVEN a mock metadata object and an accredited one balsamic_metadata = report_api_balsamic.analysis_api.get_latest_metadata(case_id) @@ -105,10 +105,8 @@ def test_get_report_accreditation(report_api_balsamic, case_id): balsamic_accredited_metadata.config.panel.capture_kit = "gmsmyeloid" # WHEN performing the accreditation validation - unaccredited_report = report_api_balsamic.get_report_accreditation(None, balsamic_metadata) - accredited_report = report_api_balsamic.get_report_accreditation( - None, balsamic_accredited_metadata - ) + unaccredited_report = report_api_balsamic.is_report_accredited(None, balsamic_metadata) + accredited_report = report_api_balsamic.is_report_accredited(None, balsamic_accredited_metadata) # THEN verify that only the panel "gmsmyeloid" reports are validated assert not unaccredited_report diff --git a/tests/meta/report/test_mip_dna_api.py b/tests/meta/report/test_mip_dna_api.py index c423c81434..e0f3c5e5dc 100644 --- a/tests/meta/report/test_mip_dna_api.py +++ b/tests/meta/report/test_mip_dna_api.py @@ -44,22 +44,22 @@ def test_get_sample_coverage(report_api_mip_dna, sample_store, helpers: StoreHel assert sample_coverage == {"mean_coverage": 37.342, "mean_completeness": 97.1} -def test_get_report_accreditation(report_api_mip_dna, mip_analysis_api, case_mip_dna): - """Verifies the report accreditation extraction workflow.""" +def test_is_report_accredited(report_api_mip_dna, mip_analysis_api, case_mip_dna): + """Test report accreditation extraction workflow.""" # GIVEN a list of accredited samples mip_metadata = mip_analysis_api.get_latest_metadata(case_mip_dna.internal_id) samples = report_api_mip_dna.get_samples_data(case_mip_dna, mip_metadata) # WHEN retrieving the report accreditation - accredited = report_api_mip_dna.get_report_accreditation(samples) + accredited = report_api_mip_dna.is_report_accredited(samples) # THEN check that the report is accredited assert accredited -def test_get_report_accreditation_false(report_api_mip_dna, mip_analysis_api, case_mip_dna): - """Verifies that the report is not accredited if it contains a sample application that is not accredited.""" +def test_is_report_accredited_false(report_api_mip_dna, mip_analysis_api, case_mip_dna): + """Test that the report is not accredited if it contains a sample application that is not accredited.""" # GIVEN a list of samples when one of them is not accredited mip_metadata = mip_analysis_api.get_latest_metadata(case_mip_dna.internal_id) @@ -67,7 +67,7 @@ def test_get_report_accreditation_false(report_api_mip_dna, mip_analysis_api, ca samples[0].application.accredited = False # WHEN retrieving the report accreditation - accredited = report_api_mip_dna.get_report_accreditation(samples) + accredited = report_api_mip_dna.is_report_accredited(samples) # THEN check that the report is not accredited assert not accredited diff --git a/tests/meta/upload/scout/conftest.py b/tests/meta/upload/scout/conftest.py index b0be71f190..b4f2f1e86b 100644 --- a/tests/meta/upload/scout/conftest.py +++ b/tests/meta/upload/scout/conftest.py @@ -1,5 +1,4 @@ """Fixtures for the upload Scout API tests.""" - import logging from datetime import datetime from pathlib import Path @@ -10,6 +9,8 @@ from cg.constants import DataDelivery, Pipeline from cg.constants.constants import FileFormat, PrepCategory +from cg.constants.housekeeper_tags import HK_DELIVERY_REPORT_TAG +from cg.constants.scout import UploadTrack from cg.constants.sequencing import SequencingMethod from cg.io.controller import ReadFile from cg.meta.upload.scout.balsamic_config_builder import BalsamicConfigBuilder @@ -241,6 +242,7 @@ def mip_dna_analysis_hk_bundle_data( sv_vcf_file: str, snv_research_vcf_file: str, sv_research_vcf_file: str, + delivery_report_html: Path, ) -> dict: """Return MIP DNA bundle data for Housekeeper.""" return { @@ -415,7 +417,7 @@ def balsamic_analysis_hk_bundle_data( @pytest.fixture(scope="function") def rnafusion_analysis_hk_bundle_data( - case_id: str, timestamp: datetime, rnafusion_analysis_dir: Path + case_id: str, timestamp: datetime, rnafusion_analysis_dir: Path, delivery_report_html: Path ) -> dict: """Get some bundle data for housekeeper.""" return { @@ -433,6 +435,11 @@ def rnafusion_analysis_hk_bundle_data( "archive": False, "tags": ["fusionreport", "research"], }, + { + "path": delivery_report_html.as_posix(), + "archive": False, + "tags": [HK_DELIVERY_REPORT_TAG], + }, ], } @@ -588,14 +595,19 @@ def balsamic_config_builder( @pytest.fixture(name="mip_load_config") def mip_load_config( - mip_dna_analysis_dir: Path, case_id: str, customer_id: str, snv_vcf_file: str + mip_dna_analysis_dir: Path, + case_id: str, + customer_id: str, + snv_vcf_file: str, + delivery_report_html: Path, ) -> MipLoadConfig: """Return a valid MIP load_config.""" return MipLoadConfig( owner=customer_id, family=case_id, vcf_snv=Path(mip_dna_analysis_dir, snv_vcf_file).as_posix(), - track="rare", + track=UploadTrack.RARE_DISEASE.value, + delivery_report=delivery_report_html.as_posix(), ) diff --git a/tests/meta/upload/scout/test_meta_upload_scoutapi.py b/tests/meta/upload/scout/test_meta_upload_scoutapi.py index d0e2f65aa0..7131f03317 100644 --- a/tests/meta/upload/scout/test_meta_upload_scoutapi.py +++ b/tests/meta/upload/scout/test_meta_upload_scoutapi.py @@ -9,7 +9,9 @@ from cg.models.scout.scout_load_config import MipLoadConfig, ScoutLoadConfig -def test_unlinked_family_is_linked(mip_config_builder: MipConfigBuilder): +def test_unlinked_family_is_linked( + mip_config_builder: MipConfigBuilder, delivery_report_html: Path +): """Test that is_family check fails when samples are not linked""" # GIVEN a upload scout api and case data for a case without linked individuals family_data: MipLoadConfig = MipLoadConfig( @@ -17,7 +19,8 @@ def test_unlinked_family_is_linked(mip_config_builder: MipConfigBuilder): "samples": [ {"sample_id": "ADM2", "father": "0", "mother": "0"}, {"sample_id": "ADM3", "father": "0", "mother": "0"}, - ] + ], + "delivery_report": delivery_report_html.as_posix(), } ) # WHEN running the check if case is linked @@ -26,7 +29,7 @@ def test_unlinked_family_is_linked(mip_config_builder: MipConfigBuilder): assert res is False -def test_family_is_linked(mip_config_builder: MipConfigBuilder): +def test_family_is_linked(mip_config_builder: MipConfigBuilder, delivery_report_html: Path): """Test that is_family returns true when samples are linked""" # GIVEN a upload scout api and case data for a linked case family_data: MipLoadConfig = MipLoadConfig( @@ -35,7 +38,8 @@ def test_family_is_linked(mip_config_builder: MipConfigBuilder): {"sample_id": "ADM1", "father": "ADM2", "mother": "ADM3"}, {"sample_id": "ADM2", "father": "0", "mother": "0"}, {"sample_id": "ADM3", "father": "0", "mother": "0"}, - ] + ], + "delivery_report": delivery_report_html.as_posix(), } ) # WHEN running the check if case is linked diff --git a/tests/meta/upload/scout/test_scout_config_builder.py b/tests/meta/upload/scout/test_scout_config_builder.py index 811298718a..eb6eae1b08 100644 --- a/tests/meta/upload/scout/test_scout_config_builder.py +++ b/tests/meta/upload/scout/test_scout_config_builder.py @@ -68,20 +68,6 @@ def test_rnafusion_config_builder( assert isinstance(file_handler.case_tags, CaseTags) -def test_include_delivery_report_mip(mip_config_builder: MipConfigBuilder): - """Test include delivery report.""" - # GIVEN a config builder with data - - # GIVEN a config without a delivery report - assert mip_config_builder.load_config.delivery_report is None - - # WHEN including the delivery report - mip_config_builder.include_delivery_report() - - # THEN assert that the delivery report was added - assert mip_config_builder.load_config.delivery_report is not None - - def test_include_synopsis(mip_config_builder: MipConfigBuilder): """Test include synopsis.""" # GIVEN a config builder with some data diff --git a/tests/mocks/tb_mock.py b/tests/mocks/tb_mock.py index 3355265db2..705e20ef44 100644 --- a/tests/mocks/tb_mock.py +++ b/tests/mocks/tb_mock.py @@ -12,9 +12,6 @@ def is_latest_analysis_ongoing(self, *args, **kwargs) -> bool: def add_pending_analysis(self, *args, **kwargs) -> None: return None - def mark_analyses_deleted(self, *args, **kwargs) -> None: - return None - def add_commit(self, *args, **kwargs) -> None: return None diff --git a/tests/store/api/conftest.py b/tests/store/api/conftest.py index 37e8a046c1..8089d1adba 100644 --- a/tests/store/api/conftest.py +++ b/tests/store/api/conftest.py @@ -465,7 +465,6 @@ def store_with_analyses_for_cases( started_at=timestamp_yesterday, uploaded_at=timestamp_yesterday, delivery_reported_at=None, - uploaded_to_vogue_at=timestamp_yesterday, completed_at=timestamp_yesterday, ) helpers.add_analysis( @@ -474,7 +473,6 @@ def store_with_analyses_for_cases( started_at=timestamp_now, uploaded_at=timestamp_now, delivery_reported_at=None, - uploaded_to_vogue_at=None, completed_at=timestamp_now, ) sample = helpers.add_sample(analysis_store, delivered_at=timestamp_now) @@ -505,7 +503,6 @@ def store_with_analyses_for_cases_not_uploaded_fluffy( started_at=timestamp_yesterday, uploaded_at=timestamp_yesterday, delivery_reported_at=None, - uploaded_to_vogue_at=timestamp_yesterday, pipeline=Pipeline.FLUFFY, ) helpers.add_analysis( @@ -514,7 +511,6 @@ def store_with_analyses_for_cases_not_uploaded_fluffy( started_at=timestamp_now, uploaded_at=None, delivery_reported_at=None, - uploaded_to_vogue_at=timestamp_now, pipeline=Pipeline.FLUFFY, ) sample = helpers.add_sample(analysis_store, delivered_at=timestamp_now) @@ -545,7 +541,6 @@ def store_with_analyses_for_cases_not_uploaded_microsalt( started_at=timestamp_yesterday, uploaded_at=timestamp_yesterday, delivery_reported_at=None, - uploaded_to_vogue_at=timestamp_yesterday, pipeline=Pipeline.MICROSALT, ) helpers.add_analysis( @@ -554,7 +549,6 @@ def store_with_analyses_for_cases_not_uploaded_microsalt( started_at=timestamp_now, uploaded_at=None, delivery_reported_at=None, - uploaded_to_vogue_at=timestamp_now, pipeline=Pipeline.MICROSALT, ) sample = helpers.add_sample(analysis_store, delivered_at=timestamp_now) @@ -584,7 +578,6 @@ def store_with_analyses_for_cases_to_deliver( started_at=timestamp_yesterday, uploaded_at=None, delivery_reported_at=None, - uploaded_to_vogue_at=timestamp_yesterday, completed_at=timestamp_yesterday, pipeline=Pipeline.FLUFFY, ) @@ -594,7 +587,6 @@ def store_with_analyses_for_cases_to_deliver( started_at=timestamp_now, uploaded_at=None, delivery_reported_at=None, - uploaded_to_vogue_at=None, completed_at=timestamp_now, pipeline=Pipeline.MIP_DNA, ) diff --git a/tests/store/conftest.py b/tests/store/conftest.py index aebff67ca5..f9e4bed1d3 100644 --- a/tests/store/conftest.py +++ b/tests/store/conftest.py @@ -401,7 +401,6 @@ def store_with_older_and_newer_analyses( """Return a store with older and newer analyses.""" analysis = base_store._get_query(table=Analysis).first() analysis.uploaded_at = timestamp_now - analysis.uploaded_to_vogue_at = timestamp_now analysis.cleaned_at = timestamp_now analysis.started_at = timestamp_now analysis.completed_at = timestamp_now @@ -416,7 +415,6 @@ def store_with_older_and_newer_analyses( started_at=time, completed_at=time, uploaded_at=time, - uploaded_to_vogue_at=time, cleaned_at=time, ) @@ -442,7 +440,6 @@ def store_with_analyses_for_cases( started_at=timestamp_yesterday, uploaded_at=timestamp_yesterday, delivery_reported_at=None, - uploaded_to_vogue_at=timestamp_yesterday, ) helpers.add_analysis( analysis_store, @@ -450,7 +447,6 @@ def store_with_analyses_for_cases( started_at=timestamp_now, uploaded_at=timestamp_now, delivery_reported_at=None, - uploaded_to_vogue_at=timestamp_now, ) sample = helpers.add_sample(analysis_store, delivered_at=timestamp_now) link: CaseSample = analysis_store.relate_sample( diff --git a/tests/store_helpers.py b/tests/store_helpers.py index c1827ba14c..3d57a7e76c 100644 --- a/tests/store_helpers.py +++ b/tests/store_helpers.py @@ -322,7 +322,6 @@ def add_analysis( data_delivery: DataDelivery = DataDelivery.FASTQ_QC, uploading: bool = False, config_path: str = None, - uploaded_to_vogue_at: datetime = None, ) -> Analysis: """Utility function to add an analysis for tests.""" @@ -346,8 +345,6 @@ def add_analysis( analysis.config_path = config_path if pipeline: analysis.pipeline = str(pipeline) - if uploaded_to_vogue_at: - analysis.uploaded_to_vogue_at = uploaded_to_vogue_at analysis.limitations = "A limitation" analysis.case = case