Change fixture and param names to reflect possibility of multiple dat…

…a sources

cg/apps/demultiplex/sbatch.py

@@ -3,9 +3,9 @@
 DEMULTIPLEX_COMMAND = {
     "bcl2fastq": """
 log "singularity exec --bind \
-/home/proj/{environment}/demultiplexed-runs,\
-/home/proj/{environment}/flow_cells,\
-/home/proj/{environment}/flow_cells/'$SLURM_JOB_ID':/run/user/$(id -u) \
+/home/proj/{environment}/sequencing_data/illumina_novaseq_flow_cells/demultiplexed-runs,\
+/home/proj/{environment}/sequencing_data/illumina_novaseq_flow_cells/flow_cells,\
+/home/proj/{environment}/sequencing_data/illumina_novaseq_flow_cells/flow_cells/'$SLURM_JOB_ID':/run/user/$(id -u) \
 /home/proj/{environment}/demux-on-hasta/novaseq/container/bcl2fastq_v2-20-0.sif \
 bcl2fastq --loading-threads 3 --processing-threads 15 --writing-threads 3 \
 --runfolder-dir {run_dir} --output-dir {unaligned_dir} \
@@ -14,9 +14,9 @@
 touch {demux_completed_file}"
 
 singularity exec --bind \
-/home/proj/{environment}/demultiplexed-runs,\
-/home/proj/{environment}/flow_cells,\
-/home/proj/{environment}/flow_cells/'$SLURM_JOB_ID':/run/user/$(id -u) \
+/home/proj/{environment}/sequencing_data/illumina_novaseq_flow_cells/demultiplexed-runs,\
+/home/proj/{environment}/sequencing_data/illumina_novaseq_flow_cells/flow_cells,\
+/home/proj/{environment}/sequencing_data/illumina_novaseq_flow_cells/flow_cells/'$SLURM_JOB_ID':/run/user/$(id -u) \
 /home/proj/{environment}/demux-on-hasta/novaseq/container/bcl2fastq_v2-20-0.sif \
 bcl2fastq --loading-threads 3 --processing-threads 15 --writing-threads 3 \
 --runfolder-dir {run_dir} --output-dir {unaligned_dir} \

cg/cli/backup.py

@@ -52,7 +52,7 @@ def backup_flow_cells(context: CGConfig, dry_run: bool):
     pdc_api.dry_run = dry_run
     status_db: Store = context.status_db
     flow_cells: list[FlowCellDirectoryData] = get_flow_cells_from_path(
-        flow_cells_dir=Path(context.flow_cells_dir)
+        flow_cells_dir=Path(context.illumina_novaseq_flow_cells_directory)
     )
     for flow_cell in flow_cells:
         db_flow_cell: Flowcell | None = status_db.get_flow_cell_by_name(flow_cell_name=flow_cell.id)
@@ -88,7 +88,7 @@ def encrypt_flow_cells(context: CGConfig, dry_run: bool):
     """Encrypt flow cells."""
     status_db: Store = context.status_db
     flow_cells: list[FlowCellDirectoryData] = get_flow_cells_from_path(
-        flow_cells_dir=Path(context.flow_cells_dir)
+        flow_cells_dir=Path(context.illumina_novaseq_flow_cells_directory)
     )
     for flow_cell in flow_cells:
         db_flow_cell: Flowcell | None = status_db.get_flow_cell_by_name(flow_cell_name=flow_cell.id)
@@ -128,7 +128,7 @@ def fetch_flow_cell(context: CGConfig, dry_run: bool, flow_cell_id: str | None =
         status=context.status_db,
         tar_api=tar_api,
         pdc_api=pdc_api,
-        flow_cells_dir=context.flow_cells_dir,
+        flow_cells_dir=context.illumina_novaseq_flow_cells_directory,
         dry_run=dry_run,
     )
     backup_api: BackupAPI = context.meta_apis["backup_api"]

cg/cli/clean.py

@@ -258,8 +258,8 @@ def clean_flow_cells(context: CGConfig, dry_run: bool):
     directories_to_check: list[Path] = []
     for path in [
         Path(context.data_input.input_dir_path),
-        Path(context.flow_cells_dir),
-        Path(context.demultiplexed_flow_cells_dir),
+        Path(context.illumina_novaseq_flow_cells_directory),
+        Path(context.illumina_novaseq_demultiplexed_flow_cells_directory),
         Path(context.encryption.encryption_dir),
     ]:
         directories_to_check.extend(get_directories_in_path(path))

cg/cli/compress/base.py

@@ -30,7 +30,9 @@ def compress(context: CGConfig):
     crunchy_api = context.crunchy_api
 
     compress_api = CompressAPI(
-        hk_api=hk_api, crunchy_api=crunchy_api, demux_root=context.demultiplexed_flow_cells_dir
+        hk_api=hk_api,
+        crunchy_api=crunchy_api,
+        demux_root=context.illumina_novaseq_demultiplexed_flow_cells_directory,
     )
     context.meta_apis["compress_api"] = compress_api
 
@@ -68,7 +70,7 @@ def decompress(context: CGConfig):
     compress_api = CompressAPI(
         hk_api=hk_api,
         crunchy_api=crunchy_api,
-        demux_root=context.demultiplexed_flow_cells_dir,
+        demux_root=context.illumina_novaseq_demultiplexed_flow_cells_directory,
         backup_api=spring_backup_api,
     )
 

cg/cli/demultiplex/demux.py

@@ -126,8 +126,8 @@ def demultiplex_flow_cell(
 @click.pass_obj
 def copy_novaseqx_flow_cells(context: CGConfig):
     """Copy Novaseqx flow cells ready for post processing to demultiplexed runs."""
-    flow_cells_dir: Path = Path(context.flow_cells_dir)
-    demultiplexed_runs_dir: Path = Path(context.demultiplexed_flow_cells_dir)
+    flow_cells_dir: Path = Path(context.illumina_novaseq_flow_cells_directory)
+    demultiplexed_runs_dir: Path = Path(context.illumina_novaseq_demultiplexed_flow_cells_directory)
 
     for flow_cell_dir in flow_cells_dir.iterdir():
         if is_ready_for_post_processing(
@@ -156,7 +156,7 @@ def copy_novaseqx_flow_cells(context: CGConfig):
 def confirm_flow_cell_sync(context: CGConfig, source_directory: str):
     """Checks if all relevant files for the demultiplexing have been synced.
     If so it creates a CopyComplete.txt file to show that that is the case."""
-    target_flow_cells_directory = Path(context.flow_cells_dir)
+    target_flow_cells_directory = Path(context.illumina_novaseq_flow_cells_directory)
     for source_flow_cell in Path(source_directory).iterdir():
         target_flow_cell = Path(target_flow_cells_directory, source_flow_cell.name)
         if is_flow_cell_sync_confirmed(target_flow_cell):

cg/cli/demultiplex/sample_sheet.py

@@ -6,15 +6,19 @@
 from pydantic import ValidationError
 
 from cg.apps.demultiplex.sample_sheet.create import create_sample_sheet
-from cg.apps.demultiplex.sample_sheet.read_sample_sheet import get_sample_sheet_from_file
+from cg.apps.demultiplex.sample_sheet.read_sample_sheet import (
+    get_sample_sheet_from_file,
+)
 from cg.apps.demultiplex.sample_sheet.sample_models import FlowCellSample
 from cg.apps.housekeeper.hk import HousekeeperAPI
 from cg.apps.lims.sample_sheet import get_flow_cell_samples
 from cg.constants.constants import DRY_RUN, FileFormat
 from cg.constants.demultiplexing import OPTION_BCL_CONVERTER
 from cg.exc import FlowCellError, HousekeeperFileMissingError, SampleSheetError
 from cg.io.controller import WriteFile, WriteStream
-from cg.meta.demultiplex.housekeeper_storage_functions import add_sample_sheet_path_to_housekeeper
+from cg.meta.demultiplex.housekeeper_storage_functions import (
+    add_sample_sheet_path_to_housekeeper,
+)
 from cg.models.cg_config import CGConfig
 from cg.models.flow_cell.flow_cell import FlowCellDirectoryData
 
@@ -56,7 +60,7 @@ def create_sheet(
     """
     LOG.info(f"Creating sample sheet for flow cell {flow_cell_name}")
     hk_api: HousekeeperAPI = context.housekeeper_api
-    flow_cell_path: Path = Path(context.flow_cells_dir, flow_cell_name)
+    flow_cell_path: Path = Path(context.illumina_novaseq_flow_cells_directory, flow_cell_name)
     if not flow_cell_path.exists():
         LOG.warning(f"Could not find flow cell {flow_cell_path}")
         raise click.Abort
@@ -135,7 +139,7 @@ def create_all_sheets(context: CGConfig, dry_run: bool):
     information.
     """
     hk_api: HousekeeperAPI = context.housekeeper_api
-    flow_cell_runs_dir: Path = Path(context.flow_cells_dir)
+    flow_cell_runs_dir: Path = Path(context.illumina_novaseq_flow_cells_directory)
     for sub_dir in flow_cell_runs_dir.iterdir():
         if not sub_dir.is_dir():
             continue

cg/cli/store/store.py

@@ -29,7 +29,9 @@ def store(context: CGConfig):
     crunchy_api: CrunchyAPI = context.crunchy_api
 
     compress_api = CompressAPI(
-        hk_api=housekeeper_api, crunchy_api=crunchy_api, demux_root=context.flow_cells_dir
+        hk_api=housekeeper_api,
+        crunchy_api=crunchy_api,
+        demux_root=context.illumina_novaseq_flow_cells_directory,
     )
     context.meta_apis["compress_api"] = compress_api
 

cg/meta/demultiplex/demux_post_processing.py

@@ -28,8 +28,10 @@ class DemuxPostProcessingAPI:
 
     def __init__(self, config: CGConfig) -> None:
         self.config: CGConfig = config
-        self.flow_cells_dir: Path = Path(config.flow_cells_dir)
-        self.demultiplexed_runs_dir: Path = Path(config.demultiplexed_flow_cells_dir)
+        self.flow_cells_dir: Path = Path(config.illumina_novaseq_flow_cells_directory)
+        self.demultiplexed_runs_dir: Path = Path(
+            config.illumina_novaseq_demultiplexed_flow_cells_directory
+        )
         self.status_db: Store = config.status_db
         self.hk_api: HousekeeperAPI = config.housekeeper_api
         self.dry_run: bool = False

cg/meta/meta.py

@@ -36,7 +36,7 @@ def __init__(self, config: CGConfig):
             compress_api=CompressAPI(
                 hk_api=config.housekeeper_api,
                 crunchy_api=config.crunchy_api,
-                demux_root=config.demultiplexed_flow_cells_dir,
+                demux_root=config.illumina_novaseq_demultiplexed_flow_cells_directory,
                 backup_api=SpringBackupAPI(
                     encryption_api=SpringEncryptionAPI(
                         binary_path=config.dict()["encryption"]["binary_path"]

cg/models/cg_config.py

@@ -250,12 +250,12 @@ class DataFlowConfig(BaseModel):
 class CGConfig(BaseModel):
     database: str
     delivery_path: str
-    demultiplexed_flow_cells_dir: str
+    illumina_novaseq_demultiplexed_flow_cells_directory: str
     downsample_dir: str
     downsample_script: str
     email_base_settings: EmailBaseSettings
     environment: Literal["production", "stage"] = "stage"
-    flow_cells_dir: str
+    illumina_novaseq_flow_cells_directory: str
     madeline_exe: str
     tower_binary_path: str
     max_flowcells: int | None

tests/apps/demultiplex/conftest.py

@@ -8,7 +8,6 @@
 )
 from cg.apps.demultiplex.sample_sheet.sample_sheet_creator import (
     SampleSheetCreatorBcl2Fastq,
-    SampleSheetCreatorBCLConvert,
 )
 from cg.constants.demultiplexing import SampleSheetBcl2FastqSections
 from cg.models.flow_cell.flow_cell import FlowCellDirectoryData
@@ -317,18 +316,18 @@ def bcl_convert_flow_cell_sample(raw_index_sequence: str) -> FlowCellSampleBCLCo
 
 
 @pytest.fixture
-def bcl_convert_sample_sheet_path(demultiplexed_runs: Path):
+def bcl_convert_sample_sheet_path(illumina_novaseq_demultiplexed_runs):
     return Path(
-        demultiplexed_runs,
+        illumina_novaseq_demultiplexed_runs,
         "230504_A00689_0804_BHY7FFDRX2",
         "SampleSheet.csv",
     )
 
 
 @pytest.fixture
-def bcl2fastq_sample_sheet_path(demultiplexed_runs: Path):
+def bcl2fastq_sample_sheet_path(illumina_novaseq_demultiplexed_runs):
     return Path(
-        demultiplexed_runs,
+        illumina_novaseq_demultiplexed_runs,
         "170407_ST-E00198_0209_BHHKVCALXX",
         "SampleSheet.csv",
     )

tests/apps/demultiplex/test_demultiplex_api.py

@@ -18,7 +18,9 @@ def test_is_sample_sheet_in_housekeeper_exists(
     """Test that checking the existence of an existing sample sheet in Housekeeper returns True."""
     # GIVEN a DemultiplexAPI and a flow cell with a sample sheet
     demux_api: DemultiplexingAPI = demultiplexing_context_for_demux.demultiplex_api
-    demultiplexing_context_for_demux.flow_cells_dir = tmp_bcl2fastq_flow_cell.path.parent
+    demultiplexing_context_for_demux.illumina_novaseq_flow_cells_directory = (
+        tmp_bcl2fastq_flow_cell.path.parent
+    )
 
     # GIVEN that the sample sheet is in Housekeeper
     add_sample_sheet_path_to_housekeeper(
@@ -40,7 +42,9 @@ def test_is_sample_sheet_in_housekeeper_not_in_hk(
     """Test that checking the existence of a non-existing sample sheet in Housekeeper returns False."""
     # GIVEN a DemultiplexAPI and a flow cell with a sample sheet
     demux_api: DemultiplexingAPI = demultiplexing_context_for_demux.demultiplex_api
-    demultiplexing_context_for_demux.flow_cells_dir = tmp_bcl2fastq_flow_cell.path.parent
+    demultiplexing_context_for_demux.illumina_novaseq_flow_cells_directory = (
+        tmp_bcl2fastq_flow_cell.path.parent
+    )
 
     # GIVEN that the sample sheet is not in Housekeeper
 

tests/cli/backup/conftest.py

@@ -14,6 +14,6 @@ def backup_context(cg_context: CGConfig) -> CGConfig:
         status=cg_context.status_db,
         tar_api=TarAPI(binary_path=cg_context.tar.binary_path),
         pdc_api=cg_context.pdc_api,
-        flow_cells_dir=cg_context.flow_cells_dir,
+        flow_cells_dir=cg_context.illumina_novaseq_flow_cells_directory,
     )
     return cg_context

tests/cli/clean/conftest.py

@@ -206,13 +206,15 @@ def clean_context_microsalt(
 @pytest.fixture(scope="function")
 def clean_flow_cells_context(
     cg_context: CGConfig,
-    tmp_flow_cells_directory: Path,
-    tmp_demultiplexed_runs_directory: Path,
+    tmp_illumina_novaseq_flow_cells_directory,
+    tmp_illumina_novaseq_demultiplexed_flow_cells_directory,
     store_with_flow_cell_to_clean: Store,
     housekeeper_api_with_flow_cell_to_clean: HousekeeperAPI,
 ) -> CGConfig:
-    cg_context.flow_cells_dir = tmp_flow_cells_directory
-    cg_context.demultiplexed_flow_cells_dir = tmp_demultiplexed_runs_directory
+    cg_context.illumina_novaseq_flow_cells_directory = tmp_illumina_novaseq_flow_cells_directory
+    cg_context.illumina_novaseq_demultiplexed_flow_cells_directory = (
+        tmp_illumina_novaseq_demultiplexed_flow_cells_directory
+    )
     cg_context.housekeeper_api_ = housekeeper_api_with_flow_cell_to_clean
     cg_context.status_db_ = store_with_flow_cell_to_clean
 

tests/cli/conftest.py

@@ -1,5 +1,4 @@
 """Fixtures for CLI tests."""
-from pathlib import Path
 from typing import Any
 
 import pytest
@@ -144,13 +143,15 @@ def real_crunchy_api(crunchy_config: dict[str, dict[str, Any]]):
 
 @pytest.fixture
 def real_compress_api(
-    demultiplexed_runs: Path, housekeeper_api: HousekeeperAPI, real_crunchy_api: CrunchyAPI
+    illumina_novaseq_demultiplexed_runs,
+    housekeeper_api: HousekeeperAPI,
+    real_crunchy_api: CrunchyAPI,
 ) -> CompressAPI:
     """Return a compress API context."""
     return CompressAPI(
         crunchy_api=real_crunchy_api,
         hk_api=housekeeper_api,
-        demux_root=demultiplexed_runs.as_posix(),
+        demux_root=illumina_novaseq_demultiplexed_runs.as_posix(),
     )
 
 

tests/cli/demultiplex/test_create_sample_sheet.py

@@ -40,7 +40,7 @@ def test_create_sample_sheet_no_run_parameters_fails(
     )
 
     # GIVEN that the context's flow cell directory holds the given flow cell
-    sample_sheet_context.flow_cells_dir = (
+    sample_sheet_context.illumina_novaseq_flow_cells_directory = (
         tmp_flow_cells_directory_no_run_parameters.parent.as_posix()
     )
 

tests/cli/demultiplex/test_demultiplex_flowcell.py

@@ -7,7 +7,9 @@
 from cg.apps.demultiplex.demultiplex_api import DemultiplexingAPI
 from cg.cli.demultiplex.demux import demultiplex_all, demultiplex_flow_cell
 from cg.constants.demultiplexing import BclConverter, DemultiplexingDirsAndFiles
-from cg.meta.demultiplex.housekeeper_storage_functions import add_sample_sheet_path_to_housekeeper
+from cg.meta.demultiplex.housekeeper_storage_functions import (
+    add_sample_sheet_path_to_housekeeper,
+)
 from cg.models.cg_config import CGConfig
 from cg.models.flow_cell.flow_cell import FlowCellDirectoryData
 
@@ -110,7 +112,7 @@ def test_demultiplex_dragen_flowcell(
     cli_runner: testing.CliRunner,
     tmp_flow_cell_directory_bclconvert: Path,
     demultiplexing_context_for_demux: CGConfig,
-    tmp_demultiplexed_runs_directory: Path,
+    tmp_illumina_novaseq_demultiplexed_flow_cells_directory,
     caplog,
     mocker,
 ):
@@ -162,18 +164,18 @@ def test_demultiplex_dragen_flowcell(
 def test_demultiplex_all_novaseq(
     cli_runner: testing.CliRunner,
     demultiplexing_context_for_demux: CGConfig,
-    tmp_flow_cells_demux_all_directory: Path,
+    tmp_illumina_novaseq_flow_cells_demux_all_directory,
     caplog,
 ):
     """Test the demultiplex-all command on a directory with newly sequenced NovaSeq6000 flow cells."""
     caplog.set_level(logging.INFO)
 
     # GIVEN a demultiplexing context with an API and correct structure
     demux_api: DemultiplexingAPI = demultiplexing_context_for_demux.demultiplex_api
-    assert demux_api.flow_cells_dir == tmp_flow_cells_demux_all_directory
+    assert demux_api.flow_cells_dir == tmp_illumina_novaseq_flow_cells_demux_all_directory
 
     # GIVEN sequenced flow cells with their sample sheet in Housekeeper
-    for flow_cell_dir in tmp_flow_cells_demux_all_directory.iterdir():
+    for flow_cell_dir in tmp_illumina_novaseq_flow_cells_demux_all_directory.iterdir():
         flow_cell: FlowCellDirectoryData = FlowCellDirectoryData(flow_cell_path=flow_cell_dir)
         add_sample_sheet_path_to_housekeeper(
             flow_cell_directory=flow_cell_dir,

tests/cli/demultiplex/test_finish_demux.py

@@ -1,6 +1,5 @@
 """Test finish demultiplexing CLI."""
 import logging
-from pathlib import Path
 
 from click import testing
 
@@ -13,7 +12,7 @@ def test_finish_all_cmd_dry_run(
     caplog,
     cli_runner: testing.CliRunner,
     demultiplex_context: CGConfig,
-    tmp_demultiplexed_runs_directory: Path,
+    tmp_illumina_novaseq_demultiplexed_flow_cells_directory,
 ):
     caplog.set_level(logging.INFO)
 
@@ -36,7 +35,7 @@ def test_finish_flow_cell_dry_run(
     caplog,
     cli_runner: testing.CliRunner,
     demultiplex_context: CGConfig,
-    tmp_demultiplexed_runs_directory: Path,
+    tmp_illumina_novaseq_demultiplexed_flow_cells_directory,
     bcl2fastq_flow_cell_full_name: str,
 ):
     caplog.set_level(logging.INFO)