refactor demultiplex fixtures - organise flow cell fixtures pt 1 (#2847…

…)(patch)

## Description
Part of #2842. First attempt to leave only fixtures of only 7 flow cells. Removed and renamed flow cell fixtures and their files. PR was growing too much so decided to stop this one here and continue in another PR.

### Added

- Directory `flow_cells_broken` to distinguish the files from  working flow cells from flow cells missing files (required by some tests)

### Changed

- Renamed several fixtures

### Fixed

- Removed fixtures and files for flow cells 180522_A00689_0200_BHLCKNCCXY and 170407_A00689_0209_BHHKVCALXX

tests/apps/demultiplex/conftest.py

@@ -46,12 +46,12 @@ def bcl_convert_samples_similar_index2() -> list[FlowCellSampleBCLConvert]:
 
 @pytest.fixture
 def bcl2fastq_sample_sheet_creator(
-    novaseq_flow_cell_demultiplexed_with_bcl2fastq: FlowCellDirectoryData,
+    novaseq_6000_pre_1_5_kits_flow_cell_bcl2fastq: FlowCellDirectoryData,
     novaseq_6000_pre_1_5_kits_bcl2fastq_lims_samples: list[FlowCellSampleBcl2Fastq],
 ) -> SampleSheetCreatorBcl2Fastq:
     """Returns a sample sheet creator for version 1 sample sheets with bcl2fastq format."""
     return SampleSheetCreatorBcl2Fastq(
-        flow_cell=novaseq_flow_cell_demultiplexed_with_bcl2fastq,
+        flow_cell=novaseq_6000_pre_1_5_kits_flow_cell_bcl2fastq,
         lims_samples=novaseq_6000_pre_1_5_kits_bcl2fastq_lims_samples,
     )
 

tests/apps/demultiplex/test_demultiplex_api.py

@@ -5,9 +5,7 @@
 
 from cg.apps.demultiplex.demultiplex_api import DemultiplexingAPI
 from cg.constants.demultiplexing import DemultiplexingDirsAndFiles
-from cg.meta.demultiplex.housekeeper_storage_functions import (
-    add_sample_sheet_path_to_housekeeper,
-)
+from cg.meta.demultiplex.housekeeper_storage_functions import add_sample_sheet_path_to_housekeeper
 from cg.models.cg_config import CGConfig
 from cg.models.flow_cell.flow_cell import FlowCellDirectoryData
 
@@ -199,15 +197,19 @@ def test_is_demultiplexing_possible_already_started(
 def test_remove_demultiplexing_output_directory(
     demultiplexing_api: DemultiplexingAPI,
     tmp_path: Path,
-    bcl_convert_flow_cell: FlowCellDirectoryData,
+    novaseq_6000_post_1_5_kits_flow_cell: FlowCellDirectoryData,
 ):
     """Test that the demultiplexing output directory is removed."""
     # GIVEN a flow cell with a demultiplexing output directory
     demultiplexing_api.demultiplexed_runs_dir = tmp_path
-    demultiplexing_api.create_demultiplexing_output_dir(bcl_convert_flow_cell)
-    assert demultiplexing_api.flow_cell_out_dir_path(bcl_convert_flow_cell).exists()
+    demultiplexing_api.create_demultiplexing_output_dir(novaseq_6000_post_1_5_kits_flow_cell)
+    assert demultiplexing_api.flow_cell_out_dir_path(novaseq_6000_post_1_5_kits_flow_cell).exists()
 
     # WHEN removing the demultiplexing output directory
-    demultiplexing_api.remove_demultiplexing_output_directory(flow_cell=bcl_convert_flow_cell)
+    demultiplexing_api.remove_demultiplexing_output_directory(
+        flow_cell=novaseq_6000_post_1_5_kits_flow_cell
+    )
 
-    assert not demultiplexing_api.flow_cell_out_dir_path(bcl_convert_flow_cell).exists()
+    assert not demultiplexing_api.flow_cell_out_dir_path(
+        novaseq_6000_post_1_5_kits_flow_cell
+    ).exists()

tests/apps/demultiplex/test_sample_sheet_creator.py

@@ -98,13 +98,13 @@ def test_remove_unwanted_samples_dual_index(
 
 def test_remove_unwanted_samples_no_dual_index(
     novaseq6000_flow_cell_sample_no_dual_index: FlowCellSampleBcl2Fastq,
-    novaseq_6000_flow_cell: FlowCellDirectoryData,
+    novaseq_6000_post_1_5_kits_flow_cell: FlowCellDirectoryData,
     caplog,
 ):
     """Test that samples with no dual index are removed."""
     # GIVEN a sample sheet creator with a sample without dual indexes
     sample_sheet_creator: SampleSheetCreatorBcl2Fastq = SampleSheetCreatorBcl2Fastq(
-        flow_cell=novaseq_6000_flow_cell,
+        flow_cell=novaseq_6000_post_1_5_kits_flow_cell,
         lims_samples=[novaseq6000_flow_cell_sample_no_dual_index],
     )
 

tests/apps/demultiplex/test_sample_sheet_creator_factory.py

@@ -16,18 +16,18 @@
 
 
 def test_sample_sheet_creator_factory_novaseq_6000(
-    novaseq_flow_cell_demultiplexed_with_bcl2fastq: FlowCellDirectoryData,
+    novaseq_6000_pre_1_5_kits_flow_cell_bcl2fastq: FlowCellDirectoryData,
     novaseq_6000_pre_1_5_kits_bcl2fastq_lims_samples: list[FlowCellSampleBcl2Fastq],
 ):
     """Test that a sample sheet creator defined with NovaSeq6000 data is V1."""
     # GIVEN a NovaSeq6000 flow cell and a list of NovaSeq6000 samples
 
     # GIVEN that the flow cell is demultiplexed with bcl2fastq
-    assert novaseq_flow_cell_demultiplexed_with_bcl2fastq.bcl_converter == BclConverter.BCL2FASTQ
+    assert novaseq_6000_pre_1_5_kits_flow_cell_bcl2fastq.bcl_converter == BclConverter.BCL2FASTQ
 
     # WHEN defining the sample sheet creator
     sample_sheet_creator: SampleSheetCreator = get_sample_sheet_creator(
-        flow_cell=novaseq_flow_cell_demultiplexed_with_bcl2fastq,
+        flow_cell=novaseq_6000_pre_1_5_kits_flow_cell_bcl2fastq,
         lims_samples=novaseq_6000_pre_1_5_kits_bcl2fastq_lims_samples,
         force=False,
     )

tests/cli/demultiplex/test_create_sample_sheet.py

@@ -21,27 +21,27 @@
 
 def test_create_sample_sheet_no_run_parameters_fails(
     cli_runner: testing.CliRunner,
-    tmp_flow_cells_directory_no_run_parameters: Path,
+    tmp_flow_cell_without_run_parameters_path: Path,
     sample_sheet_context: CGConfig,
-    novaseq_6000_pre_1_5_kits_bcl2fastq_lims_samples: list[FlowCellSampleBcl2Fastq],
+    hiseq_2500_custom_index_bcl_convert_lims_samples: list[FlowCellSampleBcl2Fastq],
     caplog,
     mocker,
 ):
     """Test that creating a flow cell sample sheet fails if there is no run parameters file."""
     # GIVEN a folder with a non-existing sample sheet nor RunParameters file
     flow_cell: FlowCellDirectoryData = FlowCellDirectoryData(
-        flow_cell_path=tmp_flow_cells_directory_no_run_parameters
+        flow_cell_path=tmp_flow_cell_without_run_parameters_path
     )
 
     # GIVEN flow cell samples
     mocker.patch(
         FLOW_CELL_FUNCTION_NAME,
-        return_value=novaseq_6000_pre_1_5_kits_bcl2fastq_lims_samples,
+        return_value=hiseq_2500_custom_index_bcl_convert_lims_samples,
     )
 
     # GIVEN that the context's flow cell directory holds the given flow cell
     sample_sheet_context.flow_cells_dir = (
-        tmp_flow_cells_directory_no_run_parameters.parent.as_posix()
+        tmp_flow_cell_without_run_parameters_path.parent.as_posix()
     )
 
     # WHEN running the create sample sheet command
@@ -58,15 +58,15 @@ def test_create_sample_sheet_no_run_parameters_fails(
 
 def test_create_bcl2fastq_sample_sheet(
     cli_runner: testing.CliRunner,
-    tmp_flow_cells_directory_no_sample_sheet: Path,
+    tmp_novaseq_6000_pre_1_5_kits_flow_cell_without_sample_sheet_path: Path,
     sample_sheet_context: CGConfig,
     novaseq_6000_pre_1_5_kits_bcl2fastq_lims_samples: list[FlowCellSampleBcl2Fastq],
     mocker,
 ):
     """Test that creating a Bcl2fastq sample sheet works."""
     # GIVEN a flowcell directory with some run parameters
     flow_cell: FlowCellDirectoryData = FlowCellDirectoryData(
-        flow_cell_path=tmp_flow_cells_directory_no_sample_sheet,
+        flow_cell_path=tmp_novaseq_6000_pre_1_5_kits_flow_cell_without_sample_sheet_path,
         bcl_converter=BclConverter.BCL2FASTQ,
     )
     assert flow_cell.run_parameters_path.exists()
@@ -89,7 +89,11 @@ def test_create_bcl2fastq_sample_sheet(
     # WHEN creating a sample sheet
     result = cli_runner.invoke(
         create_sheet,
-        [str(tmp_flow_cells_directory_no_sample_sheet), "--bcl-converter", BclConverter.BCL2FASTQ],
+        [
+            str(tmp_novaseq_6000_pre_1_5_kits_flow_cell_without_sample_sheet_path),
+            "--bcl-converter",
+            BclConverter.BCL2FASTQ,
+        ],
         obj=sample_sheet_context,
     )
 
@@ -116,36 +120,34 @@ class SampleSheetScenario(BaseModel):
     "scenario",
     [
         SampleSheetScenario(
-            flow_cell_directory="novaseq_6000_pre_1_5_kits_flow_cell_path",
+            flow_cell_directory="tmp_novaseq_6000_pre_1_5_kits_flow_cell_without_sample_sheet_path",
             lims_samples="novaseq_6000_pre_1_5_kits_bcl_convert_lims_samples",
             correct_sample_sheet="novaseq_6000_pre_1_5_kits_correct_sample_sheet_path",
         ),
         SampleSheetScenario(
-            flow_cell_directory="novaseq_6000_post_1_5_kits_flow_cell_path",
+            flow_cell_directory="tmp_novaseq_6000_post_1_5_kits_flow_cell_without_sample_sheet_path",
             lims_samples="novaseq_6000_post_1_5_kits_bcl_convert_lims_samples",
             correct_sample_sheet="novaseq_6000_post_1_5_kits_correct_sample_sheet_path",
         ),
         SampleSheetScenario(
-            flow_cell_directory="novaseq_x_flow_cell_directory",
+            flow_cell_directory="tmp_novaseq_x_without_sample_sheet_flow_cell_path",
             lims_samples="novaseq_x_lims_samples",
             correct_sample_sheet="novaseq_x_correct_sample_sheet",
         ),
     ],
     ids=["Old NovaSeq 6000 flow cell", "New NovaSeq 6000 flow cell", "NovaSeq X flow cell"],
 )
-def test_create_dragen_sample_sheet(
+def test_create_v2_sample_sheet(
     cli_runner: testing.CliRunner,
     scenario: SampleSheetScenario,
     sample_sheet_context: CGConfig,
     mocker,
     request: FixtureRequest,
 ):
-    """Test that creating a Dragen sample sheet works."""
+    """Test that creating a v2 sample sheet works."""
     flow_cell_directory: Path = request.getfixturevalue(scenario.flow_cell_directory)
     # GIVEN a flow cell directory with some run parameters
-    flow_cell: FlowCellDirectoryData = FlowCellDirectoryData(
-        flow_cell_directory, bcl_converter=BclConverter.BCLCONVERT
-    )
+    flow_cell: FlowCellDirectoryData = FlowCellDirectoryData(flow_cell_directory)
     assert flow_cell.run_parameters_path.exists()
 
     # GIVEN that there is no sample sheet in the flow cell dir

tests/conftest.py

@@ -1594,11 +1594,9 @@ def microsalt_dir(tmpdir_factory) -> Path:
 
 
 @pytest.fixture
-def pdc_archiving_dir(
-    tmp_flow_cell_name_no_run_parameters: str, tmp_flow_cells_directory: Path, tmp_path
-) -> Path:
+def pdc_archiving_dir(tmp_flow_cell_without_run_parameters_path: Path) -> Path:
     """Return a temporary directory for PDC archiving testing."""
-    return Path(tmp_flow_cells_directory, tmp_flow_cell_name_no_run_parameters)
+    return tmp_flow_cell_without_run_parameters_path
 
 
 @pytest.fixture

tests/fixture_plugins/demultiplex_fixtures/flow_cell_fixtures.py

@@ -6,7 +6,7 @@
 from cg.constants.demultiplexing import BclConverter, DemultiplexingDirsAndFiles
 from cg.models.flow_cell.flow_cell import FlowCellDirectoryData
 
-# Functional flow cells
+# Functional flow cell runs
 
 
 @pytest.fixture(scope="module")
@@ -42,61 +42,32 @@ def hiseq_2500_custom_index_flow_cell(
 
 
 @pytest.fixture()
-def novaseq_6000_post_1_5_kits_flow_cell_data(flow_cells_dir: Path) -> FlowCellDirectoryData:
-    return FlowCellDirectoryData(Path(flow_cells_dir, "230912_A00187_1009_AHK33MDRX3"))
-
-
-@pytest.fixture()
-def novaseq_6000_pre_1_5_kits_flow_cell_data(flow_cells_dir: Path) -> FlowCellDirectoryData:
+def novaseq_6000_pre_1_5_kits_flow_cell(flow_cells_dir: Path) -> FlowCellDirectoryData:
+    """Return a Novaseq6000 flow cell with index settings pre 1.5 kits."""
     return FlowCellDirectoryData(Path(flow_cells_dir, "190927_A00689_0069_BHLYWYDSXX"))
 
 
 @pytest.fixture()
-def novaseq_x_flow_cell_data(flow_cells_dir: Path) -> FlowCellDirectoryData:
-    return FlowCellDirectoryData(Path(flow_cells_dir, "20231108_LH00188_0028_B22F52TLT3"))
-
-
-# Broken flow cells
-
-
-@pytest.fixture(scope="session")
-def bcl2fastq_flow_cell(bcl2fastq_flow_cell_dir: Path) -> FlowCellDirectoryData:
-    """Create a flow cell object with flow cell that is demultiplexed."""
+def novaseq_6000_pre_1_5_kits_flow_cell_bcl2fastq(flow_cells_dir: Path) -> FlowCellDirectoryData:
+    """Return a Novaseq6000 flow cell with index settings pre 1.5 kits set as Bcl2Fastq."""
     return FlowCellDirectoryData(
-        flow_cell_path=bcl2fastq_flow_cell_dir, bcl_converter=BclConverter.BCL2FASTQ
+        Path(flow_cells_dir, "190927_A00689_0069_BHLYWYDSXX"), bcl_converter=BclConverter.BCL2FASTQ
     )
 
 
-@pytest.fixture(scope="session")
-def novaseq_flow_cell_demultiplexed_with_bcl2fastq(
-    bcl_convert_flow_cell_dir: Path,
-) -> FlowCellDirectoryData:
-    """Return a Novaseq6000 flow cell object demultiplexed using Bcl2fastq."""
-    return FlowCellDirectoryData(
-        flow_cell_path=bcl_convert_flow_cell_dir, bcl_converter=BclConverter.BCL2FASTQ
-    )
-
-
-@pytest.fixture(scope="module")
-def bcl_convert_flow_cell(bcl_convert_flow_cell_dir: Path) -> FlowCellDirectoryData:
-    """Create a bcl_convert flow cell object with flow cell that is demultiplexed."""
-    return FlowCellDirectoryData(
-        flow_cell_path=bcl_convert_flow_cell_dir, bcl_converter=BclConverter.BCLCONVERT
-    )
+@pytest.fixture()
+def novaseq_6000_post_1_5_kits_flow_cell(flow_cells_dir: Path) -> FlowCellDirectoryData:
+    """Return a Novaseq6000 flow cell with index settings post 1.5 kits."""
+    return FlowCellDirectoryData(Path(flow_cells_dir, "230912_A00187_1009_AHK33MDRX3"))
 
 
-@pytest.fixture(scope="function")
-def novaseq_6000_flow_cell(bcl_convert_flow_cell: FlowCellDirectoryData) -> FlowCellDirectoryData:
-    """Return a NovaSeq6000 flow cell object."""
-    return bcl_convert_flow_cell
+@pytest.fixture()
+def novaseq_x_flow_cell(novaseq_x_flow_cell_dir: Path) -> FlowCellDirectoryData:
+    """Return a NovaSeqX flow cell."""
+    return FlowCellDirectoryData(novaseq_x_flow_cell_dir)
 
 
-@pytest.fixture(scope="function")
-def novaseq_x_flow_cell(novaseq_x_flow_cell_directory: Path) -> FlowCellDirectoryData:
-    """Create a NovaSeqX flow cell object with flow cell that is demultiplexed."""
-    return FlowCellDirectoryData(
-        flow_cell_path=novaseq_x_flow_cell_directory, bcl_converter=BclConverter.BCLCONVERT
-    )
+# Demultiplexed runs
 
 
 @pytest.fixture()
@@ -113,7 +84,7 @@ def novaseqx_flow_cell_with_sample_sheet_no_fastq(
     return flow_cell
 
 
-@pytest.fixture(name="tmp_bcl2fastq_flow_cell")
+@pytest.fixture
 def tmp_bcl2fastq_flow_cell(
     tmp_demultiplexed_runs_bcl2fastq_directory: Path,
 ) -> FlowCellDirectoryData:
@@ -125,17 +96,6 @@ def tmp_bcl2fastq_flow_cell(
 
 
 @pytest.fixture
-def novaseq6000_flow_cell(
-    tmp_flow_cells_directory_malformed_sample_sheet: Path,
-) -> FlowCellDirectoryData:
-    """Return a NovaSeq6000 flow cell."""
-    return FlowCellDirectoryData(
-        flow_cell_path=tmp_flow_cells_directory_malformed_sample_sheet,
-        bcl_converter=BclConverter.BCLCONVERT,
-    )
-
-
-@pytest.fixture(name="tmp_bcl_convert_flow_cell")
 def tmp_bcl_convert_flow_cell(
     tmp_flow_cell_directory_bclconvert: Path,
 ) -> FlowCellDirectoryData:
@@ -146,7 +106,7 @@ def tmp_bcl_convert_flow_cell(
     )
 
 
-@pytest.fixture(name="tmp_unfinished_bcl2fastq_flow_cell")
+@pytest.fixture
 def unfinished_bcl2fastq_flow_cell(
     demultiplexed_runs_unfinished_bcl2fastq_flow_cell_directory: Path,
 ) -> FlowCellDirectoryData:
@@ -160,13 +120,15 @@ def unfinished_bcl2fastq_flow_cell(
 # Flow cell attributes
 
 
-@pytest.fixture(scope="session")
-def bcl2fastq_flow_cell_id(bcl2fastq_flow_cell: FlowCellDirectoryData) -> str:
+@pytest.fixture
+def bcl2fastq_flow_cell_id(
+    novaseq_6000_pre_1_5_kits_flow_cell_bcl2fastq: FlowCellDirectoryData,
+) -> str:
     """Return flow cell id from bcl2fastq flow cell object."""
-    return bcl2fastq_flow_cell.id
+    return novaseq_6000_pre_1_5_kits_flow_cell_bcl2fastq.id
 
 
-@pytest.fixture(scope="module")
-def bcl_convert_flow_cell_id(bcl_convert_flow_cell: FlowCellDirectoryData) -> str:
+@pytest.fixture
+def bcl_convert_flow_cell_id(novaseq_6000_post_1_5_kits_flow_cell: FlowCellDirectoryData) -> str:
     """Return flow cell id from bcl_convert flow cell object."""
-    return bcl_convert_flow_cell.id
+    return novaseq_6000_post_1_5_kits_flow_cell.id

tests/fixture_plugins/demultiplex_fixtures/name_fixtures.py

@@ -1,12 +1,6 @@
 import pytest
 
 
-@pytest.fixture
-def tmp_flow_cell_name_no_run_parameters() -> str:
-    """This is the name of a flow cell directory with the run parameters missing."""
-    return "180522_A00689_0200_BHLCKNCCXY"
-
-
 @pytest.fixture
 def tmp_flow_cell_name_malformed_sample_sheet() -> str:
     """ "Returns the name of a flow cell directory ready for demultiplexing with BCL convert.
@@ -15,12 +9,6 @@ def tmp_flow_cell_name_malformed_sample_sheet() -> str:
     return "201203_A00689_0200_AHVKJCDRXY"
 
 
-@pytest.fixture
-def tmp_flow_cell_name_no_sample_sheet() -> str:
-    """Return the name of a flow cell directory with the run parameters and sample sheet missing."""
-    return "170407_A00689_0209_BHHKVCALXX"
-
-
 @pytest.fixture(name="tmp_flow_cell_name_ready_for_demultiplexing_bcl2fastq")
 def tmp_flow_cell_name_ready_for_demultiplexing_bcl2fastq() -> str:
     """Returns the name of a flow cell directory ready for demultiplexing with bcl2fastq."""