Dataset v2 ideas

clinicadl/API_test.py → clinicadl/API/API_test.py

@@ -1,6 +1,6 @@
 from pathlib import Path
 
-import torchio
+import torchio.transforms as transforms
 
 from clinicadl.dataset.caps_dataset import (
     CapsDatasetPatch,
@@ -9,15 +9,19 @@
 )
 from clinicadl.dataset.caps_reader import CapsReader
 from clinicadl.dataset.concat import ConcatDataset
-from clinicadl.dataset.config.extraction import ExtractionConfig
+from clinicadl.dataset.config.extraction import (
+    ExtractionConfig,
+    ExtractionImageConfig,
+    ExtractionPatchConfig,
+)
 from clinicadl.dataset.config.preprocessing import (
     PreprocessingConfig,
     T1PreprocessingConfig,
 )
 from clinicadl.experiment_manager.experiment_manager import ExperimentManager
 from clinicadl.losses.config import CrossEntropyLossConfig
 from clinicadl.losses.factory import get_loss_function
-from clinicadl.model.clinicadl_model import ClinicaDLModel
+from clinicadl.model.clinicadl_model import ClinicaDLModel, ClinicaDLModelClassif
 from clinicadl.networks.config import ImplementedNetworks
 from clinicadl.networks.factory import (
     ConvEncoderOptions,
@@ -30,42 +34,50 @@
 from clinicadl.splitter.kfold import KFolder
 from clinicadl.splitter.split import get_single_split, split_tsv
 from clinicadl.trainer.trainer import Trainer
+from clinicadl.transforms.config import TransformsConfig
 from clinicadl.transforms.transforms import Transforms
 
 # Create the Maps Manager / Read/write manager /
 maps_path = Path("/")
-manager = ExperimentManager(maps_path, overwrite=False)
+manager = ExperimentManager(
+    maps_path, overwrite=False
+)  # a ajouter dans le manager: mlflow/ profiler/ etc ...
 
 caps_directory = Path("caps_directory")  # output of clinica pipelines
-caps_reader = CapsReader(caps_directory, manager=manager)
+caps_reader = CapsReader(caps_directory)
 
 preprocessing_1 = caps_reader.get_preprocessing("t1-linear")
-extraction_1 = caps_reader.extract_slice(preprocessing=preprocessing_1, arg_slice=2)
+caps_reader.prepare_data(
+    preprocessing=preprocessing_1, data_tsv=Path(""), n_proc=2
+)  # don't return anything -> just extract the image tensor and compute some information for each images
+
+extraction_1 = ExtractionImageConfig()
 transforms_1 = Transforms(
-    data_augmentation=[torchio.t1, torchio.t2],
-    image_transforms=[torchio.t1, torchio.t2],
-    object_transforms=[torchio.t1, torchio.t2],
+    extraction=extraction_1,
+    image_augmentation=[transforms.Crop(), transforms.Transform()],
+    image_transforms=[transforms.Blur(), transforms.Ghosting()],
+    object_transforms=[transforms.BiasField(), transforms.Motion()],
 )  # not mandatory
 
 preprocessing_2 = caps_reader.get_preprocessing("pet-linear")
-extraction_2 = caps_reader.extract_patch(preprocessing=preprocessing_2, arg_patch=2)
+caps_reader.prepare_data(preprocessing=preprocessing_2)
+extraction_2 = ExtractionPatchConfig(patch_size=20)
 transforms_2 = Transforms(
-    data_augmentation=[torchio.t2],
-    image_transforms=[torchio.t1],
-    object_transforms=[torchio.t1, torchio.t2],
+    extraction=extraction_2,
+    object_augmentation=[transforms.BiasField()],
+    image_transforms=[transforms.Motion()],
+    object_transforms=[transforms.Crop()],
 )
 
 sub_ses_tsv = Path("")
 split_dir = split_tsv(sub_ses_tsv)  # -> creer un test.tsv et un train.tsv
 
 dataset_t1_roi = caps_reader.get_dataset(
-    extraction=extraction_1,
     preprocessing=preprocessing_1,
     sub_ses_tsv=split_dir / "train.tsv",
     transforms=transforms_1,
 )  # do we give config or object for transforms ?
 dataset_pet_patch = caps_reader.get_dataset(
-    extraction=extraction_2,
     preprocessing=preprocessing_2,
     sub_ses_tsv=split_dir / "train.tsv",
     transforms=transforms_2,
@@ -94,7 +106,21 @@
     )
     network, _ = get_network_from_config(network_config)
     optimizer, _ = get_optimizer(network, AdamConfig())
-    model = ClinicaDLModel(network=network, loss=loss, optimizer=optimizer)
+    model = ClinicaDLModelClassif(network=network, loss=loss, optimizer=optimizer)
+
+    # from config
+    _, loss_config = get_loss_function(CrossEntropyLossConfig())
+    network_config = create_network_config(ImplementedNetworks.CNN)(
+        in_shape=[2, 2, 2],
+        num_outputs=1,
+        conv_args=ConvEncoderOptions(channels=[3, 2, 2]),
+    )
+
+    model = ClinicaDLModelClassif.from_config(
+        network_config=network_config,
+        loss_config=loss_config,
+        optimizer_config=AdamConfig(),
+    )
 
     trainer.train(model, split)
     # le trainer va instancier un predictor/valdiator dans le train ou dans le init
@@ -113,32 +139,30 @@
 )
 network, _ = get_network_from_config(network_config)
 optimizer, _ = get_optimizer(network, AdamConfig())
-model = ClinicaDLModel(network=network, loss=loss, optimizer=optimizer)
+model = ClinicaDLModelClassif(network=network, loss=loss, optimizer=optimizer)
 
 trainer.train(model, split)
 # le trainer va instancier un predictor/valdiator dans le train ou dans le init
 
 
 # TEST
 
-preprocessing_test: PreprocessingConfig = caps_reader.get_preprocessing("pet-linear")
-extraction_test: ExtractionConfig = caps_reader.extract_patch(
-    preprocessing=preprocessing_2, arg_patch=2
-)
+preprocessing_test = caps_reader.get_preprocessing("pet-linear")
+caps_reader.prepare_data(preprocessing=preprocessing_2)
 transforms_test = Transforms(
-    data_augmentation=[torchio.t2],
-    image_transforms=[torchio.t1],
-    object_transforms=[torchio.t1, torchio.t2],
+    extraction=extraction_2,
+    object_augmentation=[transforms.BiasField()],
+    image_transforms=[transforms.Motion()],
+    object_transforms=[transforms.Crop()],
 )
 
 dataset_test = caps_reader.get_dataset(
-    extraction=extraction_test,
     preprocessing=preprocessing_test,
     sub_ses_tsv=split_dir / "test.tsv",
     transforms=transforms_test,
 )
 
-predictor = Predictor(manager=manager)
+predictor = Predictor(model=model, manager=manager)
 predictor.predict(dataset_test=dataset_test, split=2)
 
 
@@ -149,18 +173,25 @@
 manager = ExperimentManager(maps_path, overwrite=False)
 
 caps_directory = Path("caps_directory")  # output of clinica pipelines
-caps_reader = CapsReader(caps_directory, manager=manager)
+caps_reader = CapsReader(caps_directory)
 
-extraction_1 = caps_reader.extract_image(preprocessing=T1PreprocessingConfig())
+preprocessing_config = caps_reader.prepare_data(
+    preprocessing=T1PreprocessingConfig(),
+    data_tsv=Path(""),
+    n_proc=2,
+    use_uncropped_images=False,
+)
 transforms_1 = Transforms(
-    data_augmentation=[torchio.transforms.RandomMotion]
+    object_augmentation=[transforms.RandomMotion],  # default = no transforms
+    image_transforms=[transforms.Noise],  # default = MiniMax
+    extraction=ExtractionPatchConfig(),  # default = Image
+    object_transforms=[transforms.BiasField],  # default = none
 )  # not mandatory
 
 sub_ses_tsv = Path("")
 split_dir = split_tsv(sub_ses_tsv)  # -> creer un test.tsv et un train.tsv
 
 dataset_t1_image = caps_reader.get_dataset(
-    extraction=extraction_1,
     preprocessing=T1PreprocessingConfig(),
     sub_ses_tsv=split_dir / "train.tsv",
     transforms=transforms_1,
@@ -198,10 +229,7 @@
 # sub_ses_tsv = Path("")
 # split_dir = split_tsv(sub_ses_tsv)  # -> creer un test.tsv et un train.tsv
 
-dataset_t1_image = CapsDatasetPatch.from_json(
-    extraction=extract_json,
-    sub_ses_tsv=split_dir / "train.tsv",
-)
+dataset_ = caps_reader.get_dataset_from_json(json_path=Path(""))
 config_file = Path("config_file")
 trainer = Trainer.from_json(config_file=config_file, manager=manager)
 

clinicadl/API/complicated_case.py

@@ -0,0 +1,128 @@
+from pathlib import Path
+
+import torchio.transforms as transforms
+
+from clinicadl.dataset.caps_dataset import (
+    CapsDatasetPatch,
+    CapsDatasetRoi,
+    CapsDatasetSlice,
+)
+from clinicadl.dataset.caps_reader import CapsReader
+from clinicadl.dataset.concat import ConcatDataset
+from clinicadl.dataset.config.extraction import ExtractionConfig
+from clinicadl.dataset.config.preprocessing import (
+    PreprocessingConfig,
+    T1PreprocessingConfig,
+)
+from clinicadl.experiment_manager.experiment_manager import ExperimentManager
+from clinicadl.losses.config import CrossEntropyLossConfig
+from clinicadl.losses.factory import get_loss_function
+from clinicadl.model.clinicadl_model import ClinicaDLModel
+from clinicadl.networks.config import ImplementedNetworks
+from clinicadl.networks.factory import (
+    ConvEncoderOptions,
+    create_network_config,
+    get_network_from_config,
+)
+from clinicadl.optimization.optimizer.config import AdamConfig, OptimizerConfig
+from clinicadl.optimization.optimizer.factory import get_optimizer
+from clinicadl.predictor.predictor import Predictor
+from clinicadl.splitter.kfold import KFolder
+from clinicadl.splitter.split import get_single_split, split_tsv
+from clinicadl.trainer.trainer import Trainer
+from clinicadl.transforms.config import TransformsConfig
+
+# Create the Maps Manager / Read/write manager /
+maps_path = Path("/")
+manager = ExperimentManager(
+    maps_path, overwrite=False
+)  # a ajouter dans le manager: mlflow/ profiler/ etc ...
+
+caps_directory = Path("caps_directory")  # output of clinica pipelines
+caps_reader = CapsReader(caps_directory, manager=manager)
+
+preprocessing_1 = caps_reader.get_preprocessing("t1-linear")
+caps_reader.prepare_data(
+    preprocessing=preprocessing_1, data_tsv=Path(""), n_proc=2
+)  # don't return anything -> just extract the image tensor and compute some information for each images
+
+
+transforms_1 = TransformsConfig(
+    data_augmentation=[transforms.Crop, transforms.Transform],
+    image_transforms=[transforms.Blur, transforms.Ghosting],
+    object_transforms=[transforms.BiasField, transforms.Motion],
+)  # not mandatory
+
+preprocessing_2 = caps_reader.get_preprocessing("pet-linear")
+extraction_2 = caps_reader.extract_patch(preprocessing=preprocessing_2, arg_patch=2)
+transforms_2 = TransformsConfig(
+    data_augmentation=[torchio.t2],
+    image_transforms=[torchio.t1],
+    object_transforms=[torchio.t1, torchio.t2],
+)
+
+sub_ses_tsv = Path("")
+split_dir = split_tsv(sub_ses_tsv)  # -> creer un test.tsv et un train.tsv
+
+dataset_t1_roi = caps_reader.get_dataset(
+    extraction=extraction_1,
+    preprocessing=preprocessing_1,
+    sub_ses_tsv=split_dir / "train.tsv",
+    transforms=transforms_1,
+)  # do we give config or object for transforms ?
+dataset_pet_patch = caps_reader.get_dataset(
+    extraction=extraction_2,
+    preprocessing=preprocessing_2,
+    sub_ses_tsv=split_dir / "train.tsv",
+    transforms=transforms_2,
+)
+
+dataset_multi_modality_multi_extract = ConcatDataset(
+    [dataset_t1_roi, dataset_pet_patch]
+)  # 2 train.tsv en entrée qu'il faut concat et pareil pour les transforms à faire attention
+
+config_file = Path("config_file")
+trainer = Trainer.from_json(config_file=config_file, manager=manager)
+
+# CAS CROSS-VALIDATION
+splitter = KFolder(
+    n_splits=3, caps_dataset=dataset_multi_modality_multi_extract, manager=manager
+)
+
+for split in splitter.split_iterator(split_list=[0, 1]):
+    # bien définir ce qu'il y a dans l'objet split
+
+    loss, loss_config = get_loss_function(CrossEntropyLossConfig())
+    network_config = create_network_config(ImplementedNetworks.CNN)(
+        in_shape=[2, 2, 2],
+        num_outputs=1,
+        conv_args=ConvEncoderOptions(channels=[3, 2, 2]),
+    )
+    network, _ = get_network_from_config(network_config)
+    optimizer, _ = get_optimizer(network, AdamConfig())
+    model = ClinicaDLModel(network=network, loss=loss, optimizer=optimizer)
+
+    trainer.train(model, split)
+    # le trainer va instancier un predictor/valdiator dans le train ou dans le init
+
+# TEST
+
+preprocessing_test: PreprocessingConfig = caps_reader.get_preprocessing("pet-linear")
+extraction_test: ExtractionConfig = caps_reader.extract_patch(
+    preprocessing=preprocessing_2, arg_patch=2
+)
+transforms_test = Transforms(
+    data_augmentation=[torchio.t2],
+    image_transforms=[torchio.t1],
+    object_transforms=[torchio.t1, torchio.t2],
+)
+
+dataset_test = caps_reader.get_dataset(
+    extraction=extraction_test,
+    preprocessing=preprocessing_test,
+    sub_ses_tsv=split_dir / "test.tsv",
+    transforms=transforms_test,
+)
+
+predictor = Predictor(manager=manager)
+predictor.predict(dataset_test=dataset_test, split=2)

clinicadl/API/cross_val.py

@@ -0,0 +1,67 @@
+from pathlib import Path
+
+import torchio.transforms as transforms
+
+from clinicadl.dataset.caps_dataset import (
+    CapsDatasetPatch,
+    CapsDatasetRoi,
+    CapsDatasetSlice,
+)
+from clinicadl.dataset.caps_reader import CapsReader
+from clinicadl.dataset.concat import ConcatDataset
+from clinicadl.dataset.config.extraction import ExtractionConfig
+from clinicadl.dataset.config.preprocessing import (
+    PreprocessingConfig,
+    T1PreprocessingConfig,
+)
+from clinicadl.experiment_manager.experiment_manager import ExperimentManager
+from clinicadl.losses.config import CrossEntropyLossConfig
+from clinicadl.losses.factory import get_loss_function
+from clinicadl.model.clinicadl_model import ClinicaDLModel
+from clinicadl.networks.config import ImplementedNetworks
+from clinicadl.networks.factory import (
+    ConvEncoderOptions,
+    create_network_config,
+    get_network_from_config,
+)
+from clinicadl.optimization.optimizer.config import AdamConfig, OptimizerConfig
+from clinicadl.optimization.optimizer.factory import get_optimizer
+from clinicadl.predictor.predictor import Predictor
+from clinicadl.splitter.kfold import KFolder
+from clinicadl.splitter.split import get_single_split, split_tsv
+from clinicadl.trainer.trainer import Trainer
+from clinicadl.transforms.config import TransformsConfig
+
+# SIMPLE EXPERIMENT WITH A CAPS ALREADY EXISTING
+
+maps_path = Path("/")
+manager = ExperimentManager(maps_path, overwrite=False)
+
+# sub_ses_tsv = Path("")
+# split_dir = split_tsv(sub_ses_tsv)  # -> creer un test.tsv et un train.tsv
+
+dataset_t1_image = CapsDatasetPatch.from_json(
+    extraction=extract_json,
+    sub_ses_tsv=split_dir / "train.tsv",
+)
+config_file = Path("config_file")
+trainer = Trainer.from_json(config_file=config_file, manager=manager)
+
+# CAS CROSS-VALIDATION
+splitter = KFolder(n_splits=3, caps_dataset=dataset_t1_image, manager=manager)
+
+for split in splitter.split_iterator(split_list=[0, 1]):
+    # bien définir ce qu'il y a dans l'objet split
+
+    loss, loss_config = get_loss_function(CrossEntropyLossConfig())
+    network_config = create_network_config(ImplementedNetworks.CNN)(
+        in_shape=[2, 2, 2],
+        num_outputs=1,
+        conv_args=ConvEncoderOptions(channels=[3, 2, 2]),
+    )
+    network, _ = get_network_from_config(network_config)
+    optimizer, _ = get_optimizer(network, AdamConfig())
+    model = ClinicaDLModel(network=network, loss=loss, optimizer=optimizer)
+
+    trainer.train(model, split)
+    # le trainer va instancier un predictor/valdiator dans le train ou dans le init