[ENH] Add derivatives metadata (#844)

qsiprep/interfaces/bids.py

@@ -22,7 +22,7 @@
 import gzip
 import os
 import re
-from json import loads
+from json import dump, loads
 from shutil import copyfileobj
 
 from bids.layout import Config
@@ -38,7 +38,7 @@
     isdefined,
     traits,
 )
-from nipype.utils.filemanip import copyfile
+from nipype.utils.filemanip import copyfile, fname_presuffix
 from niworkflows.interfaces.bids import DerivativesDataSink as BaseDerivativesDataSink
 from niworkflows.interfaces.bids import _DerivativesDataSinkInputSpec
 
@@ -252,6 +252,29 @@ def _run_interface(self, runtime):
         return super(DerivativesMaybeDataSink, self)._run_interface(runtime)
 
 
+class _DerivativesSidecarInputSpec(BaseInterfaceInputSpec):
+    sidecar_data = traits.Dict()
+    source_file = File()
+
+
+class _DerivativesSidecarOutputSpec(TraitedSpec):
+    derivatives_json = File(exists=True, desc="Derivatives Sidecar")
+
+
+class DerivativesSidecar(SimpleInterface):
+    input_spec = _DerivativesSidecarInputSpec
+    output_spec = _DerivativesSidecarOutputSpec
+
+    def _run_interface(self, runtime):
+        json_fname = fname_presuffix(
+            self.inputs.source_file, use_ext=False, suffix=".json", newpath=runtime.cwd
+        )
+        with open(json_fname, "w") as jsonf:
+            dump(self.inputs.sidecar_data, jsonf, sort_keys=True, indent=4)
+        self._results["derivatives_json"] = json_fname
+        return runtime
+
+
 def _splitext(fname):
     fname, ext = os.path.splitext(os.path.basename(fname))
     if ext == ".gz":

qsiprep/tests/data/drbuddi_rpe_outputs.txt

@@ -16,5 +16,6 @@ sub-tinytensors/dwi/sub-tinytensors_space-T1w_desc-preproc_dwi.b
 sub-tinytensors/dwi/sub-tinytensors_space-T1w_desc-preproc_dwi.bval
 sub-tinytensors/dwi/sub-tinytensors_space-T1w_desc-preproc_dwi.bvec
 sub-tinytensors/dwi/sub-tinytensors_space-T1w_desc-preproc_dwi.nii.gz
+sub-tinytensors/dwi/sub-tinytensors_space-T1w_desc-preproc_dwi.json
 sub-tinytensors/dwi/sub-tinytensors_space-T1w_desc-preproc_dwi.b_table.txt
 sub-tinytensors/dwi/sub-tinytensors_space-T1w_dwiref.nii.gz

qsiprep/tests/data/drbuddi_shoreline_epi_outputs.txt

@@ -16,5 +16,6 @@ sub-tinytensors/dwi/sub-tinytensors_dir-PA_space-T1w_desc-preproc_dwi.b
 sub-tinytensors/dwi/sub-tinytensors_dir-PA_space-T1w_desc-preproc_dwi.bval
 sub-tinytensors/dwi/sub-tinytensors_dir-PA_space-T1w_desc-preproc_dwi.bvec
 sub-tinytensors/dwi/sub-tinytensors_dir-PA_space-T1w_desc-preproc_dwi.nii.gz
+sub-tinytensors/dwi/sub-tinytensors_dir-PA_space-T1w_desc-preproc_dwi.json
 sub-tinytensors/dwi/sub-tinytensors_dir-PA_space-T1w_desc-preproc_dwi.b_table.txt
 sub-tinytensors/dwi/sub-tinytensors_dir-PA_space-T1w_dwiref.nii.gz

qsiprep/tests/data/drbuddi_tensorline_epi_outputs.txt

@@ -15,6 +15,7 @@ sub-PNC/dwi/sub-PNC_acq-realistic_space-T1w_desc-preproc_dwi.b
 sub-PNC/dwi/sub-PNC_acq-realistic_space-T1w_desc-preproc_dwi.bval
 sub-PNC/dwi/sub-PNC_acq-realistic_space-T1w_desc-preproc_dwi.bvec
 sub-PNC/dwi/sub-PNC_acq-realistic_space-T1w_desc-preproc_dwi.nii.gz
+sub-PNC/dwi/sub-PNC_acq-realistic_space-T1w_desc-preproc_dwi.json
 sub-PNC/dwi/sub-PNC_acq-realistic_space-T1w_desc-preproc_dwi.b_table.txt
 sub-PNC/dwi/sub-PNC_acq-realistic_space-T1w_desc-tensor_cnr.nii.gz
 sub-PNC/dwi/sub-PNC_acq-realistic_space-T1w_dwiref.nii.gz

qsiprep/tests/data/dscsdsi_outputs.txt

@@ -27,5 +27,6 @@ sub-tester/dwi/sub-tester_acq-HASC55AP_space-T1w_desc-preproc_dwi.b
 sub-tester/dwi/sub-tester_acq-HASC55AP_space-T1w_desc-preproc_dwi.bval
 sub-tester/dwi/sub-tester_acq-HASC55AP_space-T1w_desc-preproc_dwi.bvec
 sub-tester/dwi/sub-tester_acq-HASC55AP_space-T1w_desc-preproc_dwi.nii.gz
+sub-tester/dwi/sub-tester_acq-HASC55AP_space-T1w_desc-preproc_dwi.json
 sub-tester/dwi/sub-tester_acq-HASC55AP_space-T1w_desc-preproc_dwi.b_table.txt
 sub-tester/dwi/sub-tester_acq-HASC55AP_space-T1w_dwiref.nii.gz

qsiprep/tests/data/dsdti_nofmap_outputs.txt

@@ -27,5 +27,6 @@ sub-PNC/dwi/sub-PNC_acq-realistic_space-T1w_desc-preproc_dwi.b
 sub-PNC/dwi/sub-PNC_acq-realistic_space-T1w_desc-preproc_dwi.bval
 sub-PNC/dwi/sub-PNC_acq-realistic_space-T1w_desc-preproc_dwi.bvec
 sub-PNC/dwi/sub-PNC_acq-realistic_space-T1w_desc-preproc_dwi.nii.gz
+sub-PNC/dwi/sub-PNC_acq-realistic_space-T1w_desc-preproc_dwi.json
 sub-PNC/dwi/sub-PNC_acq-realistic_space-T1w_desc-preproc_dwi.b_table.txt
 sub-PNC/dwi/sub-PNC_acq-realistic_space-T1w_dwiref.nii.gz

qsiprep/tests/data/dsdti_synfmap_outputs.txt

@@ -27,5 +27,6 @@ sub-PNC/dwi/sub-PNC_acq-realistic_space-T1w_desc-preproc_dwi.b
 sub-PNC/dwi/sub-PNC_acq-realistic_space-T1w_desc-preproc_dwi.bval
 sub-PNC/dwi/sub-PNC_acq-realistic_space-T1w_desc-preproc_dwi.bvec
 sub-PNC/dwi/sub-PNC_acq-realistic_space-T1w_desc-preproc_dwi.nii.gz
+sub-PNC/dwi/sub-PNC_acq-realistic_space-T1w_desc-preproc_dwi.json
 sub-PNC/dwi/sub-PNC_acq-realistic_space-T1w_desc-preproc_dwi.b_table.txt
 sub-PNC/dwi/sub-PNC_acq-realistic_space-T1w_dwiref.nii.gz

qsiprep/tests/data/dsdti_topup_outputs.txt

@@ -27,5 +27,6 @@ sub-PNC/dwi/sub-PNC_acq-realistic_space-T1w_desc-preproc_dwi.b
 sub-PNC/dwi/sub-PNC_acq-realistic_space-T1w_desc-preproc_dwi.bval
 sub-PNC/dwi/sub-PNC_acq-realistic_space-T1w_desc-preproc_dwi.bvec
 sub-PNC/dwi/sub-PNC_acq-realistic_space-T1w_desc-preproc_dwi.nii.gz
+sub-PNC/dwi/sub-PNC_acq-realistic_space-T1w_desc-preproc_dwi.json
 sub-PNC/dwi/sub-PNC_acq-realistic_space-T1w_desc-preproc_dwi.b_table.txt
 sub-PNC/dwi/sub-PNC_acq-realistic_space-T1w_dwiref.nii.gz

qsiprep/tests/data/forrest_gump_outputs.txt

@@ -29,5 +29,6 @@ sub-01/ses-forrestgump/dwi/sub-01_ses-forrestgump_space-T1w_desc-preproc_dwi.b
 sub-01/ses-forrestgump/dwi/sub-01_ses-forrestgump_space-T1w_desc-preproc_dwi.bval
 sub-01/ses-forrestgump/dwi/sub-01_ses-forrestgump_space-T1w_desc-preproc_dwi.bvec
 sub-01/ses-forrestgump/dwi/sub-01_ses-forrestgump_space-T1w_desc-preproc_dwi.nii.gz
+sub-01/ses-forrestgump/dwi/sub-01_ses-forrestgump_space-T1w_desc-preproc_dwi.json
 sub-01/ses-forrestgump/dwi/sub-01_ses-forrestgump_space-T1w_desc-preproc_dwi.b_table.txt
 sub-01/ses-forrestgump/dwi/sub-01_ses-forrestgump_space-T1w_dwiref.nii.gz

qsiprep/tests/data/intramodal_template_outputs.txt

@@ -50,6 +50,7 @@ sub-tester/ses-1/dwi/sub-tester_ses-1_acq-HASC55PA_space-T1w_desc-preproc_dwi.b
 sub-tester/ses-1/dwi/sub-tester_ses-1_acq-HASC55PA_space-T1w_desc-preproc_dwi.bval
 sub-tester/ses-1/dwi/sub-tester_ses-1_acq-HASC55PA_space-T1w_desc-preproc_dwi.bvec
 sub-tester/ses-1/dwi/sub-tester_ses-1_acq-HASC55PA_space-T1w_desc-preproc_dwi.nii.gz
+sub-tester/ses-1/dwi/sub-tester_ses-1_acq-HASC55PA_space-T1w_desc-preproc_dwi.json
 sub-tester/ses-1/dwi/sub-tester_ses-1_acq-HASC55PA_space-T1w_dwiref.nii.gz
 sub-tester/ses-2
 sub-tester/ses-2/dwi
@@ -62,4 +63,5 @@ sub-tester/ses-2/dwi/sub-tester_ses-2_acq-HASC55AP_space-T1w_desc-preproc_dwi.b
 sub-tester/ses-2/dwi/sub-tester_ses-2_acq-HASC55AP_space-T1w_desc-preproc_dwi.bval
 sub-tester/ses-2/dwi/sub-tester_ses-2_acq-HASC55AP_space-T1w_desc-preproc_dwi.bvec
 sub-tester/ses-2/dwi/sub-tester_ses-2_acq-HASC55AP_space-T1w_desc-preproc_dwi.nii.gz
+sub-tester/ses-2/dwi/sub-tester_ses-2_acq-HASC55AP_space-T1w_desc-preproc_dwi.json
 sub-tester/ses-2/dwi/sub-tester_ses-2_acq-HASC55AP_space-T1w_dwiref.nii.gz

qsiprep/tests/data/maternal_brain_project_outputs.txt

@@ -30,5 +30,6 @@ sub-01/ses-01/dwi/sub-01_ses-01_space-T1w_desc-preproc_dwi.b
 sub-01/ses-01/dwi/sub-01_ses-01_space-T1w_desc-preproc_dwi.bval
 sub-01/ses-01/dwi/sub-01_ses-01_space-T1w_desc-preproc_dwi.bvec
 sub-01/ses-01/dwi/sub-01_ses-01_space-T1w_desc-preproc_dwi.nii.gz
+sub-01/ses-01/dwi/sub-01_ses-01_space-T1w_desc-preproc_dwi.json
 sub-01/ses-01/dwi/sub-01_ses-01_space-T1w_desc-preproc_dwi.b_table.txt
 sub-01/ses-01/dwi/sub-01_ses-01_space-T1w_dwiref.nii.gz

qsiprep/tests/tests.py

@@ -72,4 +72,4 @@ def mock_config():
     shutil.rmtree(config.execution.output_dir)
 
     if not _old_fs:
-        del os.environ["FREESURFER_HOME"]
+        del os.environ["FREESURFER_HOME"]

qsiprep/utils/bids.py

@@ -45,6 +45,8 @@
 from bids import BIDSLayout
 from bids.layout import Query
 
+from .. import config
+
 IMPORTANT_DWI_FIELDS = [
     # From image headers:
     "Obliquity",
@@ -401,3 +403,22 @@ def update_metadata_from_nifti_header(metadata, nifti_file):
         metadata["NumVolumes"] = 1.0
     orient = nb.orientations.aff2axcodes(img.affine)
     metadata["ImageOrientation"] = "".join(orient) + "+"
+
+
+def scan_groups_to_sidecar(scan_groups):
+    """Create a sidecar that reflects how the preprocessed image was created."""
+
+    # Add the information about how the images were grouped and which fieldmaps were used
+    derivatives_metadata = {"scan_grouping": scan_groups}
+
+    # Get metadata from the individual scans that were combined to make this preprocessed image
+    concatenated_dwi_files = scan_groups.get("dwi_series")
+    fieldmap_info = scan_groups.get("fieldmap_info")
+    if fieldmap_info.get("suffix") == "rpe_series":
+        concatenated_dwi_files.extend(fieldmap_info.get("rpe_series", []))
+    scan_metadata = {}
+    for dwi_file in concatenated_dwi_files:
+        dwi_file_name = Path(dwi_file).name
+        scan_metadata[dwi_file_name] = config.execution.layout.get_metadata(dwi_file)
+    derivatives_metadata["source_metadata"] = scan_metadata
+    return derivatives_metadata

qsiprep/workflows/dwi/finalize.py

@@ -15,12 +15,14 @@
 from ... import config
 from ...engine import Workflow
 from ...interfaces import DerivativesDataSink
+from ...interfaces.bids import DerivativesSidecar
 from ...interfaces.dsi_studio import DSIStudioBTable
 from ...interfaces.dwi_merge import MergeFinalConfounds, SplitResampledDWIs
 from ...interfaces.gradients import ExtractB0s
 from ...interfaces.mrtrix import DWIBiasCorrect, MRTrixGradientTable
 from ...interfaces.nilearn import Merge
 from ...interfaces.reports import GradientPlot, SeriesQC
+from ...utils.bids import scan_groups_to_sidecar
 from .derivatives import init_dwi_derivatives_wf
 from .qc import init_mask_overlap_wf, init_modelfree_qc_wf
 from .resampling import init_dwi_trans_wf
@@ -351,6 +353,26 @@ def init_dwi_finalize_wf(
         mem_gb=DEFAULT_MEMORY_MIN_GB,
     )
 
+    # Write a metadata sidecar for the derivatives
+    merged_sidecar = pe.Node(
+        DerivativesSidecar(
+            sidecar_data=scan_groups_to_sidecar(scan_groups), source_file=source_file
+        ),
+        name="merged_sidecar",
+    )
+    ds_merged_sidecar = pe.Node(
+        DerivativesDataSink(
+            extension=".json",
+            source_file=source_file,
+            space="T1w",
+            desc="preproc",
+            base_directory=config.execution.output_dir,
+        ),
+        name="ds_merged_sidecar",
+        run_without_submitting=True,
+        mem_gb=DEFAULT_MEMORY_MIN_GB,
+    )
+
     # Write the carpetplot data (which is the text output from eddy)
     ds_carpetplot_data = pe.Node(
         DerivativesDataSink(
@@ -392,6 +414,7 @@ def init_dwi_finalize_wf(
             ('t1_mask', 'inputnode.anatomical_mask')]),
         (gtab_t1, outputnode, [('gradient_file', 'gradient_table_t1')]),
         (btab_t1, outputnode, [('btable_file', 'btable_t1')]),
+        (merged_sidecar, ds_merged_sidecar, [('derivatives_json', 'in_file')]),
         (outputnode, dwi_derivatives_wf,
          [('dwi_t1', 'inputnode.dwi_t1'),
           ('dwi_mask_t1', 'inputnode.dwi_mask_t1'),