-
Notifications
You must be signed in to change notification settings - Fork 3
/
DiCER_lightweight.sh
executable file
·263 lines (210 loc) · 11.1 KB
/
DiCER_lightweight.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
#!/bin/bash
# This here is a lightweight version of DiCER, this does not assume you have run fmriprep, it just takes your own preprocessing,
# and then applies DiCER to the result.
# This does require some inputs though:
# These inputs are needed all to make the report and to perform DiCER
print_usage() {
printf "DiCER_lightweight\n
This tool performs (Di)ffuse (C)luster (E)stimation and (R)egression on data without fmriprep preprocessing. Here, we take fmri data that has NOT been detrended or demeaned (important!) and either a tissue tissue
classification which is a file that has the same dimensions as the functional image with the following labels: 1=CSF,2=GM,3=WM,4=Restricted GM this restricted GM just takes the GM mask and takes the top 70 percent
of signals (i.e. top 70 relative to the mean) to estimate noisy signals.\n
Usage with tissue map: DiCER_lightweight.sh -i input_nifti -t tissue_file -w output_folder -s subjectID -c confounds.tsv\n\n
Usage without tissue map: DiCER_lightweight.sh -i input_nifti -a T1w -w output_folder -s subjectID -c confounds.tsv\n\n
Optional (and recommonded) flag is -d, this detrends and high-pass filters the data. This allows better estimation of regressors, and is a very light cleaning of your data.\n
Optional (may be needed for high-res data) flag -p ds_factor (e.g. -p 3), this enforces a sparse-sampling of the tissue mask based on the global signal correlation, this needed for high res data to be tractable. \n
\n
\n
SURFACE fMRI DATA\n
If you happen to have surface-registered data with freesurfer (or equivalent) this can work by the following\n
DiCER_lightweight.sh -f -i input_nifti -w output_folder -s subjectID -p 6\n\n
Note, the surface file here will be in a nifti format (you can change this with tools like mri_convert etc) in the standard \n
type of of input that freesurfer uses for FS-FAST. Also downsampling is needed, -p 6 is recommened for data on things such as fsaverage \n
Kevin Aquino. 2019, email: [email protected]\n\n\nDiCER website: https://github.com/BMHLab/DiCER \n\n"
}
tissue=""
detrend="false"
makeTissueMap="false"
use_confounds="false"
ds_factor="0"
freesurfer="false"
mov_exists="false"
# Everything will be in g-zipped niftis
FSLOUTPUTTYPE=NIFTI_GZ
# Have to make options if you need to generate a tissue file do so i.e. only if you specifiy the tissue file
while getopts 'i:t:a:w:s:c:m:dp:fh' flag; do
case "${flag}" in
i) input_file="${OPTARG}" ;;
t) tissue="${OPTARG}" ;;
a) anatomical="${OPTARG}"
makeTissueMap="true";;
w) output_folder="${OPTARG}" ;;
s) subject="${OPTARG}" ;;
c) confounds="${OPTARG}"
use_confounds="true";;
m) movement_params="${OPTARG}"
mov_exists="true";;
d) detrend="true" ;;
p) ds_factor="${OPTARG}";;
f) freesurfer="true" ;;
h) print_usage
exit 1;;
*) print_usage
exit 1 ;;
esac
done
# Make a temporary folder if it doesnt exist for all the segmentation and errata outputs
if [ ! -d "$output_folder/tmp_dir/" ]; then
mkdir -p $output_folder/tmp_dir/
fi
# Setting up extra variables once you have everything!
folder=$output_folder #this is the working directory.
input=$output_folder"/"$input_file
confounds=$output_folder$confounds
# Surface niftis!
if $freesurfer;then
# Change the dimensions of the nifti to make it work with the rest of the script, currently doesn't handle giftis or ciftis directly, making this shortcut to have the code work all the way through
outFile=$output_folder"/func_temp.nii.gz"
python hcp_processing/reshapeSurfaceNifti.py -f $input -o $outFile
input=$outFile
orig=$input_file
input_file="func_temp.nii.gz"
# Here just making a tissue file based just on the surface time series, i.e. setting it that all time series on the vertices are used in the estimation of regressors.
fslmaths $input -Tmean -abs -bin -mul 4 $output_folder"tissue.nii.gz"
tissue="tissue.nii.gz"
# Here just making sure that the tissue map isn't made.
makeTissueMap="false"
# will probably have to re-arrange the nifti -- then at the end of the whole shebang will have to re-sort.
fi
tissue_mask=$output_folder"/"$tissue
# Make the tissue map if you have specificed the tissue map!
#
# TISSUE SEGMENTATION!
#
if $makeTissueMap;then
printf "\n\nPeforming FAST tissue segmentation with anatomical image $anatomical\n"
# Perform FAST segmentation:
fast -o $output_folder"/tmp_dir/"$subject $output_folder"/"$anatomical
# Get the segmentation file:
segmentation_file=$output_folder"/tmp_dir/"$subject"_seg"
# Now apply the flirt command to get the segmentation into the func space:
seg_temp=$output_folder"/tmp_dir/"$subject"_seg_temp"
flirt -in $segmentation_file -ref $input -out $seg_temp -applyxfm -interp nearestneighbour -usesqform
# Now using the standard convention in FAST we generate the tissue types
# GM - be a little more conservative above what FAST gives out in its hard segmentation:
flirt -in $output_folder"/tmp_dir/"$subject"_pve_1.nii.gz" -out $output_folder"/tmp_dir/"$subject"_ds_gm.nii.gz" -ref $input -applyxfm -usesqform
gm_mask_tmp=$output_folder"/tmp_dir/"$subject"_gm_mask"
fslmaths $output_folder"/tmp_dir/"$subject"_ds_gm.nii.gz" -thr 0.5 -bin $gm_mask_tmp
# Use the probability mask and threshold that one
# fslmaths $seg_temp -thr 2 -uthr 2 -bin $gm_mask_tmp
# Generate masks for GM to make it more restrictive:
mean_ts=$output_folder"/tmp_dir/"$subject"_mean_ts"
fslmaths $input -Tmean $mean_ts
# Taking the mean ts image, and just focusing in on grey matter
fslmaths $mean_ts -mul $gm_mask_tmp $mean_ts
# Now find the min/max
read min max <<< $(fslstats $mean_ts -r)
# Normalize the image and threshold the map to make a mask of epi of the top 60% of image intensity
gm_mask_restrictive=$output_folder"/tmp_dir/"$subject"_mask_restrictive"
fslmaths $mean_ts -div $max -thr 0.3 -bin $gm_mask_restrictive
fslmaths $gm_mask_restrictive -mul $gm_mask_tmp -mul 2 $gm_mask_restrictive
# Now we have everything to work with and combine it all together now (fix up too many volumes)
tissue_mask=$output_folder"/"$subject"_dtissue_func.nii.gz"
fslmaths $seg_temp -add $gm_mask_restrictive $tissue_mask
fslroi $tissue_mask $tissue_mask 0 1
printf "\n\nSaved tissue mask in functional space and saved as: $tissue_mask\n"
fi
# Detrending and high-pass filtering data::
if $detrend;then
printf "\n\Detrending and high-pass filtering $input..\n\n\n"
base_input=`basename $input .nii.gz`
output_detrended=$output_folder"/"$base_input"_detrended_hpf.nii.gz"
# Find a mask epi
mask_epi=$output_folder"/tmp_dir/"$subject"_mask_epi.nii.gz"
fslmaths $tissue_mask -bin $mask_epi
sh fmriprepProcess/preprocess_fmriprep.sh $input $output_detrended $output_folder $mask_epi
# Now change all the inputs to work on the deterended versions
input=$output_detrended
input_file=$base_input"_detrended_hpf.nii.gz"
fi
if [ $ds_factor -gt 0 ];then
printf "\n Creating a sparse-sampling of the tissue mask based on the correlation to the global signal for each voxel.\n"
echo $input
echo $tissue_mask
echo $output_folder"/tmp_dir/"$subject"gsReorder.nii.gz"
echo ""
python fmriprepProcess/gsReorder.py -f $input -ts $tissue_mask -of $output_folder"/tmp_dir/"$subject"gsReorder.nii.gz"
base_tissue_mask=`basename $tissue_mask .nii.gz`
tissue_mask_ds=$output_folder"/"$base_tissue_mask"_dsFactor_"$ds_factor".nii.gz"
python utils/sparse_sample_tissue.py -o $output_folder"/tmp_dir/"$subject"gsReorder.nii.gz" -ds $ds_factor -ts $tissue_mask -tsd $tissue_mask_ds
tissue_mask=$tissue_mask_ds
fi
printf "\n\nPerfoming DiCER..\n\n\n"
python carpetCleaning/clusterCorrect.py $tissue_mask '.' $input $folder $subject
# Regress out all the regressors
regressor_dbscan=$subject"_dbscan_liberal_regressors.csv"
base_dicer_o=`basename $input .nii.gz`
dicer_output=$output_folder"/"$base_dicer_o"_dbscan.nii.gz"
printf "\n\nRegressing $input with DiCER signals and clean output is at $dicer_output \n\n\n"
python carpetCleaning/vacuum_dbscan.py -f $input_file -db $regressor_dbscan -s $subject -d $folder"/"
# Next stage: do the reporting, all done through "tapestry"
export MPLBACKEND="agg"
# Do the cluster re-ordering:
printf "\n\nPeforming Cluster re-ordering of $input \n\n\n"
python fmriprepProcess/clusterReorder.py $tissue_mask '.' $input $folder $subject
# if $freesurfer;then
# cluster_tissue_ordering=$output_folder"/tmp_dir/"$base_dicer_o"_clusterorder.nii.gz"
# else
cluster_tissue_ordering=$output_folder"/"$base_dicer_o"_clusterorder.nii.gz"
# fi
printf "\n\nPeforming GS re-ordering of $input (again use the mask) \n\n\n"
python fmriprepProcess/gsReorder.py -f $input -ts $tissue_mask -of $output_folder"/"$subject_"gsReorder.nii.gz"
gs_reordering_file=$output_folder"/"$subject_"gsReorder.nii.gz"
printf "\n\nPeforming GMR of $input \n\n\n"
gm_signal=$output_folder"/"$subject"_GMsignal.txt"
fslmeants -i $input -o $gm_signal
GMR_output=$output_folder"/"$base_dicer_o"_GMR.nii.gz"
fsl_regfilt -i $input -d $gm_signal -f 1 -o $GMR_output
# Run the automated report:
printf "\n\nRunning the carpet reports! This is to visualize the data in a way to evaluate the corrections \n\n\n"
if (! $use_confounds);then
# Here calculate DVARS and FD if the mov file has been created
DVARS_txt=$output_folder"/"$subject"_DVARS.txt"
# Calculate DVARS:
sh utils/calculate_dvars.sh $input $output_folder/tmp_dir/ $tissue_mask $DVARS_txt
echo $DVARS_txt
# Now generate the confounds file
bind -u complete # Here to ensure the tabs are not included in the pasting
declare -i nRow
nRow=$(cat $DVARS_txt | wc -l)
echo $nRow
zerofile=$output_folder/tmp_dir/zeros.txt
yes 0 | head -$nRow > $zerofile
printf "DVARS\n0\n$(cat $DVARS_txt)" > $DVARS_txt
# printf "0\n$(cat $DVARS_txt)" > $DVARS_txt
printf "col0\n$(cat $zerofile)" > $zerofile
if $mov_exists;then
# Here calculate Framewise displacement
FD_file=$output_folder"/"$subject"_FD_calc.txt"
python utils/calculate_FD.py -mov $output_folder"/"$movement_params -out $FD_file
# FD_file_tmp=$output_folder"/"$subject"_FD_calc_tmp.txt"
# echo "0
# $(cat $FD_file)" > $FD_file
printf "FD\n$(cat $FD_file)" > $FD_file
paste -d "\t" $zerofile $zerofile $zerofile $DVARS_txt $zerofile $zerofile $FD_file > $output_folder"/"$subject"confounds.tsv"
else
fakefdfile=$output_folder/tmp_dir/fake_fd.txt
yes 0 | head -$nRow > $fakefdfile
printf "FD\n0\n$(cat $fakefdfile)" > $fakefdfile
paste -d "\t" $zerofile $zerofile $zerofile $DVARS_txt $zerofile $zerofile $fakefdfile > $output_folder"/"$subject"confounds.tsv"
fi
confounds=$output_folder"/"$subject"confounds.tsv"
fi
# Generate the carpet report here:
python carpetReport/tapestry.py -f $input","$GMR_output","$dicer_output -fl "INPUT,GMR,DICER" -o $cluster_tissue_ordering,$gs_reordering_file -l "CLUST,GSO" -s $subject -d $output_folder"/" -ts $tissue_mask -reg $output_folder"/"$regressor_dbscan -cf $confounds
# Have to at the end work with vacuuming the original file
# Surface niftis!
if $freesurfer;then
printf "\n\n Now using the regression time series and regressing them from the original input \n\n\n"
input=$output_folder"/"$input_file
python carpetCleaning/vacuum_dbscan.py -f $orig -db $regressor_dbscan -s $subject -d $folder"/"
fi