From c7b2dd4f8978ae48cbcd713bea63a9675c1b4ec3 Mon Sep 17 00:00:00 2001
From: Tashrif Billah <35086881+tashrifbillah@users.noreply.github.com>
Date: Wed, 11 Oct 2023 16:12:25 -0400
Subject: [PATCH 1/7] add snapshot of wma800 directory

---
 docs/TUTORIAL.md | 23 +++++++++++++++++++++++
 1 file changed, 23 insertions(+)

diff --git a/docs/TUTORIAL.md b/docs/TUTORIAL.md
index c03036a..9cd54ef 100644
--- a/docs/TUTORIAL.md
+++ b/docs/TUTORIAL.md
@@ -540,6 +540,29 @@ for performing whole-brain tractography parcellation. The outputs of that script
 * a parcellation of the entire white matter into 800 fiber clusters
 * a parcellation of anatomical fiber tracts organized according to the brain lobes they connect
 
+Here is an abridged snapshot of the output directory (you should examine the `*csv` files):
+
+```python
+wma800/
+└── sub-1122_ses-1_dir-416
+    ├── AnatomicalTracts
+    │   └── diffusion_measurements_anatomical_tracts.csv
+    ├── FiberClustering
+    │   ├── InitialClusters
+    │   ├── OutlierRemovedClusters
+    │   ├── SeparatedClusters
+    │   │   ├── diffusion_measurements_commissural.csv
+    │   │   ├── diffusion_measurements_left_hemisphere.csv
+    │   │   ├── diffusion_measurements_right_hemisphere.csv
+    │   │   ├── tracts_commissural
+    │   │   ├── tracts_left_hemisphere
+    │   │   └── tracts_right_hemisphere
+    │   └── TransformedClusters
+    └── TractRegistration
+        └── sub-1122_ses-1_dir-416
+            └── output_tractography
+```
+
 
 Configuration:
 

From f731e83e51661564d1067aa647e99d749ea5b295 Mon Sep 17 00:00:00 2001
From: Tashrif Billah <35086881+tashrifbillah@users.noreply.github.com>
Date: Wed, 11 Oct 2023 16:29:38 -0400
Subject: [PATCH 2/7] instruction for running wma800 on various servers

---
 docs/TUTORIAL.md | 36 ++++++++++++++++++++++++++++++++++++
 1 file changed, 36 insertions(+)

diff --git a/docs/TUTORIAL.md b/docs/TUTORIAL.md
index 9cd54ef..5835caf 100644
--- a/docs/TUTORIAL.md
+++ b/docs/TUTORIAL.md
@@ -583,6 +583,8 @@ Consult `wm_apply_ORG_atlas_to_subject.sh --help` to know the details of these p
 
 Run it:
 
+##### Through Luigi pipeline
+
 ```bash
 export LUIGI_CONFIG_PATH=/path/to/dwi_pipe_params.cfg
 
@@ -590,6 +592,40 @@ exec/ExecuteTask --task Wma800 \
 --bids-data-dir /data/pnl/DIAGNOSE_CTE_U01/rawdata -c 1001 --dwi-template sub-*/ses-01/dwi/*_dwi.nii.gz
 ```
 
+##### On a PNL workstation physically or through NoMachine
+
+```
+NEW_SOFT_DIR=/rfanfs/pnl-zorro/software/pnlpipe3/
+source ${NEW_SOFT_DIR}/bashrc3
+
+wm_apply_ORG_atlas_to_subject.sh \
+-i /path/to/your.vtk \
+-o /path/to/output/dir/ \
+-a ${NEW_SOFT_DIR}/ORG-Atlases-1.2 \
+-s ${NEW_SOFT_DIR}/Slicer-4.10.2-linux-amd64/SlicerWithExtensions.sh \
+-m "${NEW_SOFT_DIR}/Slicer-4.10.2-linux-amd64/SlicerWithExtensions.sh --launch FiberTractMeasurements" \
+-n 8 -c 2 -d 1
+```
+
+The same command applies for running it on PNL's GRX node: `dna007.partners.org` except a change in the software directory:
+
+> NEW_SOFT_DIR=/data/pnl/soft/pnlpipe3/
+
+
+##### Through SSH
+
+Just append the `-x 1` flag to the above command to enable virtual X server:
+
+```
+wm_apply_ORG_atlas_to_subject.sh \
+...
+-n 8 -c 2 -d 1 -x 1
+```
+
+However, running it through SSH requires availability of `xvfb-run` on that server--may it be a PNL workstation or a GRX node.
+The good news is that we have already installed `xorg-x11-server-Xvfb` in most servers. So you should be all set. 
+But if you find `xvfb-run`'s absence, reach out to PNL engineers for help.
+
 
 
 # Fs2Dwi pipeline

From b311000073047b40db9b7b8c1752cdbba883aa6c Mon Sep 17 00:00:00 2001
From: Tashrif Billah <35086881+tashrifbillah@users.noreply.github.com>
Date: Wed, 11 Oct 2023 16:38:05 -0400
Subject: [PATCH 3/7] add note about running wma800 through bsub

---
 docs/TUTORIAL.md | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/docs/TUTORIAL.md b/docs/TUTORIAL.md
index 5835caf..34bd5b9 100644
--- a/docs/TUTORIAL.md
+++ b/docs/TUTORIAL.md
@@ -611,6 +611,8 @@ The same command applies for running it on PNL's GRX node: `dna007.partners.org`
 
 > NEW_SOFT_DIR=/data/pnl/soft/pnlpipe3/
 
+You should consult [SlicerDMRI/whitematteranalysis](https://github.com/SlicerDMRI/whitematteranalysis) repository
+for meaning of the above arguments.
 
 ##### Through SSH
 
@@ -625,6 +627,7 @@ wm_apply_ORG_atlas_to_subject.sh \
 However, running it through SSH requires availability of `xvfb-run` on that server--may it be a PNL workstation or a GRX node.
 The good news is that we have already installed `xorg-x11-server-Xvfb` in most servers. So you should be all set. 
 But if you find `xvfb-run`'s absence, reach out to PNL engineers for help.
+Lastly, you can use the same `-x 1` flag to run it through `bsub` on `pri_pnl` queue in ERIS cluster.
 
 
 

From 7048e09e7c73e1e35f29ff7d69f5bfa2b4cf9369 Mon Sep 17 00:00:00 2001
From: "Billah, Tashrif" <35086881+tashrifbillah@users.noreply.github.com>
Date: Fri, 27 Oct 2023 15:03:24 -0400
Subject: [PATCH 4/7] replace big-multi by pri_pnl

---
 workflows/run_luigi_pnlpipe.lsf | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/workflows/run_luigi_pnlpipe.lsf b/workflows/run_luigi_pnlpipe.lsf
index 7e23c5e..b7d6fd3 100755
--- a/workflows/run_luigi_pnlpipe.lsf
+++ b/workflows/run_luigi_pnlpipe.lsf
@@ -18,8 +18,8 @@ sessions=/path/to/sessions.txt
 #BSUB -J luigi-pnlpipe[1-NUM_CASES]%NUM_PARALLEL
 #BSUB -o ~/luigi-pnlpipe-%J-%I.out
 #BSUB -e ~/luigi-pnlpipe-%J-%I.err
-#BSUB -q big-multi
-#BSUB -R rusage[mem=16000]
+#BSUB -q pri_pnl
+#BSUB -R rusage[mem=8000]
 #BSUB -n NUM_CORES
 
 

From 20792aa182f6bf2cdd546dd8b47e5340025c384a Mon Sep 17 00:00:00 2001
From: Tashrif Billah <35086881+tashrifbillah@users.noreply.github.com>
Date: Tue, 6 Feb 2024 16:44:12 -0500
Subject: [PATCH 5/7] typo

---
 docs/TUTORIAL.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/TUTORIAL.md b/docs/TUTORIAL.md
index 47bfe51..fb884e0 100644
--- a/docs/TUTORIAL.md
+++ b/docs/TUTORIAL.md
@@ -501,7 +501,7 @@ exec/ExecuteTask --task SynB0 \
 --t1-template sub-*/ses-*/anat/*_T1w.nii.gz
 ```
 
-Note that, `SynB0` task is run in coordination with [_synb0_eddy.sh](_synb0_eddy.sh) .
+Note that, `SynB0` task is run in coordination with [_synb0_eddy.sh](../workflows/_synb0_eddy.sh) .
 
 
 ### TopupEddy

From be770fbe42358a387ecc6c7e6d9f78433d78cc1e Mon Sep 17 00:00:00 2001
From: "Billah, Tashrif" <35086881+tashrifbillah@users.noreply.github.com>
Date: Wed, 3 Apr 2024 15:43:56 -0400
Subject: [PATCH 6/7] =?UTF-8?q?use=20-e=20flag=20to=20prevent=20supply=20o?=
 =?UTF-8?q?f=20LC=5FALL,LANG=20values=20Fixes=20these=20warnings:=20perl:?=
 =?UTF-8?q?=20warning:=20Setting=20locale=20failed.=20perl:=20warning:=20P?=
 =?UTF-8?q?lease=20check=20that=20your=20locale=20settings:=20=E2=80=83?=
 =?UTF-8?q?=E2=80=83LANGUAGE=20=3D=20(unset),=20=E2=80=83=E2=80=83LC=5FALL?=
 =?UTF-8?q?=20=3D=20(unset),=20=E2=80=83=E2=80=83LANG=20=3D=20"en=5FUS.UTF?=
 =?UTF-8?q?-8"=20=20=20=20=20are=20supported=20and=20installed=20on=20your?=
 =?UTF-8?q?=20system.=20perl:=20warning:=20Falling=20back=20to=20the=20sta?=
 =?UTF-8?q?ndard=20locale=20("C").=20Processing:..........................?=
 =?UTF-8?q?....................Done?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 workflows/_synb0_eddy.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/workflows/_synb0_eddy.sh b/workflows/_synb0_eddy.sh
index bac2046..edd4e93 100755
--- a/workflows/_synb0_eddy.sh
+++ b/workflows/_synb0_eddy.sh
@@ -31,7 +31,7 @@ mkdir -p $TMPDIR
 if [ ! -f OUTPUTS/b0_all_topup.nii.gz ]
 then
     TMPDIR=$TMPDIR \
-    singularity run -B INPUTS/:/INPUTS -B OUTPUTS/:/OUTPUTS \
+    singularity run -e -B INPUTS/:/INPUTS -B OUTPUTS/:/OUTPUTS \
     -B ${NEW_SOFT_DIR}/fs7.1.0/license.txt:/extra/freesurfer/license.txt \
     ${NEW_SOFT_DIR}/containers/synb0-disco_v3.0.sif --stripped
 fi

From 6098f9e66e94b47ba75dfbc81345f2cc32abc713 Mon Sep 17 00:00:00 2001
From: Tashrif Billah <35086881+tashrifbillah@users.noreply.github.com>
Date: Sat, 6 Apr 2024 16:20:53 -0400
Subject: [PATCH 7/7] remove obsolete script workflows/synb0_eddy.sh

---
 workflows/synb0_eddy.sh | 155 ----------------------------------------
 1 file changed, 155 deletions(-)
 delete mode 100755 workflows/synb0_eddy.sh

diff --git a/workflows/synb0_eddy.sh b/workflows/synb0_eddy.sh
deleted file mode 100755
index a4bf611..0000000
--- a/workflows/synb0_eddy.sh
+++ /dev/null
@@ -1,155 +0,0 @@
-#!/bin/bash
-
-# for time profiling
-date
-START_TIME=$(date +%s)
-
-# User will edit only this block =========================================================
-caselist=$1
-s=01
-dir=80
-acq=AP
-BIDS_DATA_DIR=/data/pnl/Collaborators/CMA/mtsintou/Emotion/rawdata
-DWI_TEMPLATE=sub-*/ses-*/dwi/sub-*_ses-*_acq-${acq}_dir-${dir}_dwi.nii.gz
-INDEX=/data/pnl/Collaborators/CMA/mtsintou/Emotion/derivatives/index.txt
-ACQPARAMS=/data/pnl/Collaborators/CMA/mtsintou/Emotion/derivatives/acqparams.txt
-LUIGI_CONFIG_PATH=/data/pnl/soft/pnlpipe3/luigi-pnlpipe/params/cte/cnn_dwi_mask_params.cfg
-# ========================================================================================
-
-
-source /rfanfs/pnl-zorro/software/pnlpipe3/bashrc3-gpu
-NEW_SOFT_DIR=/rfanfs/pnl-zorro/software/pnlpipe3/
-
-
-# for a caselist, this script must be run in a for loop
-# https://github.com/pnlbwh/luigi-pnlpipe/wiki/Run-HCP-pipeline-on-PNL-GPU-machines-in-a-parallel-manner
-if [ -f $caselist ]
-then
-    c=`head -${LSB_JOBINDEX} $caselist | tail -1`
-
-    NUM_GPUS=`nvidia-smi -L | wc -l`
-    export CUDA_VISIBLE_DEVICES=$(( ${LSB_JOBINDEX}%${NUM_GPUS} ))
-else
-    c=$caselist
-fi
-
-
-echo "1. run Luigi pipeline and prepare DWI for synb0 container"
-export LUIGI_CONFIG_PATH
-/rfanfs/pnl-zorro/software/pnlpipe3/luigi-pnlpipe/exec/ExecuteTask --task CnnMask \
---bids-data-dir $BIDS_DATA_DIR \
---dwi-template "$DWI_TEMPLATE" \
--c ${c} -s ${s}
-# double quotes around $DWI_TEMPLATE are mandatory
-
-
-
-DERIVATIVES=$(dirname $BIDS_DATA_DIR)/derivatives/pnlpipe/
-SES_FOLDER=$DERIVATIVES/sub-${c}/ses-${s}
-pushd .
-cd $SES_FOLDER
-mkdir -p INPUTS OUTPUTS
-
-
-if [ ! -z `ls $SES_FOLDER/dwi/*_desc-XcUnEdEp_dwi.nii.gz` ]
-then
-    echo $c was processed before
-    exit
-fi
-
-
-echo "2. prepare b0 and T1 for synb0 container"
-_unring_prefix=`ls dwi/sub-${c}_ses-${s}_*desc-XcUn_dwi.nii.gz`
-unring_prefix=${_unring_prefix//.nii.gz}
-unring_mask=`ls dwi/sub-${c}_ses-${s}_*desc-dwiXcUnCNN_mask.nii.gz`
-if [ ! -f INPUTS/b0.nii.gz ]
-then
-    fslmaths ${unring_prefix}.nii.gz -mul $unring_mask ${unring_prefix}.nii.gz && \
-    bse.py -i ${unring_prefix}.nii.gz -o INPUTS/b0.nii.gz
-fi
-
-T1=anat/sub-${c}_ses-${s}_desc-XcMaN4_T1w.nii.gz
-if [ -f $T1 ]
-then
-    cp $T1 INPUTS/T1.nii.gz
-else
-    echo "Run structural pipeline first:"
-    echo "https://github.com/pnlbwh/luigi-pnlpipe/blob/hcp/docs/Process_DIAGNOSE-CTE_data.md#structural-pipeline"
-fi
-
-cp $ACQPARAMS INPUTS/
-cp $INDEX INPUTS/
-
-
-echo "3. run synb0 container"
-TMPDIR=$HOME/tmp/
-mkdir -p $TMPDIR
-if [ ! -f OUTPUTS/b0_all_topup.nii.gz ]
-then
-    TMPDIR=$TMPDIR \
-    singularity run -B INPUTS/:/INPUTS -B OUTPUTS/:/OUTPUTS \
-    -B ${NEW_SOFT_DIR}/fs7.1.0/license.txt:/extra/freesurfer/license.txt \
-    ${NEW_SOFT_DIR}/containers/synb0-disco_v3.0.sif --stripped
-fi
-
-echo "4. create mask of topup (synb0) corrected b0"
-# CNN method
-_caselist=$(mktemp --suffix=.txt)
-realpath OUTPUTS/b0_all_topup.nii.gz > $_caselist
-echo "0 0" > OUTPUTS/b0_all_topup.bval
-dwi_masking.py -i $_caselist -f ${NEW_SOFT_DIR}/CNN-Diffusion-MRIBrain-Segmentation/model_folder
-mask=`ls OUTPUTS/*-multi_BrainMask.nii.gz`
-rm $_caselist
-
-if [ -z $mask ]
-then
-    echo topup mask creation failed
-    exit 1
-fi
-
-
-eddy_out=OUTPUTS/sub-${c}_ses-${s}_dir-${dir}_desc-XcUnEdEp_dwi
-# initial guess was masking the --imain would improve quality of eddy corrected DWI
-# however, the b0_all_topup_mask is underinclusive
-# so it only crops off the frontal distortion
-# so omit masking at this step
-# fslmaths ${unring_prefix}.nii.gz -mul $mask ${unring_prefix}.nii.gz
-echo "5. run eddy_cuda"
-eddy_cuda \
-  --imain=${unring_prefix}.nii.gz \
-  --bvecs=${unring_prefix}.bvec \
-  --bvals=${unring_prefix}.bval \
-  --mask=$mask \
-  --topup=OUTPUTS/topup \
-  --acqp=INPUTS/acqparams.txt \
-  --index=INPUTS/index.txt \
-  --repol --data_is_shelled --verbose \
-  --out=$eddy_out || { exit 1; }
-
-
-
-echo "6. organize outputs"
-# provide masked eddy_out for clarity, quality, and convenience
-fslmaths ${eddy_out}.nii.gz -mul $mask ${eddy_out}.nii.gz
-
-bids_prefix=dwi/sub-${c}_ses-${s}_dir-${dir}_desc-XcUnEdEp_dwi
-mv ${eddy_out}.nii.gz ${bids_prefix}.nii.gz
-mv ${eddy_out}.eddy_rotated_bvecs ${bids_prefix}.bvec
-cp ${unring_prefix}.bval ${bids_prefix}.bval
-
-mask_prefix=${bids_prefix//_dwi/}
-mask_prefix=${mask_prefix//XcUn/dwiXcUn}
-mv $mask ${mask_prefix}_mask.nii.gz
-mv OUTPUTS/b0_all_topup_bse.nii.gz ${mask_prefix}_bse.nii.gz
-
-
-
-echo "Luigi-SynB0-Eddy pipeline has completed"
-echo "See outputs at $PWD/dwi/"
-
-popd
-
-# for time profiling
-date
-ELAPSED_TIME=$(($(date +%s) - $START_TIME))
-echo "Elapsed time: $(($ELAPSED_TIME/60)) min $(($ELAPSED_TIME%60)) sec"