From fd528d0223a2d2f0ad3107a46e7c9c47023c296c Mon Sep 17 00:00:00 2001
From: "Billah, Tashrif" <35086881+tashrifbillah@users.noreply.github.com>
Date: Wed, 11 Sep 2024 22:19:05 -0400
Subject: [PATCH 01/33] get FS license from ${FREESURFER_HOME}/license.txt

---
 workflows/_synb0_eddy.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/workflows/_synb0_eddy.sh b/workflows/_synb0_eddy.sh
index edd4e93..ef0fc05 100755
--- a/workflows/_synb0_eddy.sh
+++ b/workflows/_synb0_eddy.sh
@@ -32,7 +32,7 @@ if [ ! -f OUTPUTS/b0_all_topup.nii.gz ]
 then
     TMPDIR=$TMPDIR \
     singularity run -e -B INPUTS/:/INPUTS -B OUTPUTS/:/OUTPUTS \
-    -B ${NEW_SOFT_DIR}/fs7.1.0/license.txt:/extra/freesurfer/license.txt \
+    -B ${FREESURFER_HOME}/license.txt:/extra/freesurfer/license.txt \
     ${NEW_SOFT_DIR}/containers/synb0-disco_v3.0.sif --stripped
 fi
 

From 5e429439f2319cc99e0d137f4e65c29409e4b7b0 Mon Sep 17 00:00:00 2001
From: "Billah, Tashrif" <35086881+tashrifbillah@users.noreply.github.com>
Date: Thu, 12 Sep 2024 11:25:50 -0400
Subject: [PATCH 02/33] [SynB0] section was missing

---
 params/dwi_pipe_params.cfg | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/params/dwi_pipe_params.cfg b/params/dwi_pipe_params.cfg
index 5086980..b7730aa 100644
--- a/params/dwi_pipe_params.cfg
+++ b/params/dwi_pipe_params.cfg
@@ -80,6 +80,8 @@ wma_cleanup: 0
 
 [CnnMask]
 
+[SynB0]
+
 [PnlEddy]
 
 [FslEddy]

From 9340af84b6627dbb3c18bf38e9c67d9cbd6fdf97 Mon Sep 17 00:00:00 2001
From: "Billah, Tashrif" <35086881+tashrifbillah@users.noreply.github.com>
Date: Thu, 12 Sep 2024 11:27:16 -0400
Subject: [PATCH 03/33] use 8 threads for gibbs unringing

---
 params/hcp/dwi_pipe_params.cfg | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/params/hcp/dwi_pipe_params.cfg b/params/hcp/dwi_pipe_params.cfg
index 075813e..5a4f3e0 100644
--- a/params/hcp/dwi_pipe_params.cfg
+++ b/params/hcp/dwi_pipe_params.cfg
@@ -34,7 +34,7 @@ HcpOutDir: hcppipe
 
 
 ## [GibbsUn] ##
-unring_nproc: 4
+unring_nproc: 8
 
 
 ## [Ukf] ##

From 79633eb8b7c75791bc44d22ae3a61726f51875f9 Mon Sep 17 00:00:00 2001
From: "Billah, Tashrif" <35086881+tashrifbillah@users.noreply.github.com>
Date: Thu, 12 Sep 2024 11:42:59 -0400
Subject: [PATCH 04/33] add SynB0 configs to repository

---
 params/synb0/acqparams.txt       |  2 +
 params/synb0/dwi_pipe_params.cfg | 73 ++++++++++++++++++++++++++++++++
 2 files changed, 75 insertions(+)
 create mode 100644 params/synb0/acqparams.txt
 create mode 100644 params/synb0/dwi_pipe_params.cfg

diff --git a/params/synb0/acqparams.txt b/params/synb0/acqparams.txt
new file mode 100644
index 0000000..714c52a
--- /dev/null
+++ b/params/synb0/acqparams.txt
@@ -0,0 +1,2 @@
+0 1 0 0.05
+0 1 0 0
diff --git a/params/synb0/dwi_pipe_params.cfg b/params/synb0/dwi_pipe_params.cfg
new file mode 100644
index 0000000..0450657
--- /dev/null
+++ b/params/synb0/dwi_pipe_params.cfg
@@ -0,0 +1,73 @@
+[DEFAULT]
+
+## [StructMask] ##
+mask_method: hd-bet
+mask_qc: False
+
+
+## [StructMask] [PnlEddy] [EddyEpi] ##
+debug: False
+
+
+## [CnnMask] ##
+model_folder: /software/rocky9/CNN-Diffusion-MRIBrain-Segmentation/model_folder
+percentile: 97
+filter:
+
+
+## [FslEddy] [TopupEddy] ##
+acqp: /software/rocky9/luigi-tutorial/edcrp/acqparams.txt
+index: /software/rocky9/luigi-tutorial/edcrp/index.txt
+config: /software/rocky9/pnlNipype/scripts/eddy_config.txt
+useGpu: False
+FslOutDir: fsl_eddy
+
+
+## [EddyEpi] ##
+epi_nproc: 8
+eddy_task: FslEddy
+
+
+## [HcpPipe] ##
+HcpOutDir: hcppipe
+
+
+## [GibbsUn] ##
+unring_nproc: 4
+
+
+## [Ukf] ##
+ukf_params: --numThreads,8
+eddy_epi_task: HcpPipe
+bhigh: 2000
+
+
+## [WMA800] ##
+slicer_exec: /data/pnl/soft/pnlpipe3/Slicer-4.10.2-linux-amd64/SlicerWithExtensions.sh
+FiberTractMeasurements: /data/pnl/soft/pnlpipe3/Slicer-4.10.2-linux-amd64/SlicerWithExtensions.sh --launch FiberTractMeasurements
+atlas: /data/pnl/soft/pnlpipe3/ORG-Atlases-1.2
+wma_nproc: 4
+xvfb: 1
+wma_cleanup: 
+
+[StructMask]
+
+[BseExtract]
+
+[CnnMask]
+
+[SynB0]
+
+[PnlEddy]
+
+[FslEddy]
+
+[TopupEddy]
+
+[EddyEpi]
+
+[HcpPipe]
+
+[Ukf]
+
+[Wma800]

From 0e67553bd6a6641fc2ab57f1fc7cf96ac0c01125 Mon Sep 17 00:00:00 2001
From: "Billah, Tashrif" <35086881+tashrifbillah@users.noreply.github.com>
Date: Thu, 12 Sep 2024 15:43:24 -0400
Subject: [PATCH 05/33] save fslversion in dwi/ directory

---
 workflows/dwi_pipe.py | 15 +++++++--------
 1 file changed, 7 insertions(+), 8 deletions(-)

diff --git a/workflows/dwi_pipe.py b/workflows/dwi_pipe.py
index 6b5ec25..2f4b8ff 100755
--- a/workflows/dwi_pipe.py
+++ b/workflows/dwi_pipe.py
@@ -292,8 +292,7 @@ def run(self):
                 p = Popen(cmd, shell=True)
                 p.wait()
                 
-                version_file= outDir.join('fsl_version.txt')
-                check_call(f'eddy_openmp 2>&1 | grep Part > {version_file}', shell= True)
+                check_call('cp $FSLDIR/etc/fslversion {}'.format(self.output()['dwi'].dirname), shell= True)
                 
                 # fsl_eddy.py writes with this outPrefix
                 outPrefix= outDir.join(self.input()[0]['dwi'].stem)+'_Ed'
@@ -333,6 +332,7 @@ class SynB0(Task):
     
     def run(self):
         
+
         # synb0 wrapper
         DIR= abspath(dirname(__file__))
         cmd = (' ').join([f'{DIR}/_synb0_eddy.sh',
@@ -347,9 +347,7 @@ def run(self):
         p = Popen(cmd, shell=True)
         p.wait()
         
-        version_file= self.output()['dwi'].dirname.join('fsl_version.txt')
-        check_call(f'eddy_openmp 2>&1 | grep Part > {version_file}', shell= True)
-
+        check_call('cp $FSLDIR/etc/fslversion {}'.format(self.output()['dwi'].dirname), shell= True)
 
         write_provenance(self, self.output()['dwi'])
 
@@ -519,8 +517,7 @@ def run(self):
                 p = Popen(cmd, shell=True)
                 p.wait()
 
-                version_file = outDir.join('fsl_version.txt')
-                check_call(f'eddy_openmp 2>&1 | grep Part > {version_file}', shell=True)
+                check_call('cp $FSLDIR/etc/fslversion {}'.format(self.output()['dwi'].dirname), shell=True)
 
 
                 with open(outDir.join('.outPrefix.txt')) as f:
@@ -653,7 +650,9 @@ def output(self):
             symlink(bvecHcp, bvec)
             symlink(maskHcp, mask)
             symlink(bseHcp, bse)
-        
+       
+        check_call('cp $FSLDIR/etc/fslversion {}'.format(self.output()['dwi'].dirname), shell=True)
+
         return dict(dwi=dwi, bval=bval, bvec=bvec, bse=bse, mask=mask)
 
 

From c5ca21d799df2d91dbfccb5971c8d617d6761ad6 Mon Sep 17 00:00:00 2001
From: "Billah, Tashrif" <35086881+tashrifbillah@users.noreply.github.com>
Date: Mon, 23 Sep 2024 14:21:40 -0400
Subject: [PATCH 06/33] change self.output()[dwi].dirname to just dwi.dirname

---
 workflows/dwi_pipe.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/workflows/dwi_pipe.py b/workflows/dwi_pipe.py
index 2f4b8ff..e591ac1 100755
--- a/workflows/dwi_pipe.py
+++ b/workflows/dwi_pipe.py
@@ -651,7 +651,7 @@ def output(self):
             symlink(maskHcp, mask)
             symlink(bseHcp, bse)
        
-        check_call('cp $FSLDIR/etc/fslversion {}'.format(self.output()['dwi'].dirname), shell=True)
+        check_call('cp $FSLDIR/etc/fslversion {}'.format(dwi.dirname), shell=True)
 
         return dict(dwi=dwi, bval=bval, bvec=bvec, bse=bse, mask=mask)
 

From 33a8ec5e0d5f9edef23633e8f0bf496d9c636f62 Mon Sep 17 00:00:00 2001
From: "Billah, Tashrif" <35086881+tashrifbillah@users.noreply.github.com>
Date: Mon, 23 Sep 2024 14:23:41 -0400
Subject: [PATCH 07/33] remove HPC cluster Luigi, add PNL private Luigi

---
 luigi.cfg | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/luigi.cfg b/luigi.cfg
index d90fa80..6c2cf6d 100644
--- a/luigi.cfg
+++ b/luigi.cfg
@@ -1,10 +1,11 @@
 [core]
 # for individual machine
 # default-scheduler-url = http://localhost:8082/
-# for High Performance Cluster (lsf script, bsub command)
-# default-scheduler-url = http://cmu166.research.partners.org:8082/
+# PNL hosted private server
+# default-scheduler-url = http://${LUIGI_USERNAME}:${LUIGI_PASSWORD}@pnl-elite-1.partners.org/
 # PNL hosted public server
 default-scheduler-url = https://${LUIGI_USERNAME}:${LUIGI_PASSWORD}@pnlservers.bwh.harvard.edu/luigi/
+
 log_level = INFO
 
 [scheduler]

From b21bc1ffd23be9b2241093e0e5a9f48e2f507876 Mon Sep 17 00:00:00 2001
From: "Billah, Tashrif" <35086881+tashrifbillah@users.noreply.github.com>
Date: Wed, 2 Oct 2024 17:15:07 -0400
Subject: [PATCH 08/33] update hcp dwi_pipe_params.cfg to agree with Redhat 9

---
 params/hcp/dwi_pipe_params.cfg | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/params/hcp/dwi_pipe_params.cfg b/params/hcp/dwi_pipe_params.cfg
index 5a4f3e0..b8e1caf 100644
--- a/params/hcp/dwi_pipe_params.cfg
+++ b/params/hcp/dwi_pipe_params.cfg
@@ -38,18 +38,18 @@ unring_nproc: 8
 
 
 ## [Ukf] ##
-ukf_params:
+ukf_params: --seedsPerVoxel,1
 eddy_epi_task: HcpPipe
 bhigh: 2000
 
 
 ## [WMA800] ##
-slicer_exec: /data/pnl/soft/pnlpipe3/Slicer-4.10.2-linux-amd64/SlicerWithExtensions.sh
-FiberTractMeasurements: /data/pnl/soft/pnlpipe3/Slicer-4.10.2-linux-amd64/SlicerWithExtensions.sh --launch FiberTractMeasurements
-atlas: /data/pnl/soft/pnlpipe3/ORG-Atlases-1.2
+slicer_exec: /software/rocky9/Slicer-5.6.1-linux-amd64/Slicer
+FiberTractMeasurements: /software/rocky9/Slicer-5.6.1-linux-amd64/slicer.org/Extensions-32438/SlicerDMRI/lib/Slicer-5.6/cli-modules/FiberTractMeasurements
+atlas: /software/rocky9/ORG-Atlases-1.2
 wma_nproc: 4
 xvfb: 1
-wma_cleanup: 0
+wma_cleanup: 2
 
 
 [StructMask]

From f73d75b4d99b5e2e36b4d25755a65657ea525301 Mon Sep 17 00:00:00 2001
From: "Billah, Tashrif" <35086881+tashrifbillah@users.noreply.github.com>
Date: Wed, 16 Oct 2024 14:24:32 -0400
Subject: [PATCH 09/33] * use session id as a variable * use CUDA_VERSION as a
 variable * replace /data/pnl/ by /software/rocky9/ * write less in template
 specification

---
 workflows/hcp_pnl_topup.lsf | 38 +++++++++++++++++++------------------
 1 file changed, 20 insertions(+), 18 deletions(-)

diff --git a/workflows/hcp_pnl_topup.lsf b/workflows/hcp_pnl_topup.lsf
index a7a8acc..8888c18 100755
--- a/workflows/hcp_pnl_topup.lsf
+++ b/workflows/hcp_pnl_topup.lsf
@@ -46,18 +46,20 @@ COMMENT
 bids_data_dir=/data/pnl/U01_HCP_Psychosis/data_processing/BIDS/rawdata
 
 # write four templates in the following order
-# change ses-1 in templates to the session you are processing
 # PA template, PA template
 # AP template, AP template
-raw_template="sub-*/ses-1/dwi/*_ses-1_acq-PA_dir-99_dwi.nii.gz sub-*/ses-1/dwi/*_ses-1_acq-PA_dir-107_dwi.nii.gz \
-              sub-*/ses-1/dwi/*_ses-1_acq-AP_dir-99_dwi.nii.gz sub-*/ses-1/dwi/*_ses-1_acq-AP_dir-107_dwi.nii.gz"
-unr_template="*_ses-1_acq-PA_dir-99_desc-XcUn_dwi.nii.gz *_ses-1_acq-PA_dir-107_desc-XcUn_dwi.nii.gz \
-              *_ses-1_acq-AP_dir-99_desc-XcUn_dwi.nii.gz *_ses-1_acq-AP_dir-107_desc-XcUn_dwi.nii.gz"
+raw_template="sub-*/ses-*/dwi/*_acq-PA_dir-99_dwi.nii.gz sub-*/ses-*/dwi/*_acq-PA_dir-107_dwi.nii.gz \
+              sub-*/ses-*/dwi/*_acq-AP_dir-99_dwi.nii.gz sub-*/ses-*/dwi/*_acq-AP_dir-107_dwi.nii.gz"
+unr_template="*_acq-PA_dir-99_desc-XcUn_dwi.nii.gz *_acq-PA_dir-107_desc-XcUn_dwi.nii.gz \
+              *_acq-AP_dir-99_desc-XcUn_dwi.nii.gz *_acq-AP_dir-107_desc-XcUn_dwi.nii.gz"
 
-# a single caseid or a text file with list of cases
+# a single case id or a text file with list of cases
 caselist=1004
+# a single session id
+s=01
 
 LUIGI_CONFIG_PATH=/data/pnl/soft/pnlpipe3/luigi-pnlpipe/params/hcp/dwi_pipe_params.cfg
+CUDA_VERSION=10.2
 
 # task is one of {HcpPipe,Ukf,Wma800}
 task=HcpPipe
@@ -67,9 +69,9 @@ task=HcpPipe
 
 HcpOutDir=hcppipe
 
-export HCPPIPEDIR=/data/pnl/soft/pnlpipe3/HCPpipelines
-export HCPPIPEDIR_Config=/data/pnl/soft/pnlpipe3/HCPpipelines/global/config
-export HCPPIPEDIR_Global=/data/pnl/soft/pnlpipe3/HCPpipelines/global/scripts
+export HCPPIPEDIR=/software/rocky9/HCPpipelines
+export HCPPIPEDIR_Config=/software/rocky9/HCPpipelines/global/config
+export HCPPIPEDIR_Global=/software/rocky9/HCPpipelines/global/scripts
 
 export LUIGI_CONFIG_PATH
 
@@ -78,7 +80,7 @@ if [ -z $cluster ]
 then
     source /data/pnl/soft/pnlpipe3/bashrc3-gpu
 else
-    source /rfanfs/pnl-zorro/software/pnlpipe3/bashrc3-gpu
+    source /software/rocky9/bashrc9
 fi
 
 if [ -f ${caselist} ]
@@ -90,21 +92,21 @@ else
     id=${caselist}
 fi
 
-
 # luigi-pnlpipe upto GibbsUn so data can be organized according to BIDS
 for j in $(echo $raw_template)
 do
-    /data/pnl/soft/pnlpipe3/luigi-pnlpipe/workflows/ExecuteTask.py \
+    /software/rocky9/luigi-pnlpipe/workflows/ExecuteTask.py \
     --bids-data-dir $bids_data_dir \
-    --task GibbsUn -c $id \
+    --task GibbsUn -c $id -s $s \
     --dwi-template $j
 done
 
 
 # determine --path for HCP pipe
 template=($raw_template)
-subdir=`echo ${template[0]} | sed "s+sub-\*+sub-$id+g"`
-datadir=`dirname $bids_data_dir`/derivatives/pnlpipe/`dirname $subdir`
+_subdir=`echo ${template[0]} | sed "s+sub-\*+sub-$id+g"`
+subdir=`echo ${_subdir} | sed "s+ses-\*+ses-$s+g"`
+datadir=`dirname $bids_data_dir`/derivatives/$USER-pnlpipe/`dirname $subdir`
 echo "HCP pipe data directory: $datadir"
 
 
@@ -119,7 +121,7 @@ then
     
     template=($unr_template)
     cmd="$HCPPIPEDIR/DiffusionPreprocessing/DiffPreprocPipeline.sh --path=$datadir \
-    --subject=$HcpOutDir --cuda-version=9.1 \
+    --subject=$HcpOutDir --cuda-version=$CUDA_VERSION \
     --posData=`ls $datadir/${template[0]}`@`ls $datadir/${template[1]}` \
     --negData=`ls $datadir/${template[2]}`@`ls $datadir/${template[3]}` \
     --echospacing=0.689998 --PEdir=2 --gdcoeffs=NONE \
@@ -132,8 +134,8 @@ fi
 
 # create symlinks
 template=($raw_template)
-/data/pnl/soft/pnlpipe3/luigi-pnlpipe/workflows/ExecuteTask.py \
+/software/rocky9/luigi-pnlpipe/workflows/ExecuteTask.py \
 --bids-data-dir $bids_data_dir \
 --dwi-template ${template[0]} \
---task $task -c $id
+--task $task -c $id -s $s
 

From de4152da46f12aa8fde3ad9dc6308c75f0cc9b6f Mon Sep 17 00:00:00 2001
From: "Billah, Tashrif" <35086881+tashrifbillah@users.noreply.github.com>
Date: Wed, 16 Oct 2024 15:16:35 -0400
Subject: [PATCH 10/33] remove BSUB stuff

---
 workflows/hcp_pnl_topup.lsf | 33 ---------------------------------
 1 file changed, 33 deletions(-)

diff --git a/workflows/hcp_pnl_topup.lsf b/workflows/hcp_pnl_topup.lsf
index 8888c18..7fdc595 100755
--- a/workflows/hcp_pnl_topup.lsf
+++ b/workflows/hcp_pnl_topup.lsf
@@ -3,23 +3,6 @@
 # Useful wiki for executing this in PNL GPU machines
 # https://github.com/pnlbwh/luigi-pnlpipe/wiki/Run-HCP-pipeline-on-PNL-GPU-machines-in-a-parallel-manner
 
-: << COMMENT
-Running this script w/o GPU support would take a long time (>24 hours).
-So always try to use a GPU enabled machine e.g. grx** nodes, eristwo gpu queue.
-=> Copy this script to your project directory and update the marked variables.
-=> Execute within shell as ./hcp_pnl_topup.lsf
-=> To execute through LSF, adjust [1-N]%2 and "BSUB -n 4" as explained below.
-COMMENT
-
-# ==============================================================================
-
-#BSUB -J hcp-topup[1-N]%2
-#BSUB -q gpu
-#BSUB -m ml001
-#BSUB -R rusage[mem=12000]
-#BSUB -o /data/pnl/U01_HCP_Psychosis/data_processing/output/hcp-topup-%J-%I.out
-#BSUB -e /data/pnl/U01_HCP_Psychosis/data_processing/output/hcp-topup-%J-%I.err
-#BSUB -n 4
 
 # check LSB_JOBINDEX because it won't exist in /rfanfs/
 if [ ! -z ${LSB_JOBINDEX} ]
@@ -27,22 +10,6 @@ then
     export CUDA_VISIBLE_DEVICES=$(( ${LSB_JOBINDEX}%2 ))
 fi
 
-: << COMMENT
-The formula for CUDA_VISIBLE_DEVICES is ${LSB_JOBINDEX}%G,
-which also means the maximum number of parallel cases you can process is G
-
-Adjust "BSUB -n 4" in a way that each GPU device can run no more than one job.
-You can use the formula "BSUB -n N/G" to ensure that where--
-    - N is the maximum number of jobs for that node
-    - G is the number of GPUs in that node
-Otherwise your jobs might crash due to out of memory error.
-
-Example: node ml001 has 8(=N) job slots and 2(=G) GPUs so "BSUB -n 8/2" and "BSUB -J hcp-topup[1-N]%2"
-
-COMMENT
-
-
-
 bids_data_dir=/data/pnl/U01_HCP_Psychosis/data_processing/BIDS/rawdata
 
 # write four templates in the following order

From d5b8aac52bc1b3094687c92e07458494116052da Mon Sep 17 00:00:00 2001
From: "Billah, Tashrif" <35086881+tashrifbillah@users.noreply.github.com>
Date: Wed, 16 Oct 2024 15:18:17 -0400
Subject: [PATCH 11/33] add [GibbsUn] to hcp/dwi_pipe_params.cfg

---
 params/hcp/dwi_pipe_params.cfg | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/params/hcp/dwi_pipe_params.cfg b/params/hcp/dwi_pipe_params.cfg
index b8e1caf..3516259 100644
--- a/params/hcp/dwi_pipe_params.cfg
+++ b/params/hcp/dwi_pipe_params.cfg
@@ -58,6 +58,8 @@ wma_cleanup: 2
 
 [BseMask]
 
+[GibbsUn]
+
 [CnnMask]
 
 [PnlEddy]

From 4974c58ea1d55e553e2de1e9a6fcc5241d1d1794 Mon Sep 17 00:00:00 2001
From: "Billah, Tashrif" <35086881+tashrifbillah@users.noreply.github.com>
Date: Wed, 16 Oct 2024 15:22:54 -0400
Subject: [PATCH 12/33] add [GibbsUn] to all dwi_pipe_params.cfg : it was
 missing before

---
 params/cte/dwi_pipe_params.cfg   | 4 +++-
 params/dwi_pipe_params.cfg       | 2 ++
 params/synb0/dwi_pipe_params.cfg | 2 ++
 3 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/params/cte/dwi_pipe_params.cfg b/params/cte/dwi_pipe_params.cfg
index 49e10ee..49a2707 100644
--- a/params/cte/dwi_pipe_params.cfg
+++ b/params/cte/dwi_pipe_params.cfg
@@ -21,7 +21,7 @@ filter:
 acqp: /data/pnl/DIAGNOSE_CTE_U01/acqp.txt
 index: /data/pnl/DIAGNOSE_CTE_U01/index.txt
 config: /data/pnl/DIAGNOSE_CTE_U01/eddy_config.txt
-useGpu: False
+useGpu: True
 FslOutDir: fsl_eddy
 
 
@@ -59,6 +59,8 @@ wma_cleanup: 0
 
 [BseMask]
 
+[GibbsUn]
+
 [CnnMask]
 
 [PnlEddy]
diff --git a/params/dwi_pipe_params.cfg b/params/dwi_pipe_params.cfg
index b7730aa..779568e 100644
--- a/params/dwi_pipe_params.cfg
+++ b/params/dwi_pipe_params.cfg
@@ -78,6 +78,8 @@ wma_cleanup: 0
 
 [BseMask]
 
+[GibbsUn]
+
 [CnnMask]
 
 [SynB0]
diff --git a/params/synb0/dwi_pipe_params.cfg b/params/synb0/dwi_pipe_params.cfg
index 0450657..c607a9f 100644
--- a/params/synb0/dwi_pipe_params.cfg
+++ b/params/synb0/dwi_pipe_params.cfg
@@ -56,6 +56,8 @@ wma_cleanup:
 
 [CnnMask]
 
+[GibbsUn]
+
 [SynB0]
 
 [PnlEddy]

From cefccd698104adb7e7390557d4f4ba9b49536673 Mon Sep 17 00:00:00 2001
From: "Billah, Tashrif" <35086881+tashrifbillah@users.noreply.github.com>
Date: Thu, 17 Oct 2024 10:25:13 -0400
Subject: [PATCH 13/33] * ask user to define raw_template only,
 programmatically obtain unr_template * change unr_template to just template

---
 workflows/hcp_pnl_topup.lsf | 41 +++++++++++++++++++++++++++++--------
 1 file changed, 32 insertions(+), 9 deletions(-)

diff --git a/workflows/hcp_pnl_topup.lsf b/workflows/hcp_pnl_topup.lsf
index 7fdc595..5c133db 100755
--- a/workflows/hcp_pnl_topup.lsf
+++ b/workflows/hcp_pnl_topup.lsf
@@ -17,8 +17,6 @@ bids_data_dir=/data/pnl/U01_HCP_Psychosis/data_processing/BIDS/rawdata
 # AP template, AP template
 raw_template="sub-*/ses-*/dwi/*_acq-PA_dir-99_dwi.nii.gz sub-*/ses-*/dwi/*_acq-PA_dir-107_dwi.nii.gz \
               sub-*/ses-*/dwi/*_acq-AP_dir-99_dwi.nii.gz sub-*/ses-*/dwi/*_acq-AP_dir-107_dwi.nii.gz"
-unr_template="*_acq-PA_dir-99_desc-XcUn_dwi.nii.gz *_acq-PA_dir-107_desc-XcUn_dwi.nii.gz \
-              *_acq-AP_dir-99_desc-XcUn_dwi.nii.gz *_acq-AP_dir-107_desc-XcUn_dwi.nii.gz"
 
 # a single case id or a text file with list of cases
 caselist=1004
@@ -77,6 +75,14 @@ datadir=`dirname $bids_data_dir`/derivatives/$USER-pnlpipe/`dirname $subdir`
 echo "HCP pipe data directory: $datadir"
 
 
+template=()
+for t in $(echo $raw_template)
+do
+    base=`basename $t`
+    template+=(${base//_dwi/_desc-XcUn_dwi})
+done
+
+
 # HCP pipe using GibbsUn data
 if [ ! -f $datadir/$HcpOutDir/Diffusion/eddy/eddy_unwarped_images.nii.gz ]
 then
@@ -86,16 +92,33 @@ then
     Command being issued to HCP pipeline:
     """
     
-    template=($unr_template)
-    cmd="$HCPPIPEDIR/DiffusionPreprocessing/DiffPreprocPipeline.sh --path=$datadir \
-    --subject=$HcpOutDir --cuda-version=$CUDA_VERSION \
-    --posData=`ls $datadir/${template[0]}`@`ls $datadir/${template[1]}` \
-    --negData=`ls $datadir/${template[2]}`@`ls $datadir/${template[3]}` \
-    --echospacing=0.689998 --PEdir=2 --gdcoeffs=NONE \
-    --extra-eddy-arg=--data_is_shelled --extra-eddy-arg=--repol --extra-eddy-arg=--verbose"
+
+    if [ ${#template[@]} == 4 ]
+    then
+        cmd="$HCPPIPEDIR/DiffusionPreprocessing/DiffPreprocPipeline.sh --path=$datadir \
+        --subject=$HcpOutDir --cuda-version=$CUDA_VERSION \
+        --posData=`ls $datadir/${template[0]}`@`ls $datadir/${template[1]}` \
+        --negData=`ls $datadir/${template[2]}`@`ls $datadir/${template[3]}` \
+        --echospacing=0.689998 --PEdir=2 --gdcoeffs=NONE \
+        --extra-eddy-arg=--data_is_shelled --extra-eddy-arg=--repol --extra-eddy-arg=--verbose"
+     elif [ ${#template[@]} == 2 ]
+     then
+        cmd="$HCPPIPEDIR/DiffusionPreprocessing/DiffPreprocPipeline.sh --path=$datadir \
+        --subject=$HcpOutDir --cuda-version=$CUDA_VERSION \
+        --posData=`ls $datadir/${template[0]}` \
+        --negData=`ls $datadir/${template[1]}` \
+        --echospacing=0.689998 --PEdir=2 --gdcoeffs=NONE \
+        --extra-eddy-arg=--data_is_shelled --extra-eddy-arg=--repol --extra-eddy-arg=--verbose"
+     else
+         echo Unknown number of --posData, --negData.
+         echo Execute $HCPPIPEDIR/DiffusionPreprocessing/DiffPreprocPipeline.sh manually.
+         exit 1
+     fi
+
     echo $cmd
     echo ''
     $cmd || exit 1
+
 fi
 
 

From e20be18658292543e497d7d811fcfe0763586373 Mon Sep 17 00:00:00 2001
From: "Billah, Tashrif" <35086881+tashrifbillah@users.noreply.github.com>
Date: Thu, 17 Oct 2024 10:56:40 -0400
Subject: [PATCH 14/33] note about 4 or 2 templates

---
 workflows/hcp_pnl_topup.lsf | 35 +++++++++++++++++++++++++----------
 1 file changed, 25 insertions(+), 10 deletions(-)

diff --git a/workflows/hcp_pnl_topup.lsf b/workflows/hcp_pnl_topup.lsf
index 5c133db..809943d 100755
--- a/workflows/hcp_pnl_topup.lsf
+++ b/workflows/hcp_pnl_topup.lsf
@@ -1,20 +1,25 @@
 #!/usr/bin/bash
 
-# Useful wiki for executing this in PNL GPU machines
-# https://github.com/pnlbwh/luigi-pnlpipe/wiki/Run-HCP-pipeline-on-PNL-GPU-machines-in-a-parallel-manner
+# Copy this script to your project directory and edit as needed
 
+# Useful wiki for executing this script in PNL GPU machines
+# https://github.com/pnlbwh/luigi-pnlpipe/wiki/Run-HCP-pipeline-on-PNL-GPU-machines
 
-# check LSB_JOBINDEX because it won't exist in /rfanfs/
-if [ ! -z ${LSB_JOBINDEX} ]
-then
-    export CUDA_VISIBLE_DEVICES=$(( ${LSB_JOBINDEX}%2 ))
-fi
+: << COMMENT
+HCP toolbox accepts an even number (2 or 4) of opposing templates.
+You can write four templates in the following order:
+
+    PA template PA template
+    AP template AP template
+
+Or two templates in the following order:
+
+    PA template AP template
+
+COMMENT
 
 bids_data_dir=/data/pnl/U01_HCP_Psychosis/data_processing/BIDS/rawdata
 
-# write four templates in the following order
-# PA template, PA template
-# AP template, AP template
 raw_template="sub-*/ses-*/dwi/*_acq-PA_dir-99_dwi.nii.gz sub-*/ses-*/dwi/*_acq-PA_dir-107_dwi.nii.gz \
               sub-*/ses-*/dwi/*_acq-AP_dir-99_dwi.nii.gz sub-*/ses-*/dwi/*_acq-AP_dir-107_dwi.nii.gz"
 
@@ -29,9 +34,13 @@ CUDA_VERSION=10.2
 # task is one of {HcpPipe,Ukf,Wma800}
 task=HcpPipe
 
+
+
+# You should not edit anything beyond this line
 # ==============================================================================
 
 
+
 HcpOutDir=hcppipe
 
 export HCPPIPEDIR=/software/rocky9/HCPpipelines
@@ -48,6 +57,12 @@ else
     source /software/rocky9/bashrc9
 fi
 
+# check LSB_JOBINDEX because it won't exist in /rfanfs/
+if [ ! -z ${LSB_JOBINDEX} ]
+then
+    export CUDA_VISIBLE_DEVICES=$(( ${LSB_JOBINDEX}%2 ))
+fi
+
 if [ -f ${caselist} ]
 then
     # LSF script, list of subjects

From b0b561548811be02d1c39f2164d3a46c294708d8 Mon Sep 17 00:00:00 2001
From: "Billah, Tashrif" <35086881+tashrifbillah@users.noreply.github.com>
Date: Thu, 17 Oct 2024 11:41:33 -0400
Subject: [PATCH 15/33] remove comment

---
 workflows/hcp_pnl_topup.lsf | 1 -
 1 file changed, 1 deletion(-)

diff --git a/workflows/hcp_pnl_topup.lsf b/workflows/hcp_pnl_topup.lsf
index 809943d..080ef12 100755
--- a/workflows/hcp_pnl_topup.lsf
+++ b/workflows/hcp_pnl_topup.lsf
@@ -137,7 +137,6 @@ then
 fi
 
 
-# create symlinks
 template=($raw_template)
 /software/rocky9/luigi-pnlpipe/workflows/ExecuteTask.py \
 --bids-data-dir $bids_data_dir \

From 376a1fb025555eabce8533821aaae9ccc6d9b6cd Mon Sep 17 00:00:00 2001
From: "Billah, Tashrif" <35086881+tashrifbillah@users.noreply.github.com>
Date: Thu, 17 Oct 2024 11:44:24 -0400
Subject: [PATCH 16/33] SAMBA protocol 3.0 does not support symlink creation
 anymore Reference:
 https://superuser.com/questions/1337257/clients-cant-create-symlinks-on-samba-share
 Hence, move and copyfile as needed

Co-authored-by: Ryan Zurrin <ryanzurrin@ryanzurrin.com>
---
 workflows/dwi_pipe.py | 19 +++++++++----------
 1 file changed, 9 insertions(+), 10 deletions(-)

diff --git a/workflows/dwi_pipe.py b/workflows/dwi_pipe.py
index e591ac1..5d2a6eb 100755
--- a/workflows/dwi_pipe.py
+++ b/workflows/dwi_pipe.py
@@ -5,7 +5,7 @@
 from glob import glob
 from os.path import join as pjoin, abspath, isfile, basename, dirname, isdir
 from os import symlink, getenv
-from shutil import move, rmtree
+from shutil import move, rmtree, copyfile
 
 from plumbum import local
 from subprocess import Popen, check_call
@@ -102,9 +102,9 @@ class CnnMask(Task):
     def run(self):
         
         with TemporaryDirectory() as tmpdir, local.cwd(tmpdir):
-            symlink(self.input()['dwi'],self.input()['dwi'].basename)
-            symlink(self.input()['bval'],self.input()['bval'].basename)
-            symlink(self.input()['bvec'],self.input()['bvec'].basename)
+            copyfile(self.input()['dwi'],self.input()['dwi'].basename)
+            copyfile(self.input()['bval'],self.input()['bval'].basename)
+            copyfile(self.input()['bvec'],self.input()['bvec'].basename)
             
             
             dwi_list= 'dwi_list.txt'
@@ -643,13 +643,12 @@ def output(self):
         bse= local.path(bse_prefix.split('_desc-')[0]+ '_desc-'+ desc+ '_bse.nii.gz')
         
         
-        # create symlinks
         if not isfile(dwi):
-            symlink(dwiHcp, dwi)
-            symlink(bvalHcp, bval)
-            symlink(bvecHcp, bvec)
-            symlink(maskHcp, mask)
-            symlink(bseHcp, bse)
+            move(dwiHcp, dwi)
+            move(bvalHcp, bval)
+            move(bvecHcp, bvec)
+            move(maskHcp, mask)
+            move(bseHcp, bse)
        
         check_call('cp $FSLDIR/etc/fslversion {}'.format(dwi.dirname), shell=True)
 

From c89a6fb10d6acb48456dc8221a40b5a64951aec2 Mon Sep 17 00:00:00 2001
From: "Billah, Tashrif" <35086881+tashrifbillah@users.noreply.github.com>
Date: Thu, 17 Oct 2024 14:49:11 -0400
Subject: [PATCH 17/33] add a note about how to execute

---
 workflows/hcp_pnl_topup.lsf | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/workflows/hcp_pnl_topup.lsf b/workflows/hcp_pnl_topup.lsf
index 080ef12..c551128 100755
--- a/workflows/hcp_pnl_topup.lsf
+++ b/workflows/hcp_pnl_topup.lsf
@@ -1,6 +1,7 @@
 #!/usr/bin/bash
 
 # Copy this script to your project directory and edit as needed
+# There are no arguments to this script. Execute it as ./hcp_pnl_topup.lsf
 
 # Useful wiki for executing this script in PNL GPU machines
 # https://github.com/pnlbwh/luigi-pnlpipe/wiki/Run-HCP-pipeline-on-PNL-GPU-machines
@@ -16,8 +17,13 @@ Or two templates in the following order:
 
     PA template AP template
 
+Notes:
+    * Even if one of the templates is a single b0, name it like DWI with _dwi.nii.gz suffix.
+    * All b0 volumes need to be accompanied by .bval and .bvec files. Usually, the .bvec file is in 3xN format.
+
 COMMENT
 
+
 bids_data_dir=/data/pnl/U01_HCP_Psychosis/data_processing/BIDS/rawdata
 
 raw_template="sub-*/ses-*/dwi/*_acq-PA_dir-99_dwi.nii.gz sub-*/ses-*/dwi/*_acq-PA_dir-107_dwi.nii.gz \

From 4eaa0edcc15f480cc73ecd575efaa4babd7b8cfe Mon Sep 17 00:00:00 2001
From: "Billah, Tashrif" <35086881+tashrifbillah@users.noreply.github.com>
Date: Thu, 17 Oct 2024 15:47:50 -0400
Subject: [PATCH 18/33] use SOFTDIR variable as a switch b/w data-pnl and
 software-rocky9

Co-authored-by: Ryan Zurrin <ryanzurrin@ryanzurrin.com>
---
 workflows/hcp_pnl_topup.lsf | 27 +++++++++++++++------------
 1 file changed, 15 insertions(+), 12 deletions(-)

diff --git a/workflows/hcp_pnl_topup.lsf b/workflows/hcp_pnl_topup.lsf
index c551128..0ee49d0 100755
--- a/workflows/hcp_pnl_topup.lsf
+++ b/workflows/hcp_pnl_topup.lsf
@@ -47,22 +47,25 @@ task=HcpPipe
 
 
 
-HcpOutDir=hcppipe
-
-export HCPPIPEDIR=/software/rocky9/HCPpipelines
-export HCPPIPEDIR_Config=/software/rocky9/HCPpipelines/global/config
-export HCPPIPEDIR_Global=/software/rocky9/HCPpipelines/global/scripts
-
-export LUIGI_CONFIG_PATH
-
 cluster=`hostname | grep pnl-.*.partners.org`
 if [ -z $cluster ]
 then
-    source /data/pnl/soft/pnlpipe3/bashrc3-gpu
+    SOFTDIR=/data/pnl/soft/pnlpipe3
+    source ${SOFTDIR}/bashrc3-gpu
 else
-    source /software/rocky9/bashrc9
+    SOFTDIR=/software/rocky9/
+    source ${SOFTDIR}/bashrc9
 fi
 
+
+HcpOutDir=hcppipe
+
+export HCPPIPEDIR=${SOFTDIR}/HCPpipelines
+export HCPPIPEDIR_Config=${SOFTDIR}/HCPpipelines/global/config
+export HCPPIPEDIR_Global=${SOFTDIR}/HCPpipelines/global/scripts
+
+export LUIGI_CONFIG_PATH
+
 # check LSB_JOBINDEX because it won't exist in /rfanfs/
 if [ ! -z ${LSB_JOBINDEX} ]
 then
@@ -81,7 +84,7 @@ fi
 # luigi-pnlpipe upto GibbsUn so data can be organized according to BIDS
 for j in $(echo $raw_template)
 do
-    /software/rocky9/luigi-pnlpipe/workflows/ExecuteTask.py \
+    ${SOFTDIR}/luigi-pnlpipe/workflows/ExecuteTask.py \
     --bids-data-dir $bids_data_dir \
     --task GibbsUn -c $id -s $s \
     --dwi-template $j
@@ -144,7 +147,7 @@ fi
 
 
 template=($raw_template)
-/software/rocky9/luigi-pnlpipe/workflows/ExecuteTask.py \
+${SOFTDIR}/luigi-pnlpipe/workflows/ExecuteTask.py \
 --bids-data-dir $bids_data_dir \
 --dwi-template ${template[0]} \
 --task $task -c $id -s $s

From 4e5977ab73da25030637e50ea7f78005f11f5e40 Mon Sep 17 00:00:00 2001
From: "Billah, Tashrif" <35086881+tashrifbillah@users.noreply.github.com>
Date: Tue, 22 Oct 2024 14:38:03 -0400
Subject: [PATCH 19/33] ignore luigi.parameter.UnconsumedParameterWarning

---
 workflows/ExecuteTask.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/workflows/ExecuteTask.py b/workflows/ExecuteTask.py
index cc546fe..8eb28c4 100755
--- a/workflows/ExecuteTask.py
+++ b/workflows/ExecuteTask.py
@@ -2,7 +2,7 @@
 
 import argparse
 from conversion import read_cases
-from luigi import build, configuration
+from luigi import build, configuration, parameter
 from _define_outputs import IO
 from struct_pipe import StructMask, Freesurfer
 from dwi_pipe import DwiAlign, GibbsUn, CnnMask, \
@@ -13,6 +13,8 @@
 from tempfile import gettempdir
 from glob import glob
 
+import warnings
+warnings.filterwarnings(action='ignore', category=parameter.UnconsumedParameterWarning)
 
 def _rm_tempfiles(names):
 

From d9c9a2cca31de9d2550eb740fd1666eb06ee3e09 Mon Sep 17 00:00:00 2001
From: "Billah, Tashrif" <35086881+tashrifbillah@users.noreply.github.com>
Date: Thu, 24 Oct 2024 11:51:05 -0400
Subject: [PATCH 20/33] use .name instead of .basename to ward off 
 FutureWarning

---
 workflows/_provenance.py | 2 +-
 workflows/dwi_pipe.py    | 2 +-
 workflows/struct_pipe.py | 4 ++--
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/workflows/_provenance.py b/workflows/_provenance.py
index 9b1b726..357fcb3 100644
--- a/workflows/_provenance.py
+++ b/workflows/_provenance.py
@@ -66,7 +66,7 @@ def write_provenance(obj, output=None):
     
     logfile= output.dirname.join(output.stem)+'.log.html'
     with open(logfile,'w') as f:
-        template= template.replace('{{output}}',output.basename)
+        template= template.replace('{{output}}',output.name)
         template= template.replace('{{textHistory}}',tree)
         template= template.replace('{{htmlHistory}}',history_tree)
         f.write(template)
diff --git a/workflows/dwi_pipe.py b/workflows/dwi_pipe.py
index 5d2a6eb..3b4f69e 100755
--- a/workflows/dwi_pipe.py
+++ b/workflows/dwi_pipe.py
@@ -734,7 +734,7 @@ def run(self):
         write_provenance(self, outDir)
 
     def output(self):
-        prefix= self.input().dirname.join('wma800',self.input().basename.split('.vtk')[0],
+        prefix= self.input().dirname.join('wma800',self.input().name.split('.vtk')[0],
             'FiberClustering/SeparatedClusters')
         
         clusters=[]
diff --git a/workflows/struct_pipe.py b/workflows/struct_pipe.py
index fbbc1f3..dcd35b2 100755
--- a/workflows/struct_pipe.py
+++ b/workflows/struct_pipe.py
@@ -50,7 +50,7 @@ def run(self):
     def output(self):
 
         subject_dir= dirname(self.input().replace('rawdata', self.derivatives_dir))
-        prefix= self.input().basename
+        prefix= self.input().name
         
         if '_T1w' in prefix:
             return local.path(pjoin(subject_dir, prefix.split('_T1w.nii')[0]+ '_desc-Xc_T1w.nii.gz'))
@@ -134,7 +134,7 @@ def run(self):
 
     def output(self):
 
-        prefix= self.input().basename
+        prefix= self.input().name
         
         if self.mask_method.lower() in ['mabs','hd-bet']:
             desc= 'T1wXcMabs' if '_T1w' in prefix else 'T2wXcMabs'

From d28669eaecdb0c18098259ce7a31a9ed8d86c588 Mon Sep 17 00:00:00 2001
From: "Billah, Tashrif" <35086881+tashrifbillah@users.noreply.github.com>
Date: Wed, 30 Oct 2024 17:12:21 -0400
Subject: [PATCH 21/33] * standalone way of obtaining ANTs, UKFTractography,
 dcm2niix hashes * new way of obtaining FSL version * store Linux, NVIDIA, GPU
 info

---
 workflows/getenv.sh | 32 ++++++++++++++------------------
 1 file changed, 14 insertions(+), 18 deletions(-)

diff --git a/workflows/getenv.sh b/workflows/getenv.sh
index 2ef5c0e..4400479 100755
--- a/workflows/getenv.sh
+++ b/workflows/getenv.sh
@@ -14,27 +14,23 @@ echo luigi-pnlpipe,`$cmd` > $log_file
 cd ../pnlNipype
 echo pnlNipype,`$cmd` >> $log_file
 
-# pnlpipe hash
-cd ../pnlpipe
-echo pnlpipe,`$cmd` >> $log_file
-
-# ANTs, UKFTractography, dcm2niix, tract_querier hashes
-cd pnlpipe_software
-for s in ANTs UKFTractography dcm2niix tract_querier
-do
-    hash_line=`grep "DEFAULT_HASH = " $s.py`
-    IFS=" = ", read -ra tmp <<< $hash_line
-    hash=`echo ${tmp[1]} | sed "s/'//g"`
-    echo $s,$hash >> $log_file
-done
-
+# ANTs, UKFTractography, dcm2niix hashes
+antsRegistration --version | head -n 1 | sed 's/ Version: /,/' >> $log_file
+echo UKFTractography,$(cd $(dirname `which UKFTractography`) && git rev-parse --short=7 HEAD) >> $log_file
+echo dcm2niix,`dcm2niix --version | tail -n 1` >> $log_file
 
 # FSL version
-hash_line=`eddy_openmp --help 2>&1 | grep "Part of FSL"`
-IFS=:, read -ra tmp <<< $hash_line
-hash=`echo ${tmp[1]} | sed "s/)//"`
-echo FSL,$hash >> $log_file
+echo FSL,`cat $FSLDIR/etc/fslversion` >> $log_file
 
 # FreeSurfer version
 echo FreeSurfer,`cat $FREESURFER_HOME/build-stamp.txt` >> $log_file
 
+# Linux version
+echo Computer,`cat /etc/system-release` `uname -nr` >> $log_file
+
+# NVIDIA version
+echo NVIDIA,`nvidia-smi | grep NVIDIA-SMI` >> $log_file
+
+# GPUs
+echo GPUs,`nvidia-smi -L` >> $log_file
+

From e0c885e222706628edb8d75349c7b1911c91aa9c Mon Sep 17 00:00:00 2001
From: "Billah, Tashrif" <35086881+tashrifbillah@users.noreply.github.com>
Date: Fri, 8 Nov 2024 11:47:28 -0500
Subject: [PATCH 22/33] split hash_file by \n character

---
 workflows/_provenance.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/workflows/_provenance.py b/workflows/_provenance.py
index 357fcb3..e8de67c 100644
--- a/workflows/_provenance.py
+++ b/workflows/_provenance.py
@@ -23,7 +23,7 @@ def _get_env():
     
     # read hashes
     with open(hash_file) as f:
-        content= f.read().split()
+        content= f.read().strip().split('\n')
     
     # save hashes in a dictionary for integrating with json provenance
     hash_dict={}

From b77ba3868f69d8aa76bddfb1e0d314fe208b3685 Mon Sep 17 00:00:00 2001
From: "Billah, Tashrif" <35086881+tashrifbillah@users.noreply.github.com>
Date: Fri, 8 Nov 2024 11:48:28 -0500
Subject: [PATCH 23/33] remove specific path

---
 workflows/_deps_tree.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/workflows/_deps_tree.py b/workflows/_deps_tree.py
index 056668c..2be7ccc 100644
--- a/workflows/_deps_tree.py
+++ b/workflows/_deps_tree.py
@@ -61,7 +61,7 @@ def get_record_id(task_id):
     # useful debug commands
     # full path necessary after .open
     # sqlite3
-    #   sqlite> .open /home/tb571/luigi-task-hist.db
+    #   sqlite> .open /path/to/luigi-task-hist.db
     #   sqlite> SELECT * FROM tasks;
     #   sqlite> .quit
     cur = conn.cursor()

From 0a56838d2e2686d058fd8f2db25eb28e0a2082b1 Mon Sep 17 00:00:00 2001
From: "Billah, Tashrif" <35086881+tashrifbillah@users.noreply.github.com>
Date: Fri, 15 Nov 2024 17:37:21 -0500
Subject: [PATCH 24/33] replace .basename with .name

---
 workflows/dwi_pipe.py    | 8 ++++----
 workflows/struct_pipe.py | 2 +-
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/workflows/dwi_pipe.py b/workflows/dwi_pipe.py
index 3b4f69e..fd49e53 100755
--- a/workflows/dwi_pipe.py
+++ b/workflows/dwi_pipe.py
@@ -102,14 +102,14 @@ class CnnMask(Task):
     def run(self):
         
         with TemporaryDirectory() as tmpdir, local.cwd(tmpdir):
-            copyfile(self.input()['dwi'],self.input()['dwi'].basename)
-            copyfile(self.input()['bval'],self.input()['bval'].basename)
-            copyfile(self.input()['bvec'],self.input()['bvec'].basename)
+            copyfile(self.input()['dwi'],self.input()['dwi'].name)
+            copyfile(self.input()['bval'],self.input()['bval'].name)
+            copyfile(self.input()['bvec'],self.input()['bvec'].name)
             
             
             dwi_list= 'dwi_list.txt'
             with open(dwi_list,'w') as f:
-                f.write(pjoin(tmpdir,self.input()['dwi'].basename))
+                f.write(pjoin(tmpdir,self.input()['dwi'].name))
             
 
             cmd = (' ').join(['dwi_masking.py',
diff --git a/workflows/struct_pipe.py b/workflows/struct_pipe.py
index dcd35b2..0a96e2d 100755
--- a/workflows/struct_pipe.py
+++ b/workflows/struct_pipe.py
@@ -211,7 +211,7 @@ def run(self):
 
 
     def output(self):
-        prefix= self.input()['aligned'].basename
+        prefix= self.input()['aligned'].name
         
         if '_T1w' in prefix:
             outPrefix= pjoin(self.input()['aligned'].dirname, prefix.split('_T1w.nii')[0])

From d1e7ebc33af73fa755a8fcfe6d597251b063a711 Mon Sep 17 00:00:00 2001
From: "Billah, Tashrif" <35086881+tashrifbillah@users.noreply.github.com>
Date: Tue, 19 Nov 2024 11:24:41 -0500
Subject: [PATCH 25/33] restore symlinks but relatively hcpipe/ output needs to
 be in place to check the status of its completion in future. So we cannot
 move them to BIDS directory. * remove obsolete comments

---
 workflows/dwi_pipe.py | 30 +++++++++++-------------------
 1 file changed, 11 insertions(+), 19 deletions(-)

diff --git a/workflows/dwi_pipe.py b/workflows/dwi_pipe.py
index fd49e53..4ad3f17 100755
--- a/workflows/dwi_pipe.py
+++ b/workflows/dwi_pipe.py
@@ -588,19 +588,11 @@ def output(self):
                                       'via HcpOutDir parameter in {getenv("LUIGI_CONFIG_PATH")}')
         
         # construct HCP pipe outputs
-        '''
-        Observe the following output files in ${StudyFolder}/${Subject}:
-        dwi:   Diffusion/eddy/eddy_unwarped_images.nii.gz
-        bvals: Diffusion/eddy/Pos_Neg.bvals
-        bvecs: Diffusion/eddy/eddy_unwarped_images.eddy_rotated_bvecs
-        mask:  Diffusion/eddy/nodif_brain_mask.nii.gz
-        bse:   Diffusion/topup/hifib0.nii.gz
-        '''
-        dwiHcp= f'{hcpOutDir}/Diffusion/eddy/eddy_unwarped_images.nii.gz'
-        bvalHcp= f'{hcpOutDir}/Diffusion/eddy/Pos_Neg.bvals'
-        bvecHcp= f'{hcpOutDir}/Diffusion/eddy/eddy_unwarped_images.eddy_rotated_bvecs'
-        maskHcp= f'{hcpOutDir}/Diffusion/eddy/nodif_brain_mask.nii.gz'
-        bseHcp= f'{hcpOutDir}/Diffusion/topup/hifib0.nii.gz'
+        dwiHcp= f'{self.HcpOutDir}/Diffusion/eddy/eddy_unwarped_images.nii.gz'
+        bvalHcp= f'{self.HcpOutDir}/Diffusion/eddy/Pos_Neg.bvals'
+        bvecHcp= f'{self.HcpOutDir}/Diffusion/eddy/eddy_unwarped_images.eddy_rotated_bvecs'
+        maskHcp= f'{self.HcpOutDir}/Diffusion/eddy/nodif_brain_mask.nii.gz'
+        bseHcp= f'{self.HcpOutDir}/Diffusion/topup/hifib0.nii.gz'
 
         
         # determine luigi-pnlpipe outputs
@@ -615,7 +607,7 @@ def output(self):
 
         # find dir field
         if '_dir-' in dwiRaw:
-            dir= load_nifti(dwiHcp).shape[3]
+            dir= load_nifti(pjoin(hcpEddyDir,'eddy_unwarped_images.nii.gz')).shape[3]
             eddy_epi_prefix= local.path(re.sub('_dir-(.+?)_', f'_dir-{dir}_', eddy_epi_prefix))
 
         dwi = local.path(eddy_epi_prefix+ '_dwi.nii.gz')
@@ -644,11 +636,11 @@ def output(self):
         
         
         if not isfile(dwi):
-            move(dwiHcp, dwi)
-            move(bvalHcp, bval)
-            move(bvecHcp, bvec)
-            move(maskHcp, mask)
-            move(bseHcp, bse)
+            symlink(dwiHcp, dwi)
+            symlink(bvalHcp, bval)
+            symlink(bvecHcp, bvec)
+            symlink(maskHcp, mask)
+            symlink(bseHcp, bse)
        
         check_call('cp $FSLDIR/etc/fslversion {}'.format(dwi.dirname), shell=True)
 

From 99692c0ff6922483e09853da4e231e09a3923fed Mon Sep 17 00:00:00 2001
From: "Billah, Tashrif" <35086881+tashrifbillah@users.noreply.github.com>
Date: Tue, 19 Nov 2024 15:30:38 -0500
Subject: [PATCH 26/33] update luigi install instruction

---
 docs/README.md | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/docs/README.md b/docs/README.md
index 2c2cd0c..e1553c2 100644
--- a/docs/README.md
+++ b/docs/README.md
@@ -99,15 +99,16 @@ Finally, see [here](#running-luigi-pnlpipe) for instructions about running the p
 
 ## luigi package
 
-A *client* (PNL external collaborator) can use the official luigi package installed in `pnlpipe3` conda environment.
-However, a *server* should install Tashrif's development for `/history/by_task_id/` URL to function:
+Tashrif's development of `/history/by_task_id/` feature got merged in the official `luigi` package.
+Until a release is available, you should install it as:
 
-    pip install git+https://github.com/tashrifbillah/luigi.git@89c9aa750de8ae2badabe435d98c02e44a1aa8b4
+    pip install git+https://github.com/spotify/luigi.git@172128c3de7a41411a10e61e3c675b76595793e2
 
-*luigi-pnlpipe* itself will not fail without Tashrif's development on the *server* side. That means, you can 
-also use the official luigi package on the *server* side. But you will not be able to redirect to 
-`/history/by_task_id/` URLs generated in `*.log.html` provenance files. Notably, the provenance files 
-are generated on the *client* side by [_provenance.py](https://github.com/pnlbwh/luigi-pnlpipe/blob/afa6c8a86d481d8fe5d04ba1ceb533b5da740c32/workflows/_provenance.py) when *luigi-pipeline* is run.
+However, `luigi` history feature does not work with the latest `sqlalchemy`. Install an older version as:
+
+    pip install sqlalchemy==1.4.54
+
+Reference: https://github.com/spotify/luigi/issues/3320
 
 
 ## CNN-Diffusion-MRIBrain-Segmentation

From 7f3edcfe50a6ebbe2a219f81888bbc1b7198379d Mon Sep 17 00:00:00 2001
From: "Billah, Tashrif" <35086881+tashrifbillah@users.noreply.github.com>
Date: Tue, 19 Nov 2024 15:33:13 -0500
Subject: [PATCH 27/33] start luigid server as a service

---
 luigid.service  | 17 +++++++++++++++++
 start_server.sh |  4 ++++
 2 files changed, 21 insertions(+)
 create mode 100644 luigid.service
 create mode 100755 start_server.sh

diff --git a/luigid.service b/luigid.service
new file mode 100644
index 0000000..c3a10c0
--- /dev/null
+++ b/luigid.service
@@ -0,0 +1,17 @@
+[Unit]
+Description=PNL Luigi server at http://pnl-elite-1.partners.org/
+After=network-online.target nginx.service
+Wants=network-online.target nginx.service
+
+[Service]
+User=sf284
+Type=oneshot
+RemainAfterExit=yes
+ExecStart=/opt/start_server.sh
+ExecStop=pkill luigid
+KillMode=process
+TimeoutSec=60
+
+[Install]
+WantedBy=multi-user.target
+
diff --git a/start_server.sh b/start_server.sh
new file mode 100755
index 0000000..94ac4cf
--- /dev/null
+++ b/start_server.sh
@@ -0,0 +1,4 @@
+#!/bin/bash
+
+cd /opt/
+miniconda3/bin/luigid --background --logdir luigi-server

From 910edfeef6939452c541aec6feeafa4b00c9dda4 Mon Sep 17 00:00:00 2001
From: "Billah, Tashrif" <35086881+tashrifbillah@users.noreply.github.com>
Date: Mon, 9 Dec 2024 16:28:55 -0500
Subject: [PATCH 28/33] symlink cannot be created within mount points
 whatsoever, so reinstate move() * create a placeholder so that future HCP
 pipe attempt can skip re-run Fixes issue #73

---
 workflows/dwi_pipe.py | 27 ++++++++++++++++-----------
 1 file changed, 16 insertions(+), 11 deletions(-)

diff --git a/workflows/dwi_pipe.py b/workflows/dwi_pipe.py
index 4ad3f17..1b51ef2 100755
--- a/workflows/dwi_pipe.py
+++ b/workflows/dwi_pipe.py
@@ -588,11 +588,11 @@ def output(self):
                                       'via HcpOutDir parameter in {getenv("LUIGI_CONFIG_PATH")}')
         
         # construct HCP pipe outputs
-        dwiHcp= f'{self.HcpOutDir}/Diffusion/eddy/eddy_unwarped_images.nii.gz'
-        bvalHcp= f'{self.HcpOutDir}/Diffusion/eddy/Pos_Neg.bvals'
-        bvecHcp= f'{self.HcpOutDir}/Diffusion/eddy/eddy_unwarped_images.eddy_rotated_bvecs'
-        maskHcp= f'{self.HcpOutDir}/Diffusion/eddy/nodif_brain_mask.nii.gz'
-        bseHcp= f'{self.HcpOutDir}/Diffusion/topup/hifib0.nii.gz'
+        dwiHcp= f'{hcpOutDir}/Diffusion/eddy/eddy_unwarped_images.nii.gz'
+        bvalHcp= f'{hcpOutDir}/Diffusion/eddy/Pos_Neg.bvals'
+        bvecHcp= f'{hcpOutDir}/Diffusion/eddy/eddy_unwarped_images.eddy_rotated_bvecs'
+        maskHcp= f'{hcpOutDir}/Diffusion/eddy/nodif_brain_mask.nii.gz'
+        bseHcp= f'{hcpOutDir}/Diffusion/topup/hifib0.nii.gz'
 
         
         # determine luigi-pnlpipe outputs
@@ -636,12 +636,17 @@ def output(self):
         
         
         if not isfile(dwi):
-            symlink(dwiHcp, dwi)
-            symlink(bvalHcp, bval)
-            symlink(bvecHcp, bvec)
-            symlink(maskHcp, mask)
-            symlink(bseHcp, bse)
-       
+            move(dwiHcp, dwi)
+            move(bvalHcp, bval)
+            move(bvecHcp, bvec)
+            move(maskHcp, mask)
+            move(bseHcp, bse)
+            
+            # create a placeholder so that future HCP pipe attempt can skip rerun
+            with open(dwiHcp,'w') as f:
+                f.write('')
+
+
         check_call('cp $FSLDIR/etc/fslversion {}'.format(dwi.dirname), shell=True)
 
         return dict(dwi=dwi, bval=bval, bvec=bvec, bse=bse, mask=mask)

From 9d456b15bda7e46173a6d2448d6bb69f53c4e549 Mon Sep 17 00:00:00 2001
From: "Billah, Tashrif" <35086881+tashrifbillah@users.noreply.github.com>
Date: Thu, 2 Jan 2025 11:07:35 -0500
Subject: [PATCH 29/33] use conda env export to generate env

---
 workflows/_provenance.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/workflows/_provenance.py b/workflows/_provenance.py
index e8de67c..98c20ee 100644
--- a/workflows/_provenance.py
+++ b/workflows/_provenance.py
@@ -1,6 +1,6 @@
 from _deps_tree import print_tree, print_history_tree
 from os.path import join as pjoin, dirname, isfile
-from os import getpid, environ
+from os import getpid
 from subprocess import check_call, check_output
 from tempfile import gettempdir
 
@@ -34,7 +34,7 @@ def _get_env():
     # export conda env
     env_file= pjoin(gettempdir(), f'env-{getpid()}.yml')
     if not isfile(env_file):
-        check_output(f"{environ['CONDA_EXE']} env export > {env_file}", shell=True)
+        check_output(f"conda env export > {env_file}", shell=True)
     
     with open(env_file) as f:
         hash_dict['conda_env']= f.read()

From 420d0c6022edcadb462273cc330721a5ce8d419a Mon Sep 17 00:00:00 2001
From: "Billah, Tashrif" <35086881+tashrifbillah@users.noreply.github.com>
Date: Thu, 2 Jan 2025 14:13:24 -0500
Subject: [PATCH 30/33] all the adventure needed to omit softlink and data
 duplication: * introduced run() method, move output from eddy/ to dwi/
 directory * read index.txt to get number of directions * made intermediate
 vars part of class to prevent re-declaration * change HcpPipe to regular Task
 instead of ExternalTask

---
 workflows/dwi_pipe.py | 51 ++++++++++++++++++++++---------------------
 1 file changed, 26 insertions(+), 25 deletions(-)

diff --git a/workflows/dwi_pipe.py b/workflows/dwi_pipe.py
index 1b51ef2..1df8107 100755
--- a/workflows/dwi_pipe.py
+++ b/workflows/dwi_pipe.py
@@ -569,10 +569,27 @@ def output(self):
 
 
 @inherits(SelectDwiFiles, DwiAlign)
-class HcpPipe(ExternalTask):
+class HcpPipe(Task):
 
     HcpOutDir= Parameter(default='hcppipe')
     
+    def run(self):
+    
+        if not isfile(self.output()['dwi']):
+            move(self.dwiHcp, self.output()['dwi'])
+            move(self.bvalHcp, self.output()['bval'])
+            move(self.bvecHcp, self.output()['bvec'])
+            move(self.maskHcp, self.output()['mask'])
+            move(self.bseHcp, self.output()['bse'])
+            
+            # create a placeholder so that future HCP pipe attempt can skip rerun
+            with open(self.dwiHcp,'w') as f:
+                f.write('')
+
+
+        check_call('cp $FSLDIR/etc/fslversion {}'.format(self.output()['dwi'].dirname), shell=True)
+
+
     def output(self):
 
         # read one dwi to learn name and containing directory
@@ -587,18 +604,15 @@ def output(self):
             raise NotADirectoryError(f'{hcpEddyDir} does not exist. Provide HCP pipe output directory '
                                       'via HcpOutDir parameter in {getenv("LUIGI_CONFIG_PATH")}')
         
+
         # construct HCP pipe outputs
-        dwiHcp= f'{hcpOutDir}/Diffusion/eddy/eddy_unwarped_images.nii.gz'
-        bvalHcp= f'{hcpOutDir}/Diffusion/eddy/Pos_Neg.bvals'
-        bvecHcp= f'{hcpOutDir}/Diffusion/eddy/eddy_unwarped_images.eddy_rotated_bvecs'
-        maskHcp= f'{hcpOutDir}/Diffusion/eddy/nodif_brain_mask.nii.gz'
-        bseHcp= f'{hcpOutDir}/Diffusion/topup/hifib0.nii.gz'
+        self.dwiHcp= f'{hcpOutDir}/Diffusion/eddy/eddy_unwarped_images.nii.gz'
+        self.bvalHcp= f'{hcpOutDir}/Diffusion/eddy/Pos_Neg.bvals'
+        self.bvecHcp= f'{hcpOutDir}/Diffusion/eddy/eddy_unwarped_images.eddy_rotated_bvecs'
+        self.maskHcp= f'{hcpOutDir}/Diffusion/eddy/nodif_brain_mask.nii.gz'
+        self.bseHcp= f'{hcpOutDir}/Diffusion/topup/hifib0.nii.gz'
 
         
-        # determine luigi-pnlpipe outputs
-        # in https://github.com/pnlbwh/luigi-pnlpipe/commit/fc3a1a5319d027e3dad9e6afb393e7399a3d3c62
-        # lines 549-581 nearly replicates lines 480-505 of TopupEddy task
-
         # remove _acq-*
         eddy_epi_prefix= dwiRaw.rsplit('_dwi.nii.gz')[0]
         eddy_epi_prefix= eddy_epi_prefix.replace('_acq-PA','')
@@ -607,7 +621,8 @@ def output(self):
 
         # find dir field
         if '_dir-' in dwiRaw:
-            dir= load_nifti(pjoin(hcpEddyDir,'eddy_unwarped_images.nii.gz')).shape[3]
+            with open(pjoin(hcpEddyDir,'index.txt')) as f:
+                dir= len(f.read().split())
             eddy_epi_prefix= local.path(re.sub('_dir-(.+?)_', f'_dir-{dir}_', eddy_epi_prefix))
 
         dwi = local.path(eddy_epi_prefix+ '_dwi.nii.gz')
@@ -635,20 +650,6 @@ def output(self):
         bse= local.path(bse_prefix.split('_desc-')[0]+ '_desc-'+ desc+ '_bse.nii.gz')
         
         
-        if not isfile(dwi):
-            move(dwiHcp, dwi)
-            move(bvalHcp, bval)
-            move(bvecHcp, bvec)
-            move(maskHcp, mask)
-            move(bseHcp, bse)
-            
-            # create a placeholder so that future HCP pipe attempt can skip rerun
-            with open(dwiHcp,'w') as f:
-                f.write('')
-
-
-        check_call('cp $FSLDIR/etc/fslversion {}'.format(dwi.dirname), shell=True)
-
         return dict(dwi=dwi, bval=bval, bvec=bvec, bse=bse, mask=mask)
 
 

From 84967b856e4f67f99351de87f7034cf7e7529030 Mon Sep 17 00:00:00 2001
From: "Billah, Tashrif" <35086881+tashrifbillah@users.noreply.github.com>
Date: Wed, 8 Jan 2025 16:37:04 -0500
Subject: [PATCH 31/33] standardize luigi.cfg for distribution

---
 luigi.cfg | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/luigi.cfg b/luigi.cfg
index 6c2cf6d..3ac85da 100644
--- a/luigi.cfg
+++ b/luigi.cfg
@@ -1,10 +1,12 @@
 [core]
+# uncomment one of the Luigi servers
+
 # for individual machine
 # default-scheduler-url = http://localhost:8082/
 # PNL hosted private server
 # default-scheduler-url = http://${LUIGI_USERNAME}:${LUIGI_PASSWORD}@pnl-elite-1.partners.org/
 # PNL hosted public server
-default-scheduler-url = https://${LUIGI_USERNAME}:${LUIGI_PASSWORD}@pnlservers.bwh.harvard.edu/luigi/
+# default-scheduler-url = https://${LUIGI_USERNAME}:${LUIGI_PASSWORD}@pnlservers.bwh.harvard.edu/luigi/
 
 log_level = INFO
 

From cdc5b5cd559371f40c680b802cab5a8ad9275e74 Mon Sep 17 00:00:00 2001
From: "Billah, Tashrif" <35086881+tashrifbillah@users.noreply.github.com>
Date: Wed, 8 Jan 2025 16:49:29 -0500
Subject: [PATCH 32/33] include instructions for launching a Luigi server

---
 docs/README.md | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/docs/README.md b/docs/README.md
index e1553c2..3294a87 100644
--- a/docs/README.md
+++ b/docs/README.md
@@ -110,6 +110,15 @@ However, `luigi` history feature does not work with the latest `sqlalchemy`. Ins
 
 Reference: https://github.com/spotify/luigi/issues/3320
 
+If you are an outside collaborator, you can launch a Luigi server as:
+
+    luigi-pnlpipe/start_server.sh
+
+And subsequently, uncomment this line in `luigi-pnlpipe/luigi.cfg` to use it:
+
+    default-scheduler-url = http://localhost:8082/
+
+Psychiatry Neuroimaging Laboratory staff have it available readily. They do not need to launch one.
 
 ## CNN-Diffusion-MRIBrain-Segmentation
 

From 1e9e4a0f18a56164645c2ea09329d2c1e619b04c Mon Sep 17 00:00:00 2001
From: "Billah, Tashrif" <35086881+tashrifbillah@users.noreply.github.com>
Date: Wed, 8 Jan 2025 16:53:50 -0500
Subject: [PATCH 33/33] use Slicer-5.6.1-linux-amd64 in SynB0 config

---
 params/synb0/dwi_pipe_params.cfg | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/params/synb0/dwi_pipe_params.cfg b/params/synb0/dwi_pipe_params.cfg
index c607a9f..6231d30 100644
--- a/params/synb0/dwi_pipe_params.cfg
+++ b/params/synb0/dwi_pipe_params.cfg
@@ -43,8 +43,8 @@ bhigh: 2000
 
 
 ## [WMA800] ##
-slicer_exec: /data/pnl/soft/pnlpipe3/Slicer-4.10.2-linux-amd64/SlicerWithExtensions.sh
-FiberTractMeasurements: /data/pnl/soft/pnlpipe3/Slicer-4.10.2-linux-amd64/SlicerWithExtensions.sh --launch FiberTractMeasurements
+slicer_exec: /software/rocky9/Slicer-5.6.1-linux-amd64/Slicer
+FiberTractMeasurements: /software/rocky9/Slicer-5.6.1-linux-amd64/slicer.org/Extensions-32438/SlicerDMRI/lib/Slicer-5.6/cli-modules/FiberTractMeasurements
 atlas: /data/pnl/soft/pnlpipe3/ORG-Atlases-1.2
 wma_nproc: 4
 xvfb: 1