From 4be8eb72d094ae69e58bcc1c40e9b8303705f57e Mon Sep 17 00:00:00 2001 From: Danielle Callan Date: Thu, 2 May 2024 15:30:29 -0400 Subject: [PATCH 1/3] wip: attempt to support nf-core workflows --- StepClasses/lib/perl/RunAndMonitorNextflow.pm | 38 +++++++++++++++---- 1 file changed, 30 insertions(+), 8 deletions(-) diff --git a/StepClasses/lib/perl/RunAndMonitorNextflow.pm b/StepClasses/lib/perl/RunAndMonitorNextflow.pm index 107ab41..007bb60 100644 --- a/StepClasses/lib/perl/RunAndMonitorNextflow.pm +++ b/StepClasses/lib/perl/RunAndMonitorNextflow.pm @@ -77,7 +77,23 @@ sub getConfigDeclaration { } sub runAndMonitor { - my ($self, $test, $user, $submitServer, $transferServer, $jobInfoFile, $logFile, $nextflowStdoutFile, $workingDir, $time, $queue, $nextflowWorkflow, $isGit, $clusterNextflowConfigFile) = @_; + my ( + $self, + $test, + $user, + $submitServer, + $transferServer, + $jobInfoFile, + $logFile, + $nextflowStdoutFile, + $workingDir, + $time, + $queue, + $nextflowWorkflow, + $isGit, + $clusterNextflowConfigFile + $isNfCoreWorkflow + ) = @_; if ($self->getSharedConfigRelaxed('masterWorkflowDataDir')) { $self->log("Skipping runAndMonitorNextflow -- slave workflows don't run nextflow"); @@ -92,15 +108,21 @@ sub runAndMonitor { # first see if by any chance we are already done (would happen if somehow the flow lost track of the job) return 1 if $self->_checkClusterTaskLogForDone($logFile, $user, $transferServer); - #my $nextflowCmd = "nextflow run $nextflowWorkflow -with-trace -c $clusterNextflowConfigFile -resume >$nextflowStdoutFile 2>&1"; + my $nextflowCmd = "nextflow" + if ($isNfCoreWorkflow) { + # here we want to inherit config from nf-core, and cant guarantee the default branch is 'main' + $nextflowCmd = "$nextflowCmd -c $clusterNextflowConfigFile run $nextflowWorkflow -with-trace -resume >$nextflowStdoutFile 2>&1"; + } else { #use "-C" instead of "-c" to avoid taking from anything besides the specified config -my $nextflowCmd = "nextflow -C $clusterNextflowConfigFile run $nextflowWorkflow -with-trace -r main -resume >$nextflowStdoutFile 2>&1"; - if($isGit){ - $nextflowCmd = "nextflow pull $nextflowWorkflow; $nextflowCmd"; - } + $nextflowCmd = "$nextflowCmd -C $clusterNextflowConfigFile run $nextflowWorkflow -with-trace -r main -resume >$nextflowStdoutFile 2>&1"; + } + + if($isGit || $isNfCoreWorkflow) { + $nextflowCmd = "nextflow pull $nextflowWorkflow; $nextflowCmd"; + } - # prepend slash to ;, >, and & so that the command is submitted whole - $nextflowCmd =~ s{([;>&])}{\\$1}g; + # prepend slash to ;, >, and & so that the command is submitted whole + $nextflowCmd =~ s{([;>&])}{\\$1}g; my $submitCmd = $self->getNodeClass()->getQueueSubmitCommand($queue, $nextflowCmd); From f73ea72ddc42a0d49ff994d106222b75bb6dd69d Mon Sep 17 00:00:00 2001 From: Danielle Callan Date: Fri, 3 May 2024 14:33:17 -0400 Subject: [PATCH 2/3] add nfcore params file arg --- StepClasses/lib/perl/RunAndMonitorNextflow.pm | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/StepClasses/lib/perl/RunAndMonitorNextflow.pm b/StepClasses/lib/perl/RunAndMonitorNextflow.pm index 007bb60..ff236cd 100644 --- a/StepClasses/lib/perl/RunAndMonitorNextflow.pm +++ b/StepClasses/lib/perl/RunAndMonitorNextflow.pm @@ -91,8 +91,9 @@ sub runAndMonitor { $queue, $nextflowWorkflow, $isGit, - $clusterNextflowConfigFile - $isNfCoreWorkflow + $clusterNextflowConfigFile, + $clusterNextflowParamsFile, + $isNfCoreWorkflow, ) = @_; if ($self->getSharedConfigRelaxed('masterWorkflowDataDir')) { @@ -111,7 +112,7 @@ sub runAndMonitor { my $nextflowCmd = "nextflow" if ($isNfCoreWorkflow) { # here we want to inherit config from nf-core, and cant guarantee the default branch is 'main' - $nextflowCmd = "$nextflowCmd -c $clusterNextflowConfigFile run $nextflowWorkflow -with-trace -resume >$nextflowStdoutFile 2>&1"; + $nextflowCmd = "$nextflowCmd -c $clusterNextflowConfigFile run $nextflowWorkflow -with-trace -params-file $clusterNextflowParamsFile -resume >$nextflowStdoutFile 2>&1"; } else { #use "-C" instead of "-c" to avoid taking from anything besides the specified config $nextflowCmd = "$nextflowCmd -C $clusterNextflowConfigFile run $nextflowWorkflow -with-trace -r main -resume >$nextflowStdoutFile 2>&1"; From 9141024e2cec5f90f068fcb4401b756d3b5cb86d Mon Sep 17 00:00:00 2001 From: Danielle Callan Date: Fri, 3 May 2024 15:26:27 -0400 Subject: [PATCH 3/3] allow versioning of nf-core pipelines --- StepClasses/lib/perl/RunAndMonitorNextflow.pm | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/StepClasses/lib/perl/RunAndMonitorNextflow.pm b/StepClasses/lib/perl/RunAndMonitorNextflow.pm index ff236cd..c417b89 100644 --- a/StepClasses/lib/perl/RunAndMonitorNextflow.pm +++ b/StepClasses/lib/perl/RunAndMonitorNextflow.pm @@ -94,6 +94,7 @@ sub runAndMonitor { $clusterNextflowConfigFile, $clusterNextflowParamsFile, $isNfCoreWorkflow, + $pipelineVersion, ) = @_; if ($self->getSharedConfigRelaxed('masterWorkflowDataDir')) { @@ -112,7 +113,7 @@ sub runAndMonitor { my $nextflowCmd = "nextflow" if ($isNfCoreWorkflow) { # here we want to inherit config from nf-core, and cant guarantee the default branch is 'main' - $nextflowCmd = "$nextflowCmd -c $clusterNextflowConfigFile run $nextflowWorkflow -with-trace -params-file $clusterNextflowParamsFile -resume >$nextflowStdoutFile 2>&1"; + $nextflowCmd = "$nextflowCmd -c $clusterNextflowConfigFile run $nextflowWorkflow -with-trace -r $pipelineVersion -params-file $clusterNextflowParamsFile -resume >$nextflowStdoutFile 2>&1"; } else { #use "-C" instead of "-c" to avoid taking from anything besides the specified config $nextflowCmd = "$nextflowCmd -C $clusterNextflowConfigFile run $nextflowWorkflow -with-trace -r main -resume >$nextflowStdoutFile 2>&1";