Add onlyJobState option for SLURM executor (#6659)

Co-authored-by: Chris Hakkaart <[email protected]>
Co-authored-by: tcutts <[email protected]>

Author: 3 people

docs/executor.md

@@ -446,6 +446,10 @@ Nextflow does not provide direct support for SLURM multi-clusters. If you need t
 Some SLURM clusters require memory allocations to be specified with `--mem-per-cpu` instead of `--mem`. You can specify `executor.perCpuMemAllocation = true` in the Nextflow configuration to enable this behavior. Nextflow will automatically compute the memory per CPU for each task (by default 1 CPU is used).
 :::
 
+:::{versionadded} 25.12.0-edge
+Since SLURM 24, `squeue` supports an `--only-job-state` option that ignores the partition (`-p`) or user (`-u`) filters. To enable this behavior, specify `executor.$slurm.onlyJobState = true` in your Nextflow configuration. If `SchedulerParameters=enable_job_state_cache` is enabled, you can expect improved Nextflow performance and reduced load on the SLURM controller. See [`enable_job_state_cache`](https://slurm.schedmd.com/slurm.conf.html#OPT_enable_job_state_cache) and [`--only-job-state`](https://slurm.schedmd.com/squeue.html#OPT_only-job-state) for more information.
+:::
+
 (tcs-executor)=
 
 ## TCS

docs/reference/config.md

@@ -715,6 +715,13 @@ The following settings are available:
 : *Used only by the {ref}`slurm-executor` executor.*
 : When `true`, memory allocations for SLURM jobs are specified as `--mem-per-cpu <task.memory / task.cpus>` instead of `--mem <task.memory>`.
 
+`executor.onlyJobState`
+: :::{versionadded} 25.12.0-edge
+  :::
+: *Used only by the {ref}`slurm-executor` executor.*
+: *Requires SLURM 24.05 or later.*
+: When `true`, job status queries use `squeue --only-job-state` without partition (`-p`) or user (`-u`) filters. This can reduce the load on the SLURM controller, especially if your SLURM administrator has enabled `SchedulerParameters=enable_job_state_cache` in your [SLURM configuration](https://slurm.schedmd.com/slurm.conf.html#OPT_enable_job_state_cache). See [`--only-job-state`](https://slurm.schedmd.com/squeue.html#OPT_only-job-state) for more information (default: `false`).
+
 `executor.perJobMemLimit`
 : *Used only by the {ref}`lsf-executor` executor.*
 : Enables the *per-job* memory limit mode for LSF jobs.

modules/nextflow/src/main/groovy/nextflow/executor/ExecutorConfig.groovy

@@ -108,6 +108,14 @@ class ExecutorConfig implements ConfigScope {
     """)
     final boolean perJobMemLimit
 
+    @ConfigOption
+    @Description("""
+        *Used only by the SLURM executor.*
+
+        When `true`, job status queries use `squeue --only-job-state` without partition (`-p`) or user (`-u`) filters. This is required for SLURM installations where `squeue` only accepts the `--only-job-state` option (default: `false`).
+    """)
+    final boolean onlyJobState
+
     @ConfigOption
     @Description("""
         *Used only by the LSF executor.*
@@ -171,6 +179,7 @@ class ExecutorConfig implements ConfigScope {
         name = opts.name
         perCpuMemAllocation = opts.perCpuMemAllocation as boolean
         perJobMemLimit = opts.perJobMemLimit as boolean
+        onlyJobState = opts.onlyJobState as boolean
         perTaskReserve = opts.perTaskReserve as boolean
         pollInterval = opts.pollInterval as Duration
         queueGlobalStatus = opts.queueGlobalStatus as boolean

modules/nextflow/src/main/groovy/nextflow/executor/SlurmExecutor.groovy

@@ -40,6 +40,7 @@ class SlurmExecutor extends AbstractGridExecutor implements TaskArrayExecutor {
     static private Pattern SUBMIT_REGEX = ~/Submitted batch job (\d+)/
 
     private boolean perCpuMemAllocation
+    private boolean onlyJobState
 
     private boolean hasSignalOpt(TaskConfig config) {
         final opts = config.getClusterOptionsAsString()
@@ -157,7 +158,13 @@ class SlurmExecutor extends AbstractGridExecutor implements TaskArrayExecutor {
     @Override
     protected List<String> queueStatusCommand(Object queue) {
 
-        final result = ['squeue','--noheader','-o','%i %t', '-t', 'all']
+        final result = ['squeue','--noheader', '-o','%i %t', '-t', 'all']
+
+        if( onlyJobState ) {
+            result << '--only-job-state'
+            // -p and -u cannot be used with --only-job-state
+            return result
+        }
 
         if( queue )
             result << '-p' << queue.toString()
@@ -213,6 +220,7 @@ class SlurmExecutor extends AbstractGridExecutor implements TaskArrayExecutor {
     void register() {
         super.register()
         perCpuMemAllocation = config.getExecConfigProp(name, 'perCpuMemAllocation', false)
+        onlyJobState = config.getExecConfigProp(name, 'onlyJobState', false)
     }
 
     @Override

modules/nextflow/src/test/groovy/nextflow/executor/ExecutorConfigTest.groovy

@@ -116,5 +116,27 @@ class ExecutorConfigTest extends Specification {
         config.getExecConfigProp( 'hazelcast', 'jobName', 'alpha', [NXF_EXECUTOR_JOBNAME:'hola']) == 'hola'
     }
 
+    def 'test onlyJobState property'() {
+
+        when:
+        def config = new ExecutorConfig(['$slurm':[onlyJobState: true] ])
+        then:
+        config.getExecConfigProp('slurm', 'onlyJobState', false) == true
+        config.getExecConfigProp('lsf', 'onlyJobState', false) == false
+
+        when:
+        config = new ExecutorConfig([onlyJobState: true, '$slurm':[onlyJobState: false] ])
+        then:
+        config.getExecConfigProp('slurm', 'onlyJobState', false) == false
+        config.getExecConfigProp('lsf', 'onlyJobState', false) == true
+
+        when:
+        config = new ExecutorConfig([:])
+        then:
+        config.getExecConfigProp('slurm', 'onlyJobState', false) == false
+        config.getExecConfigProp('slurm', 'onlyJobState', true) == false  // Property exists and is false, so default is not used
+        config.getExecConfigProp('slurm', 'nonExistentProp', true) == true  // Non-existent property uses default
+    }
+
 
 }

modules/nextflow/src/test/groovy/nextflow/executor/SlurmExecutorTest.groovy

@@ -312,6 +312,54 @@ class SlurmExecutorTest extends Specification {
         exec.queueStatusCommand('xxx') == ['squeue','--noheader','-o','%i %t','-t','all','-p','xxx','-u', usr]
     }
 
+    def 'should use onlyJobState option and omit queue and user' () {
+        given:
+        def exec = createExecutor()
+        exec.@onlyJobState = true
+
+        when:
+        def result = exec.queueStatusCommand(null)
+
+        then:
+        result == ['squeue','--noheader','-o','%i %t','-t','all','--only-job-state']
+        !result.contains('-u')
+        !result.contains('-p')
+    }
+
+    def 'should use onlyJobState option and ignore queue parameter' () {
+        given:
+        def exec = createExecutor()
+        exec.@onlyJobState = true
+
+        when:
+        def result = exec.queueStatusCommand('myqueue')
+
+        then:
+        result == ['squeue','--noheader','-o','%i %t','-t','all','--only-job-state']
+        !result.contains('-p')
+        !result.contains('myqueue')
+        !result.contains('-u')
+    }
+
+    def 'should include queue and user when onlyJobState is disabled' () {
+        given:
+        def exec = createExecutor()
+        exec.@onlyJobState = false
+
+        when:
+        def usr = System.getProperty('user.name')
+        def resultNoQueue = exec.queueStatusCommand(null)
+        def resultWithQueue = exec.queueStatusCommand('myqueue')
+
+        then:
+        resultNoQueue == ['squeue','--noheader','-o','%i %t','-t','all','-u', usr]
+        !resultNoQueue.contains('--only-job-state')
+
+        and:
+        resultWithQueue == ['squeue','--noheader','-o','%i %t','-t','all','-p','myqueue','-u', usr]
+        !resultWithQueue.contains('--only-job-state')
+    }
+
     def 'should get array index name and start' () {
         given:
         def executor = createExecutor()