[TRTLLM-13304][test] Add VisualGen multi-node E2E and external-launch unit coverage

jenkins/L0_MergeRequest.groovy

@@ -1109,6 +1109,7 @@ def getMultiGpuFileChanged(pipeline, testFilter, globalVars)
         "tests/integration/defs/cpp/test_multi_gpu.py",
         "tests/integration/test_lists/test-db/l0_b200_multi_gpus_perf_sanity.yml",
         "tests/integration/test_lists/test-db/l0_b200_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node1_gpu8.yml",
+        "tests/integration/test_lists/test-db/l0_b200_visual_gen_multi_nodes.yml",
         "tests/integration/test_lists/test-db/l0_b200_visual_gen_perf_sanity.yml",
         "tests/integration/test_lists/test-db/l0_dgx_b200.yml",
         "tests/integration/test_lists/test-db/l0_dgx_b300.yml",

jenkins/L0_Test.groovy

@@ -1362,6 +1362,16 @@ def getNodeArgs(int nodeCount, int gpuCount, boolean setSegment = false) {
     return args
 }
 
+def getVisualGenMultinodeParentNodeArgs(int nodeCount, int gpuCount) {
+    int gpusPerNode = ((gpuCount / nodeCount) as BigDecimal).setScale(0, BigDecimal.ROUND_CEILING).intValue()
+    return [
+        "--nodes=${nodeCount}",
+        "--ntasks=${nodeCount}",
+        "--ntasks-per-node=1",
+        "--gpus-per-node=${gpusPerNode}",
+    ]
+}
+
 def getPytestBaseCommandLine(
     String llmSrc,
     String stageName,
@@ -1671,7 +1681,8 @@ def runLLMTestlistWithSbatch(pipeline, platform, testList, config=VANILLA_CONFIG
 
                 // Generate Pytest command
                 String pytestUtil = ""
-                if (nodeCount > 1) {
+                def visualGenMultinodeSlurmMode = testList == "l0_b200_visual_gen_multi_nodes"
+                if (nodeCount > 1 && !visualGenMultinodeSlurmMode) {
                     pytestUtil = "$llmSrcNode/tensorrt_llm/llmapi/trtllm-llmapi-launch"
                 }
                 def uploadPath = "${env.JOB_NAME}/${env.BUILD_NUMBER}"
@@ -1715,7 +1726,9 @@ def runLLMTestlistWithSbatch(pipeline, platform, testList, config=VANILLA_CONFIG
                 // Generate Job Launch Script
                 def container = LLM_DOCKER_IMAGE.replace("urm.nvidia.com/", "urm.nvidia.com#")
                 def mounts = getMountListForSlurmTest(cluster, true).join(",")
-                String[] taskArgs = getNodeArgs(nodeCount, gpuCount, disaggMultiNodeMode)
+                String[] taskArgs = visualGenMultinodeSlurmMode ?
+                    getVisualGenMultinodeParentNodeArgs(nodeCount, gpuCount) :
+                    getNodeArgs(nodeCount, gpuCount, disaggMultiNodeMode)
                 if (taskArgs == null) {
                     error "Invalid Slurm test stage name is set"
                 }
@@ -1875,6 +1888,7 @@ def runLLMTestlistWithSbatch(pipeline, platform, testList, config=VANILLA_CONFIG
                     export resourcePathNode=$resourcePathNode
                     export pytestCommand="$pytestCommand"
                     export coverageConfigFile="$coverageConfigFile"
+                    export TRTLLM_VISUAL_GEN_MULTINODE_SLURM_PARENT=${visualGenMultinodeSlurmMode ? "1" : "0"}
                     export HF_TOKEN=$HF_TOKEN
                     if [ -f "${s3SecretKeyPathNode}" ]; then
                         set +x
@@ -1923,7 +1937,7 @@ def runLLMTestlistWithSbatch(pipeline, platform, testList, config=VANILLA_CONFIG
                     """
                 } else {
                     if(nodeCount > 1) {
-                        srunArgs.add("--mpi=pmix")
+                        srunArgs.add(visualGenMultinodeSlurmMode ? "--mpi=none" : "--mpi=pmix")
                     }
 
                     def scriptContent = """
@@ -5302,6 +5316,7 @@ def launchTestJobs(pipeline, testFilter)
         "DGX_B300-4_GPUs-PyTorch-Post-Merge-2": ["auto:dgx-b300-flex", "l0_dgx_b300", 2, 2, 4, 1, true],
         // VisualGen PerfSanity post-merge test
         "DGX_B200-8_GPUs-PyTorch-VisualGen-PerfSanity-Post-Merge-1": ["auto:dgx-b200-flex", "l0_b200_visual_gen_perf_sanity", 1, 1, 8, 1, true],
+        "DGX_B200-16_GPUs-2_Nodes-PyTorch-VisualGen-Post-Merge-1": ["auto:dgx-b200-flex", "l0_b200_visual_gen_multi_nodes", 1, 1, 16, 2],
         // Single-GPU Gemma4 PerfSanity regression gate and baseline
         "DGX_B200-PyTorch-PerfSanity-1": ["auto:dgx-b200-flex", "l0_b200_perf_sanity", 1, 1, 1, 1, true],
         // PerfSanity post-merge tests

jenkins/scripts/slurm_run.sh

@@ -56,6 +56,38 @@ env | sort
 
 echo "Full Command: $pytestCommand"
 
+if [[ "${TRTLLM_VISUAL_GEN_MULTINODE_SLURM_PARENT:-0}" == "1" ]]; then
+    install_done_dir="${jobWorkspace}/visual_gen_multinode_install_done"
+    mkdir -p "$install_done_dir"
+    if [[ "${SLURM_LOCALID:-0}" == "0" ]]; then
+        touch "${install_done_dir}/node_${SLURM_NODEID:-0}"
+    fi
+
+    if [[ "${SLURM_PROCID:-0}" == "0" ]]; then
+        expected_nodes="${SLURM_JOB_NUM_NODES:-${SLURM_NNODES:-1}}"
+        install_wait_timeout="${TRTLLM_VISUAL_GEN_MULTINODE_INSTALL_TIMEOUT:-3600}"
+        install_wait_deadline=$((SECONDS + install_wait_timeout))
+        while true; do
+            marker_count="$(find "$install_done_dir" -maxdepth 1 -type f -name 'node_*' -print | wc -l)"
+            if [[ "$marker_count" -ge "$expected_nodes" ]]; then
+                break
+            fi
+            if ((SECONDS >= install_wait_deadline)); then
+                echo "Timed out after ${install_wait_timeout}s waiting for VisualGen install markers (${marker_count}/${expected_nodes})."
+                echo "Markers present in $install_done_dir:"
+                find "$install_done_dir" -maxdepth 1 -type f -name 'node_*' -print | sort || true
+                exit 1
+            fi
+            echo "Waiting for VisualGen multi-node install markers in $install_done_dir (${marker_count}/${expected_nodes})"
+            sleep 10
+        done
+        pytestCommand="env -u SLURM_PROCID -u SLURM_NTASKS -u SLURM_LOCALID -u SLURM_NODEID -u SLURM_GTIDS ${pytestCommand}"
+    else
+        echo "Rank${SLURM_PROCID} finished setup; rank0 will run the VisualGen SLURM parent pytest"
+        exit 0
+    fi
+fi
+
 # For single-node test runs or disaggregated benchmark/server runs, clear all
 # environment variables related to Slurm and MPI. This prevents test processes
 # (e.g., pytest) from incorrectly initializing MPI when running under a