Fix FlopsProfiler crash when dp_world_size is None under sequence parallelism

deepspeed/profiling/flops_profiler/profiler.py

@@ -335,7 +335,12 @@ def print_model_profile(self, profile_step=1, module_depth=-1, top_modules=1, de
         line_fmt = '{:<70}  {:<8}'
         if self.ds_engine:
             print(line_fmt.format('world size: ', self.ds_engine.world_size))
-            print(line_fmt.format('data parallel size: ', self.ds_engine.dp_world_size))
+            # With sequence parallelism (Ulysses) the data-parallel replication is folded into the
+            # sequence-data-parallel group, so dp_world_size is None; report that group's size instead.
+            dp_world_size = self.ds_engine.dp_world_size
+            if dp_world_size is None:
+                dp_world_size = self.ds_engine.seq_dp_world_size
+            print(line_fmt.format('data parallel size: ', dp_world_size))
             print(line_fmt.format('model parallel size: ', self.ds_engine.mp_world_size))
             print(line_fmt.format('batch size per GPU: ', self.ds_engine.train_micro_batch_size_per_gpu()))
             if self.ds_engine.has_moe_layers:

tests/unit/profiling/flops_profiler/test_flops_profiler.py

@@ -3,10 +3,12 @@
 
 # DeepSpeed Team
 
+import re
 import torch
 import pytest
 import deepspeed
-from deepspeed.profiling.flops_profiler import get_model_profile
+from types import SimpleNamespace
+from deepspeed.profiling.flops_profiler import get_model_profile, FlopsProfiler
 from unit.simple_model import SimpleModel, random_dataloader
 from unit.common import DistributedTest
 from deepspeed.utils.torch import required_torch_version
@@ -119,3 +121,33 @@ def test_flops_profiler_in_inference(self):
         assert within_range(flops, 866076672, TOLERANCE)
         assert within_range(macs, 426516480, TOLERANCE)
         assert params == 61706
+
+
+def test_print_model_profile_with_none_dp_world_size(capsys):
+    # Regression test for https://github.com/deepspeedai/DeepSpeed/issues/7483
+    # Under sequence parallelism (Ulysses) the engine reports dp_world_size as None, which used to
+    # crash print_model_profile with "unsupported format string passed to NoneType.__format__".
+    model = torch.nn.Sequential(torch.nn.Linear(128, 128))
+    prof = FlopsProfiler(model)
+    # Mimic a DeepSpeed engine configured with Ulysses sequence parallelism, where dp_world_size is
+    # None and the effective data-parallel replication is the sequence-data-parallel group.
+    prof.ds_engine = SimpleNamespace(world_size=8,
+                                     dp_world_size=None,
+                                     seq_dp_world_size=4,
+                                     mp_world_size=1,
+                                     has_moe_layers=False,
+                                     train_micro_batch_size_per_gpu=lambda: 1,
+                                     wall_clock_breakdown=lambda: False)
+
+    prof.start_profile()
+    # A few forward passes give the profiled modules a non-zero measured duration.
+    for _ in range(3):
+        model(torch.randn(64, 128))
+    prof.print_model_profile(profile_step=1, detailed=False)
+    prof.end_profile()
+
+    out = capsys.readouterr().out
+    match = re.search(r"data parallel size:\s+(\S+)", out)
+    assert match is not None
+    # The sequence-data-parallel world size is reported in place of the None dp_world_size.
+    assert match.group(1) == "4"