ADLR/megatron-lm!2448 - Updating nightly

Co-authored-by: Shanmugam Ramasamy <[email protected]>
NVIDIA · Dec 16, 2024 · de25d48 · de25d48
1 parent d4e72c0
commit de25d48
Show file tree

Hide file tree

Showing 5 changed files with 5 additions and 0 deletions.
diff --git a/.../gpt3_345m_nightly_dgx_a100_1N8G_mcore_tp2_pp2_ep2_te_4experts2parallel/model_config.yaml b/.../gpt3_345m_nightly_dgx_a100_1N8G_mcore_tp2_pp2_ep2_te_4experts2parallel/model_config.yaml
@@ -50,4 +50,5 @@ MODEL_ARGS:
   --ckpt-format: torch_dist
   --data-cache-path: ${DATA_CACHE_PATH}
   --bf16: true
+  --attention-backend: unfused
 TEST_TYPE: regular
diff --git a/...ts/test_cases/gpt/gpt3_nightly_mcore_te_tp2_pp1_modelopt_distill_resume/model_config.yaml b/...ts/test_cases/gpt/gpt3_nightly_mcore_te_tp2_pp1_modelopt_distill_resume/model_config.yaml
@@ -62,4 +62,5 @@ MODEL_ARGS:
   --ckpt-format: torch_dist
   --data-cache-path: ${DATA_CACHE_PATH}
   --bf16: true
+  --attention-backend: unfused
 TEST_TYPE: ckpt-resume
diff --git a/...ases/t5/t5_220m_nightly_dgx_a100_1N8G_mcore_te_tp1_pp1_vp1_resume_torch/model_config.yaml b/...ases/t5/t5_220m_nightly_dgx_a100_1N8G_mcore_te_tp1_pp1_vp1_resume_torch/model_config.yaml
@@ -51,4 +51,5 @@ MODEL_ARGS:
   --deterministic-mode: true
   --attention-softmax-in-fp32: true
   --ckpt-format: torch
+  --attention-backend: unfused
 TEST_TYPE: ckpt-resume
diff --git a/..._tests/test_cases/t5/t5_220m_nightly_dgx_a100_1N8G_mcore_te_tp2_pp1_vp1/model_config.yaml b/..._tests/test_cases/t5/t5_220m_nightly_dgx_a100_1N8G_mcore_te_tp2_pp1_vp1/model_config.yaml
@@ -51,4 +51,5 @@ MODEL_ARGS:
   --deterministic-mode: true
   --attention-softmax-in-fp32: true
   --ckpt-format: torch
+  --attention-backend: unfused
 TEST_TYPE: regular
diff --git a/...t5/t5_220m_nightly_dgx_a100_1N8G_mcore_te_tp2_pp1_vp1_sequence_parallel/model_config.yaml b/...t5/t5_220m_nightly_dgx_a100_1N8G_mcore_te_tp2_pp1_vp1_sequence_parallel/model_config.yaml
@@ -52,4 +52,5 @@ MODEL_ARGS:
   --deterministic-mode: true
   --attention-softmax-in-fp32: true
   --ckpt-format: torch
+  --attention-backend: unfused
 TEST_TYPE: regular