[DRAFT - do not review] Savitha/claude lepton dev experiment

bionemo-recipes/recipes/llama3_native_te/lepton_backup/lepton_configs/og2_fp8_refactor_all_fp8_fp32mw.yaml

@@ -0,0 +1,70 @@
+# OpenGenome2 7B - FP8 Refactor Branch Test
+# Same settings as og2-7b-fp32mw-pq2-cfi-false-lf100 but:
+#   - On savitha/og2-fp8-refactor branch (FP8 logic moved into model)
+#   - FP8 enabled on ALL layers (including first/last, no BF16 override)
+#   - FP32 master weights
+#   - No CP (standard FSDP2 only)
+#
+# Data: /data/opengenome2/parquet2
+# 6 nodes H100, THD format, GQA, FP8 + FP32 master weights
+# GBS = mbs * grad_acc * dp_size = 1 * 8 * 48 = 384
+defaults:
+  - _self_
+
+job_name: "og2-7b-fp8-refactor-all-fp8-fp32mw"
+node_group: "yo-bom-lepton-001"
+resource_shape: "gpu.8xh100-sxm"
+
+num_nodes: 6
+gpus_per_node: 8
+num_train_steps: 182314
+micro_batch_size: 1
+grad_acc_steps: 8
+
+dataset_path: "/data/opengenome2/parquet2"
+data_dir: ""
+num_workers: 8
+buffer_size: 10000
+
+repo_root: "/data/savithas/bionemo-framework"
+code_path: "/data/savithas/bionemo-framework/bionemo-recipes/recipes/opengenome2_llama_native_te"
+train_script: "train_fsdp2.py"
+hydra_config: "og2_7b_thd_gqa"
+
+git_branch: "savitha/og2-fp8-refactor"
+
+validation_enabled: false
+
+spike_no_more_embedding_init: true
+skip_embedding_weight_decay: true
+use_megatron_scaled_init: true
+use_weight_decay_grouping: true
+use_meta_device: false
+
+# FP8 enabled on ALL layers (fp8_first_last_bf16 stays false in base config)
+fp8_enabled: true
+fp8_recipe: transformer_engine.common.recipe.Float8BlockScaling
+fp8_format: E4M3
+use_fp32_master_weights: true
+
+logger_frequency: 100
+
+checkpoint_dir: "/data/savithas/checkpoints/og2-7b-fp8-refactor-all-fp8-fp32mw"  # pragma: allowlist secret
+save_every_n_steps: 5000
+async_save: false
+
+wandb_project: "llama3-metagenome-7b"
+wandb_name: "og2-7b-fp8-refactor-all-fp8-fp32mw"
+wandb_secret: "wandb.savithas"  # pragma: allowlist secret
+
+hf_secret: "HUGGING_FACE_HUB_TOKEN.savithas"  # pragma: allowlist secret
+
+exclude_nodes:
+  - node-ip-10-50-80-195
+  - node-ip-10-50-81-231
+  - nvidia-lepton093
+  - nvidia-lepton007
+
+container:
+  image: "nvcr.io/nvidia/pytorch:25.11-py3"
+  registry_auth: "lepton-nvidia-cvai-bnmo-trng"