diff --git a/docker/patch/latest/megatron.patch b/docker/patch/latest/megatron.patch index 4b4c74d28e..b980cdc5bd 100644 --- a/docker/patch/latest/megatron.patch +++ b/docker/patch/latest/megatron.patch @@ -862,10 +862,13 @@ diff --git a/megatron/core/transformer/transformer_config.py b/megatron/core/tra index dce438520..de51edaf3 100644 --- a/megatron/core/transformer/transformer_config.py +++ b/megatron/core/transformer/transformer_config.py -@@ -229,6 +229,9 @@ class TransformerConfig(ModelParallelConfig): +@@ -229,6 +229,12 @@ class TransformerConfig(ModelParallelConfig): attention_output_gate: bool = False """Whether to apply output gate to the attention layers.""" ++ rotary_percent: Optional[float] = None ++ """Percent of rotary dimension to use for rotary position embeddings.""" ++ + post_self_attn_layernorm: bool = False + post_mlp_layernorm: bool = False + diff --git a/docker/patch/v0.5.0rc0-cu126/megatron.patch b/docker/patch/v0.5.0rc0-cu126/megatron.patch index 95d59d284a..dc01fa5d38 100644 --- a/docker/patch/v0.5.0rc0-cu126/megatron.patch +++ b/docker/patch/v0.5.0rc0-cu126/megatron.patch @@ -329,10 +329,13 @@ diff --git a/megatron/core/transformer/transformer_config.py b/megatron/core/tra index 6f557e1f..b295fd35 100644 --- a/megatron/core/transformer/transformer_config.py +++ b/megatron/core/transformer/transformer_config.py -@@ -173,6 +173,9 @@ class TransformerConfig(ModelParallelConfig): +@@ -173,6 +173,12 @@ class TransformerConfig(ModelParallelConfig): qk_layernorm: bool = False """Whether to apply `normalization` type of normalization to the query and key embeddings.""" ++ rotary_percent: Optional[float] = None ++ """Percent of rotary dimension to use for rotary position embeddings.""" ++ + post_self_attn_layernorm: bool = False + post_mlp_layernorm: bool = False + diff --git a/docker/patch/v0.5.12.post1/megatron.patch b/docker/patch/v0.5.12.post1/megatron.patch index 4b4c74d28e..b980cdc5bd 100644 --- a/docker/patch/v0.5.12.post1/megatron.patch +++ b/docker/patch/v0.5.12.post1/megatron.patch @@ -862,10 +862,13 @@ diff --git a/megatron/core/transformer/transformer_config.py b/megatron/core/tra index dce438520..de51edaf3 100644 --- a/megatron/core/transformer/transformer_config.py +++ b/megatron/core/transformer/transformer_config.py -@@ -229,6 +229,9 @@ class TransformerConfig(ModelParallelConfig): +@@ -229,6 +229,12 @@ class TransformerConfig(ModelParallelConfig): attention_output_gate: bool = False """Whether to apply output gate to the attention layers.""" ++ rotary_percent: Optional[float] = None ++ """Percent of rotary dimension to use for rotary position embeddings.""" ++ + post_self_attn_layernorm: bool = False + post_mlp_layernorm: bool = False + diff --git a/docker/patch/v0.5.5.post1/megatron.patch b/docker/patch/v0.5.5.post1/megatron.patch index cca054d1e6..66152a747b 100644 --- a/docker/patch/v0.5.5.post1/megatron.patch +++ b/docker/patch/v0.5.5.post1/megatron.patch @@ -723,10 +723,13 @@ diff --git a/megatron/core/transformer/transformer_config.py b/megatron/core/tra index d55bebe7e..1eecbbd38 100644 --- a/megatron/core/transformer/transformer_config.py +++ b/megatron/core/transformer/transformer_config.py -@@ -173,6 +173,10 @@ class TransformerConfig(ModelParallelConfig): +@@ -173,6 +173,13 @@ class TransformerConfig(ModelParallelConfig): qk_layernorm: bool = False """Whether to apply `normalization` type of normalization to the query and key embeddings.""" ++ rotary_percent: Optional[float] = None ++ """Percent of rotary dimension to use for rotary position embeddings.""" ++ + post_self_attn_layernorm: bool = False + post_mlp_layernorm: bool = False + use_gated_attention: bool = False diff --git a/docker/patch/v0.5.6/megatron.patch b/docker/patch/v0.5.6/megatron.patch index 9d0d6011ce..07360f9d72 100644 --- a/docker/patch/v0.5.6/megatron.patch +++ b/docker/patch/v0.5.6/megatron.patch @@ -736,10 +736,13 @@ diff --git a/megatron/core/transformer/transformer_config.py b/megatron/core/tra index d55bebe7e..1eecbbd38 100644 --- a/megatron/core/transformer/transformer_config.py +++ b/megatron/core/transformer/transformer_config.py -@@ -173,6 +173,10 @@ class TransformerConfig(ModelParallelConfig): +@@ -173,6 +173,13 @@ class TransformerConfig(ModelParallelConfig): qk_layernorm: bool = False """Whether to apply `normalization` type of normalization to the query and key embeddings.""" ++ rotary_percent: Optional[float] = None ++ """Percent of rotary dimension to use for rotary position embeddings.""" ++ + post_self_attn_layernorm: bool = False + post_mlp_layernorm: bool = False + use_gated_attention: bool = False diff --git a/docker/patch/v0.5.7/megatron.patch b/docker/patch/v0.5.7/megatron.patch index 6d2a233949..f347dec12f 100644 --- a/docker/patch/v0.5.7/megatron.patch +++ b/docker/patch/v0.5.7/megatron.patch @@ -650,10 +650,13 @@ diff --git a/megatron/core/transformer/transformer_config.py b/megatron/core/tra index e2705bd9f..a0aa109b5 100644 --- a/megatron/core/transformer/transformer_config.py +++ b/megatron/core/transformer/transformer_config.py -@@ -210,6 +210,9 @@ class TransformerConfig(ModelParallelConfig): +@@ -210,6 +210,12 @@ class TransformerConfig(ModelParallelConfig): attention_output_gate: bool = False """Whether to apply output gate to the attention layers.""" ++ rotary_percent: Optional[float] = None ++ """Percent of rotary dimension to use for rotary position embeddings.""" ++ + post_self_attn_layernorm: bool = False + post_mlp_layernorm: bool = False + diff --git a/docker/patch/v0.5.9/megatron.patch b/docker/patch/v0.5.9/megatron.patch index 2e6ae436b1..11de8adde9 100644 --- a/docker/patch/v0.5.9/megatron.patch +++ b/docker/patch/v0.5.9/megatron.patch @@ -673,10 +673,13 @@ diff --git a/megatron/core/transformer/transformer_config.py b/megatron/core/tra index e2705bd9f..a0aa109b5 100644 --- a/megatron/core/transformer/transformer_config.py +++ b/megatron/core/transformer/transformer_config.py -@@ -210,6 +210,9 @@ class TransformerConfig(ModelParallelConfig): +@@ -210,6 +210,12 @@ class TransformerConfig(ModelParallelConfig): attention_output_gate: bool = False """Whether to apply output gate to the attention layers.""" ++ rotary_percent: Optional[float] = None ++ """Percent of rotary dimension to use for rotary position embeddings.""" ++ + post_self_attn_layernorm: bool = False + post_mlp_layernorm: bool = False +