【开源实习】MiniCPM-Llama3模型迁移

models/miniCPM/_init_.py

@@ -0,0 +1,16 @@
+from .miniCPM_config import MiniCPMConfig
+from .miniCPM_model import MiniCPMModel
+from .miniCPM_tokenizer import MiniCPMTokenizer
+
+__all__ = ["MiniCPMConfig", "MiniCPMModel", "MiniCPMTokenizer"]
+
+# tests/ut/models/minicpm/test_tokenizer_minicpm.py
+import mindspore
+from mindnlp.models.miniCPM import miniCPM_model, miniCPM_config
+
+def test_minicpm_forward():
+    config = MiniCPMConfig()
+    model = MiniCPMModel(config)
+    dummy_input = mindspore.Tensor([[1, 2, 3], [4, 5, 6]], mindspore.int32)
+    output = model(dummy_input)
+    assert output.shape == (2, 3, config.hidden_size)

models/miniCPM/miniCPM_config.py

@@ -0,0 +1,49 @@
+class MiniCPMConfig:
+    model_type = "minicpm"
+
+    def __init__(
+        self,
+        hidden_size=4096,
+        intermediate_size=14336,
+        num_attention_heads=32,
+        num_hidden_layers=32,
+        num_key_value_heads=8,
+        vocab_size=128256,
+        max_position_embeddings=8192,
+        initializer_range=0.02,
+        rms_norm_eps=1e-5,
+        pad_token_id=None,
+        bos_token_id=128000,
+        eos_token_id=128001,
+        hidden_act="silu",
+        rope_theta=500000.0,
+        attention_dropout=0.0,
+        tie_word_embeddings=False,
+        use_cache=False,
+        torch_dtype="float16",
+        **kwargs,
+    ):
+        # ❌ 去掉 super().__init__()
+
+        self.pad_token_id = pad_token_id
+        self.bos_token_id = bos_token_id
+        self.eos_token_id = eos_token_id
+
+        self.hidden_size = hidden_size
+        self.intermediate_size = intermediate_size
+        self.num_attention_heads = num_attention_heads
+        self.num_hidden_layers = num_hidden_layers
+        self.num_key_value_heads = num_key_value_heads
+        self.vocab_size = vocab_size
+        self.max_position_embeddings = max_position_embeddings
+        self.initializer_range = initializer_range
+        self.rms_norm_eps = rms_norm_eps
+        self.hidden_act = hidden_act
+        self.rope_theta = rope_theta
+        self.attention_dropout = attention_dropout
+        self.tie_word_embeddings = tie_word_embeddings
+        self.use_cache = use_cache
+        self.torch_dtype = torch_dtype
+
+        for k, v in kwargs.items():
+            setattr(self, k, v)

models/miniCPM/miniCPM_model.py

@@ -0,0 +1,99 @@
+import mindspore
+import mindspore.nn as nn
+import mindspore.ops as ops
+import math
+#from miniCPM_config import MiniCPMConfig
+from .miniCPM_config import MiniCPMConfig
+
+
+
+class RMSNorm(nn.Cell):
+    def __init__(self, hidden_size, eps=1e-5):
+        super().__init__()
+        self.eps = eps
+        self.weight = mindspore.Parameter(ops.ones(hidden_size), name="rmsnorm_weight")
+
+    def construct(self, hidden_states):
+        norm = hidden_states.pow(2).mean(-1, keep_dims=True).add(self.eps).sqrt()
+        return self.weight * hidden_states / norm
+
+
+class MLP(nn.Cell):
+    def __init__(self, config: MiniCPMConfig):
+        super().__init__()
+        self.gate_proj = nn.Dense(config.hidden_size, config.intermediate_size)
+        self.up_proj = nn.Dense(config.hidden_size, config.intermediate_size)
+        self.down_proj = nn.Dense(config.intermediate_size, config.hidden_size)
+        self.act = nn.SiLU()
+
+    def construct(self, x):
+        return self.down_proj(self.act(self.gate_proj(x)) * self.up_proj(x))
+
+
+class Attention(nn.Cell):
+    def __init__(self, config: MiniCPMConfig):
+        super().__init__()
+        self.num_heads = config.num_attention_heads
+        self.head_dim = config.hidden_size // config.num_attention_heads
+        self.scale = self.head_dim ** -0.5
+
+        self.q_proj = nn.Dense(config.hidden_size, config.hidden_size)
+        self.k_proj = nn.Dense(config.hidden_size, config.hidden_size)
+        self.v_proj = nn.Dense(config.hidden_size, config.hidden_size)
+        self.out_proj = nn.Dense(config.hidden_size, config.hidden_size)
+
+        self.softmax = nn.Softmax(axis=-1)
+
+    def construct(self, x):
+        B, T, C = x.shape
+        q = self.q_proj(x).view(B, T, self.num_heads, self.head_dim).transpose(0, 2, 1, 3)
+        k = self.k_proj(x).view(B, T, self.num_heads, self.head_dim).transpose(0, 2, 1, 3)
+        v = self.v_proj(x).view(B, T, self.num_heads, self.head_dim).transpose(0, 2, 1, 3)
+
+        attn_scores = ops.matmul(q, k.transpose(0, 1, 3, 2)) * self.scale
+        attn_weights = self.softmax(attn_scores)
+        attn_output = ops.matmul(attn_weights, v)
+
+        attn_output = attn_output.transpose(0, 2, 1, 3).view(B, T, C)
+        return self.out_proj(attn_output)
+
+
+class DecoderLayer(nn.Cell):
+    def __init__(self, config: MiniCPMConfig):
+        super().__init__()
+        self.ln1 = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+        self.self_attn = Attention(config)
+        self.ln2 = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+        self.mlp = MLP(config)
+
+    def construct(self, x):
+        x = x + self.self_attn(self.ln1(x))
+        x = x + self.mlp(self.ln2(x))
+        return x
+
+
+class MiniCPMModel(nn.Cell):
+    def __init__(self, config: MiniCPMConfig):
+        super().__init__()
+        self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size)
+        self.layers = nn.CellList([DecoderLayer(config) for _ in range(config.num_hidden_layers)])
+        self.ln_f = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+
+    def construct(self, input_ids):
+        x = self.embed_tokens(input_ids)
+        for layer in self.layers:
+            x = layer(x)
+        x = self.ln_f(x)
+        return x
+
+
+class MiniCPMForCausalLM(nn.Cell):
+    def __init__(self, config: MiniCPMConfig):
+        super().__init__()
+        self.model = MiniCPMModel(config)
+        self.lm_head = nn.Dense(config.hidden_size, config.vocab_size, has_bias=False)
+
+    def construct(self, input_ids):
+        hidden_states = self.model(input_ids)
+        logits = self.lm_head(hidden_states)
+        return logits

models/miniCPM/miniCPM_tokenizer.py

@@ -0,0 +1,36 @@
+from tokenizers import Tokenizer
+import os
+
+class MiniCPMTokenizer:
+    def __init__(self, tokenizer_file: str):
+        self.tokenizer_file = tokenizer_file
+        self.tokenizer = Tokenizer.from_file(tokenizer_file)
+
+    def tokenize(self, text):
+        return self.tokenizer.encode(text).tokens
+
+    def convert_tokens_to_ids(self, tokens):
+        return [self.tokenizer.token_to_id(tok) for tok in tokens]
+
+    def convert_ids_to_tokens(self, ids):
+        return [self.tokenizer.id_to_token(i) for i in ids]
+
+    def encode(self, text):
+        return self.tokenizer.encode(text).ids
+
+    def decode(self, token_ids):
+        return self.tokenizer.decode(token_ids)
+
+if __name__ == "__main__":
+    tokenizer_path = "D:/个人/MiniCPM_Llama3/minicpm_assets/tokenizer.json"
+    tokenizer = MiniCPMTokenizer(tokenizer_path)
+
+    text = "你好 MiniCPM"
+    tokens = tokenizer.tokenize(text)
+    ids = tokenizer.convert_tokens_to_ids(tokens)
+    decoded = tokenizer.decode(ids)
+
+    print("原文：", text)
+    print("分词：", tokens)
+    print("Token IDs：", ids)
+    print("解码：", decoded)