init

config.json

@@ -36,5 +36,6 @@
   "rope_theta": 10000.0,
   "scale_emb": 12,
   "scale_depth": 1.4,
+  "mup_denominator": 32,
   "tie_word_embeddings": false
 }

configuration_minicpm.py

@@ -137,6 +137,7 @@ class MiniCPMConfig(PretrainedConfig):
         scale_emb=1,
         dim_model_base=1,
         scale_depth=1,
+        mup_denominator=32,
         sparse_config=None,
         **kwargs):
 
@@ -165,6 +166,8 @@ class MiniCPMConfig(PretrainedConfig):
         self.scale_emb = scale_emb
         self.dim_model_base = dim_model_base
         self.scale_depth = scale_depth
+        # only used for Eagle Head
+        self.mup_denominator = mup_denominator
 
         # sparse config
         self.sparse_config = sparse_config