Training in progress, step 100

README.md

@@ -1,17 +1,17 @@
 ---
-base_model: google/gemma-3-4b-it
+base_model: google/gemma-3-1b-it
 library_name: transformers
 model_name: nmt_21
 tags:
 - generated_from_trainer
-- trl
 - grpo
+- trl
 licence: license
 ---
 
 # Model Card for nmt_21
 
-This model is a fine-tuned version of [google/gemma-3-4b-it](https://huggingface.co/google/gemma-3-4b-it).
+This model is a fine-tuned version of [google/gemma-3-1b-it](https://huggingface.co/google/gemma-3-1b-it).
 It has been trained using [TRL](https://github.com/huggingface/trl).
 
 ## Quick start
@@ -27,14 +27,14 @@ print(output["generated_text"])
 
 ## Training procedure
 
-[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/oleg-dats/nmt/runs/syf13vuv) 
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/oleg-dats/nmt/runs/f4rb3j16) 
 
 
 This model was trained with GRPO, a method introduced in [DeepSeekMath: Pushing the Limits of Mathematical Reasoning in Open Language Models](https://huggingface.co/papers/2402.03300).
 
 ### Framework versions
 
-- TRL: 0.23.1
+- TRL: 0.23.0
 - Transformers: 4.56.2
 - Pytorch: 2.5.1+cu124
 - Datasets: 4.1.1

config.json

@@ -1,94 +1,62 @@
 {
+  "_sliding_window_pattern": 6,
   "architectures": [
-    "Gemma3ForConditionalGeneration"
+    "Gemma3ForCausalLM"
   ],
-  "boi_token_index": 255999,
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "attn_logit_softcapping": null,
   "bos_token_id": 2,
+  "cache_implementation": "hybrid",
   "dtype": "bfloat16",
-  "eoi_token_index": 256000,
   "eos_token_id": 1,
-  "image_token_index": 262144,
+  "final_logit_softcapping": null,
+  "head_dim": 256,
+  "hidden_activation": "gelu_pytorch_tanh",
+  "hidden_size": 1152,
   "initializer_range": 0.02,
-  "mm_tokens_per_image": 256,
-  "model_type": "gemma3",
+  "intermediate_size": 6912,
+  "layer_types": [
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention"
+  ],
+  "max_position_embeddings": 32768,
+  "model_type": "gemma3_text",
+  "num_attention_heads": 4,
+  "num_hidden_layers": 26,
+  "num_key_value_heads": 1,
   "pad_token_id": 0,
-  "text_config": {
-    "_sliding_window_pattern": 6,
-    "attention_bias": false,
-    "attention_dropout": 0.0,
-    "attn_logit_softcapping": null,
-    "final_logit_softcapping": null,
-    "head_dim": 256,
-    "hidden_activation": "gelu_pytorch_tanh",
-    "hidden_size": 2560,
-    "initializer_range": 0.02,
-    "intermediate_size": 10240,
-    "layer_types": [
-      "sliding_attention",
-      "sliding_attention",
-      "sliding_attention",
-      "sliding_attention",
-      "sliding_attention",
-      "full_attention",
-      "sliding_attention",
-      "sliding_attention",
-      "sliding_attention",
-      "sliding_attention",
-      "sliding_attention",
-      "full_attention",
-      "sliding_attention",
-      "sliding_attention",
-      "sliding_attention",
-      "sliding_attention",
-      "sliding_attention",
-      "full_attention",
-      "sliding_attention",
-      "sliding_attention",
-      "sliding_attention",
-      "sliding_attention",
-      "sliding_attention",
-      "full_attention",
-      "sliding_attention",
-      "sliding_attention",
-      "sliding_attention",
-      "sliding_attention",
-      "sliding_attention",
-      "full_attention",
-      "sliding_attention",
-      "sliding_attention",
-      "sliding_attention",
-      "sliding_attention"
-    ],
-    "max_position_embeddings": 131072,
-    "model_type": "gemma3_text",
-    "num_attention_heads": 8,
-    "num_hidden_layers": 34,
-    "num_key_value_heads": 4,
-    "query_pre_attn_scalar": 256,
-    "rms_norm_eps": 1e-06,
-    "rope_local_base_freq": 10000.0,
-    "rope_scaling": {
-      "factor": 8.0,
-      "rope_type": "linear"
-    },
-    "rope_theta": 1000000.0,
-    "sliding_window": 1024,
-    "use_cache": true,
-    "vocab_size": 262208
-  },
+  "query_pre_attn_scalar": 256,
+  "rms_norm_eps": 1e-06,
+  "rope_local_base_freq": 10000,
+  "rope_scaling": null,
+  "rope_theta": 1000000,
+  "sliding_window": 512,
   "transformers_version": "4.56.2",
-  "vision_config": {
-    "attention_dropout": 0.0,
-    "hidden_act": "gelu_pytorch_tanh",
-    "hidden_size": 1152,
-    "image_size": 896,
-    "intermediate_size": 4304,
-    "layer_norm_eps": 1e-06,
-    "model_type": "siglip_vision_model",
-    "num_attention_heads": 16,
-    "num_channels": 3,
-    "num_hidden_layers": 27,
-    "patch_size": 14,
-    "vision_use_head": false
-  }
+  "use_cache": true,
+  "vocab_size": 262144
 }

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f5b69b8ecb5e61e6e3bf4f72db44e88c98f557be6a92255fddb887b7f0d4f97f
+size 1999811208

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1962c6464b1c9286492998d5209bebd4b71eaeb55518cd47e972fb26d1807131
+oid sha256:32d82cec56e20a42331cadab044e78a230fceda57b3097692eaf8e50ff1609a0
 size 6776