Upload folder using huggingface_hub

checkpoint-15260/config.json

@@ -0,0 +1,29 @@
+{
+  "_name_or_path": "/mnt/parscratch/users/acp23ay/private/models/Qwen2.5-7B-my-madlad-mean/",
+  "architectures": [
+    "Qwen2ForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 151643,
+  "eos_token_id": 151643,
+  "hidden_act": "silu",
+  "hidden_size": 3584,
+  "initializer_range": 0.02,
+  "intermediate_size": 18944,
+  "max_position_embeddings": 131072,
+  "max_window_layers": 28,
+  "model_type": "qwen2",
+  "num_attention_heads": 28,
+  "num_hidden_layers": 28,
+  "num_key_value_heads": 4,
+  "rms_norm_eps": 1e-06,
+  "rope_theta": 1000000.0,
+  "sliding_window": null,
+  "tie_word_embeddings": false,
+  "torch_dtype": "float32",
+  "transformers_version": "4.44.0.dev0",
+  "use_cache": true,
+  "use_mrope": false,
+  "use_sliding_window": false,
+  "vocab_size": 161672
+}

checkpoint-15260/generation_config.json

@@ -0,0 +1,6 @@
+{
+  "bos_token_id": 151643,
+  "eos_token_id": 151643,
+  "max_new_tokens": 2048,
+  "transformers_version": "4.44.0.dev0"
+}

checkpoint-15260/model-00001-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0c15a4b889e2722f0603078f254304ffde6314a0c9e35dff84648fccd0ddf64c
+size 4842817304

checkpoint-15260/model-00002-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:855851c22a0213c322c44b2d933eed67bc9ff8215091c3eb491ec94b662ffadd
+size 4998852224

checkpoint-15260/model-00003-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0c748c56f7d09ef7934bc2e6d2bbb3a6b4fec8beeb1759acb8db07b32764bd38
+size 4984124304

checkpoint-15260/model-00004-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:089cf99962fd351ceb590220ea351f1d18937b35376077982b276416f768317a
+size 4932743992

checkpoint-15260/model-00005-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b61bee4a8b5e41f744ec0820def58c39c0c333e6787b7e24146468348f082a0
+size 4998852296

checkpoint-15260/model-00006-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0e622c2f51f1c6575eeef38241d5c5fbdf8d1ffb9877ac7fe78ba280afd7f284
+size 3662865184

checkpoint-15260/model-00007-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6c35201231aaec2291a6b8b1e07376faefb6209d61a2c876e22a2f6421750aec
+size 2317729920

checkpoint-15260/model.safetensors.index.json

@@ -0,0 +1,346 @@
+{
+  "metadata": {
+    "total_size": 30737946624
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00007-of-00007.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.self_attn.k_proj.bias": "model-00001-of-00007.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.self_attn.q_proj.bias": "model-00001-of-00007.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.self_attn.v_proj.bias": "model-00001-of-00007.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.self_attn.k_proj.bias": "model-00001-of-00007.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.self_attn.q_proj.bias": "model-00001-of-00007.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.self_attn.v_proj.bias": "model-00001-of-00007.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.self_attn.k_proj.bias": "model-00003-of-00007.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.self_attn.q_proj.bias": "model-00003-of-00007.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.self_attn.v_proj.bias": "model-00003-of-00007.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.self_attn.k_proj.bias": "model-00003-of-00007.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.self_attn.q_proj.bias": "model-00003-of-00007.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.self_attn.v_proj.bias": "model-00003-of-00007.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.self_attn.k_proj.bias": "model-00003-of-00007.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.self_attn.q_proj.bias": "model-00003-of-00007.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.self_attn.v_proj.bias": "model-00003-of-00007.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.13.self_attn.k_proj.bias": "model-00003-of-00007.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.self_attn.q_proj.bias": "model-00003-of-00007.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.self_attn.v_proj.bias": "model-00003-of-00007.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.self_attn.k_proj.bias": "model-00004-of-00007.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.self_attn.q_proj.bias": "model-00004-of-00007.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.self_attn.v_proj.bias": "model-00004-of-00007.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.self_attn.k_proj.bias": "model-00004-of-00007.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.self_attn.q_proj.bias": "model-00004-of-00007.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.self_attn.v_proj.bias": "model-00004-of-00007.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.self_attn.k_proj.bias": "model-00004-of-00007.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.self_attn.q_proj.bias": "model-00004-of-00007.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.self_attn.v_proj.bias": "model-00004-of-00007.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.self_attn.k_proj.bias": "model-00004-of-00007.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.self_attn.q_proj.bias": "model-00004-of-00007.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.self_attn.v_proj.bias": "model-00004-of-00007.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.18.self_attn.k_proj.bias": "model-00004-of-00007.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.self_attn.q_proj.bias": "model-00004-of-00007.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.self_attn.v_proj.bias": "model-00004-of-00007.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.19.self_attn.k_proj.bias": "model-00005-of-00007.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.19.self_attn.q_proj.bias": "model-00005-of-00007.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.19.self_attn.v_proj.bias": "model-00005-of-00007.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.2.self_attn.k_proj.bias": "model-00001-of-00007.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.self_attn.q_proj.bias": "model-00001-of-00007.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.self_attn.v_proj.bias": "model-00001-of-00007.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.20.self_attn.k_proj.bias": "model-00005-of-00007.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.20.self_attn.q_proj.bias": "model-00005-of-00007.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.20.self_attn.v_proj.bias": "model-00005-of-00007.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.self_attn.k_proj.bias": "model-00005-of-00007.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.self_attn.q_proj.bias": "model-00005-of-00007.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.self_attn.v_proj.bias": "model-00005-of-00007.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.self_attn.k_proj.bias": "model-00005-of-00007.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.self_attn.q_proj.bias": "model-00005-of-00007.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.self_attn.v_proj.bias": "model-00005-of-00007.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.self_attn.k_proj.bias": "model-00005-of-00007.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.self_attn.q_proj.bias": "model-00005-of-00007.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.self_attn.v_proj.bias": "model-00005-of-00007.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.24.self_attn.k_proj.bias": "model-00005-of-00007.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.24.self_attn.q_proj.bias": "model-00005-of-00007.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.self_attn.v_proj.bias": "model-00005-of-00007.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.25.self_attn.k_proj.bias": "model-00006-of-00007.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.25.self_attn.q_proj.bias": "model-00006-of-00007.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.25.self_attn.v_proj.bias": "model-00006-of-00007.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.self_attn.k_proj.bias": "model-00006-of-00007.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.self_attn.q_proj.bias": "model-00006-of-00007.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.self_attn.v_proj.bias": "model-00006-of-00007.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.self_attn.k_proj.bias": "model-00006-of-00007.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.self_attn.q_proj.bias": "model-00006-of-00007.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.self_attn.v_proj.bias": "model-00006-of-00007.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.self_attn.k_proj.bias": "model-00002-of-00007.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.self_attn.q_proj.bias": "model-00002-of-00007.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.self_attn.v_proj.bias": "model-00002-of-00007.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.self_attn.k_proj.bias": "model-00002-of-00007.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.self_attn.q_proj.bias": "model-00002-of-00007.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.self_attn.v_proj.bias": "model-00002-of-00007.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.self_attn.k_proj.bias": "model-00002-of-00007.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.self_attn.q_proj.bias": "model-00002-of-00007.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.self_attn.v_proj.bias": "model-00002-of-00007.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.self_attn.k_proj.bias": "model-00002-of-00007.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.self_attn.q_proj.bias": "model-00002-of-00007.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.self_attn.v_proj.bias": "model-00002-of-00007.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.self_attn.k_proj.bias": "model-00002-of-00007.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.self_attn.q_proj.bias": "model-00002-of-00007.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.self_attn.v_proj.bias": "model-00002-of-00007.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.8.self_attn.k_proj.bias": "model-00002-of-00007.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.8.self_attn.q_proj.bias": "model-00002-of-00007.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.8.self_attn.v_proj.bias": "model-00002-of-00007.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.self_attn.k_proj.bias": "model-00003-of-00007.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.self_attn.q_proj.bias": "model-00003-of-00007.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.self_attn.v_proj.bias": "model-00003-of-00007.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "model.norm.weight": "model-00006-of-00007.safetensors"
+  }
+}

checkpoint-15260/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:22bc8710ea63e50e6d08a18986ff1d278ba3f50f4a8a5d39d320a6d496b700c9
+size 16728842070

checkpoint-15260/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9d9cd6a0487226e5bd30d1846894c82af483733ab4381b75bae9c0745e05d405
+size 14244

checkpoint-15260/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2ce335347efaebf038b8915bc0a8d2d587a4a1aa08ad0e015b4bc7cc4fba634e
+size 1064

checkpoint-15260/trainer_state.json

@@ -0,0 +1,3477 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.38230283595550657,
+  "eval_steps": 500,
+  "global_step": 15260,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.000776630924942379,
+      "grad_norm": 19.31844711303711,
+      "learning_rate": 1.0157273918741808e-06,
+      "loss": 8.0914,
+      "step": 31
+    },
+    {
+      "epoch": 0.001553261849884758,
+      "grad_norm": 15.708772659301758,
+      "learning_rate": 2.0314547837483616e-06,
+      "loss": 7.0861,
+      "step": 62
+    },
+    {
+      "epoch": 0.002329892774827137,
+      "grad_norm": 10.70508861541748,
+      "learning_rate": 3.0471821756225426e-06,
+      "loss": 5.7478,
+      "step": 93
+    },
+    {
+      "epoch": 0.003106523699769516,
+      "grad_norm": 4.4315900802612305,
+      "learning_rate": 4.062909567496723e-06,
+      "loss": 4.763,
+      "step": 124
+    },
+    {
+      "epoch": 0.003883154624711895,
+      "grad_norm": 5.228023052215576,
+      "learning_rate": 5.078636959370905e-06,
+      "loss": 4.1438,
+      "step": 155
+    },
+    {
+      "epoch": 0.004659785549654274,
+      "grad_norm": 4.417849540710449,
+      "learning_rate": 6.094364351245085e-06,
+      "loss": 3.8013,
+      "step": 186
+    },
+    {
+      "epoch": 0.005436416474596653,
+      "grad_norm": 6.3335280418396,
+      "learning_rate": 7.110091743119267e-06,
+      "loss": 3.6086,
+      "step": 217
+    },
+    {
+      "epoch": 0.006213047399539032,
+      "grad_norm": 6.3932647705078125,
+      "learning_rate": 8.125819134993446e-06,
+      "loss": 3.4676,
+      "step": 248
+    },
+    {
+      "epoch": 0.006989678324481411,
+      "grad_norm": 6.310016632080078,
+      "learning_rate": 9.141546526867629e-06,
+      "loss": 3.3426,
+      "step": 279
+    },
+    {
+      "epoch": 0.00776630924942379,
+      "grad_norm": 4.848681449890137,
+      "learning_rate": 1.015727391874181e-05,
+      "loss": 3.2305,
+      "step": 310
+    },
+    {
+      "epoch": 0.00854294017436617,
+      "grad_norm": 7.16106653213501,
+      "learning_rate": 1.117300131061599e-05,
+      "loss": 3.1615,
+      "step": 341
+    },
+    {
+      "epoch": 0.009319571099308548,
+      "grad_norm": 4.73142671585083,
+      "learning_rate": 1.218872870249017e-05,
+      "loss": 3.0865,
+      "step": 372
+    },
+    {
+      "epoch": 0.010096202024250926,
+      "grad_norm": 4.504674911499023,
+      "learning_rate": 1.3204456094364351e-05,
+      "loss": 3.035,
+      "step": 403
+    },
+    {
+      "epoch": 0.010872832949193306,
+      "grad_norm": 4.620588779449463,
+      "learning_rate": 1.4220183486238533e-05,
+      "loss": 2.932,
+      "step": 434
+    },
+    {
+      "epoch": 0.011649463874135685,
+      "grad_norm": 4.8418965339660645,
+      "learning_rate": 1.5235910878112714e-05,
+      "loss": 2.8965,
+      "step": 465
+    },
+    {
+      "epoch": 0.012426094799078065,
+      "grad_norm": 4.36681604385376,
+      "learning_rate": 1.6251638269986893e-05,
+      "loss": 2.8569,
+      "step": 496
+    },
+    {
+      "epoch": 0.013202725724020443,
+      "grad_norm": 4.929416179656982,
+      "learning_rate": 1.7267365661861077e-05,
+      "loss": 2.8155,
+      "step": 527
+    },
+    {
+      "epoch": 0.013979356648962823,
+      "grad_norm": 4.269941806793213,
+      "learning_rate": 1.8283093053735257e-05,
+      "loss": 2.7682,
+      "step": 558
+    },
+    {
+      "epoch": 0.014755987573905201,
+      "grad_norm": 4.4052534103393555,
+      "learning_rate": 1.9298820445609438e-05,
+      "loss": 2.7076,
+      "step": 589
+    },
+    {
+      "epoch": 0.01553261849884758,
+      "grad_norm": 4.1759843826293945,
+      "learning_rate": 2.031454783748362e-05,
+      "loss": 2.7143,
+      "step": 620
+    },
+    {
+      "epoch": 0.01630924942378996,
+      "grad_norm": 3.320908546447754,
+      "learning_rate": 2.13302752293578e-05,
+      "loss": 2.6696,
+      "step": 651
+    },
+    {
+      "epoch": 0.01708588034873234,
+      "grad_norm": 3.201021194458008,
+      "learning_rate": 2.234600262123198e-05,
+      "loss": 2.6321,
+      "step": 682
+    },
+    {
+      "epoch": 0.017862511273674716,
+      "grad_norm": 3.675868272781372,
+      "learning_rate": 2.336173001310616e-05,
+      "loss": 2.6029,
+      "step": 713
+    },
+    {
+      "epoch": 0.018639142198617096,
+      "grad_norm": 3.0218663215637207,
+      "learning_rate": 2.437745740498034e-05,
+      "loss": 2.5883,
+      "step": 744
+    },
+    {
+      "epoch": 0.019415773123559476,
+      "grad_norm": 3.260049343109131,
+      "learning_rate": 2.5393184796854525e-05,
+      "loss": 2.5625,
+      "step": 775
+    },
+    {
+      "epoch": 0.020192404048501852,
+      "grad_norm": 3.212430000305176,
+      "learning_rate": 2.6408912188728702e-05,
+      "loss": 2.5631,
+      "step": 806
+    },
+    {
+      "epoch": 0.020969034973444232,
+      "grad_norm": 2.770608901977539,
+      "learning_rate": 2.7424639580602886e-05,
+      "loss": 2.5256,
+      "step": 837
+    },
+    {
+      "epoch": 0.021745665898386612,
+      "grad_norm": 2.8963048458099365,
+      "learning_rate": 2.8440366972477066e-05,
+      "loss": 2.5202,
+      "step": 868
+    },
+    {
+      "epoch": 0.022522296823328992,
+      "grad_norm": 2.7916104793548584,
+      "learning_rate": 2.9456094364351244e-05,
+      "loss": 2.476,
+      "step": 899
+    },
+    {
+      "epoch": 0.02329892774827137,
+      "grad_norm": 2.6866674423217773,
+      "learning_rate": 3.0471821756225428e-05,
+      "loss": 2.4706,
+      "step": 930
+    },
+    {
+      "epoch": 0.02407555867321375,
+      "grad_norm": 2.606696605682373,
+      "learning_rate": 3.148754914809961e-05,
+      "loss": 2.4627,
+      "step": 961
+    },
+    {
+      "epoch": 0.02485218959815613,
+      "grad_norm": 2.7148895263671875,
+      "learning_rate": 3.2503276539973785e-05,
+      "loss": 2.4254,
+      "step": 992
+    },
+    {
+      "epoch": 0.025628820523098506,
+      "grad_norm": 2.5046658515930176,
+      "learning_rate": 3.351900393184797e-05,
+      "loss": 2.4078,
+      "step": 1023
+    },
+    {
+      "epoch": 0.026405451448040886,
+      "grad_norm": 2.4350714683532715,
+      "learning_rate": 3.453473132372215e-05,
+      "loss": 2.3691,
+      "step": 1054
+    },
+    {
+      "epoch": 0.027182082372983266,
+      "grad_norm": 2.339318037033081,
+      "learning_rate": 3.555045871559633e-05,
+      "loss": 2.3795,
+      "step": 1085
+    },
+    {
+      "epoch": 0.027958713297925646,
+      "grad_norm": 3.8952598571777344,
+      "learning_rate": 3.6566186107470514e-05,
+      "loss": 2.3635,
+      "step": 1116
+    },
+    {
+      "epoch": 0.028735344222868022,
+      "grad_norm": 2.1903908252716064,
+      "learning_rate": 3.7581913499344695e-05,
+      "loss": 2.3589,
+      "step": 1147
+    },
+    {
+      "epoch": 0.029511975147810402,
+      "grad_norm": 5.00233793258667,
+      "learning_rate": 3.8597640891218876e-05,
+      "loss": 2.3063,
+      "step": 1178
+    },
+    {
+      "epoch": 0.030288606072752782,
+      "grad_norm": 4.72776460647583,
+      "learning_rate": 3.9613368283093056e-05,
+      "loss": 2.3281,
+      "step": 1209
+    },
+    {
+      "epoch": 0.03106523699769516,
+      "grad_norm": 2.184804916381836,
+      "learning_rate": 4.062909567496724e-05,
+      "loss": 2.2672,
+      "step": 1240
+    },
+    {
+      "epoch": 0.03184186792263754,
+      "grad_norm": 2.3042352199554443,
+      "learning_rate": 4.164482306684142e-05,
+      "loss": 2.2846,
+      "step": 1271
+    },
+    {
+      "epoch": 0.03261849884757992,
+      "grad_norm": 2.271709442138672,
+      "learning_rate": 4.26605504587156e-05,
+      "loss": 2.2655,
+      "step": 1302
+    },
+    {
+      "epoch": 0.0333951297725223,
+      "grad_norm": 2.3952088356018066,
+      "learning_rate": 4.367627785058978e-05,
+      "loss": 2.2523,
+      "step": 1333
+    },
+    {
+      "epoch": 0.03417176069746468,
+      "grad_norm": 2.034968614578247,
+      "learning_rate": 4.469200524246396e-05,
+      "loss": 2.2543,
+      "step": 1364
+    },
+    {
+      "epoch": 0.03494839162240705,
+      "grad_norm": 2.070462942123413,
+      "learning_rate": 4.570773263433814e-05,
+      "loss": 2.233,
+      "step": 1395
+    },
+    {
+      "epoch": 0.03572502254734943,
+      "grad_norm": 2.1028084754943848,
+      "learning_rate": 4.672346002621232e-05,
+      "loss": 2.2411,
+      "step": 1426
+    },
+    {
+      "epoch": 0.03650165347229181,
+      "grad_norm": 1.983056902885437,
+      "learning_rate": 4.77391874180865e-05,
+      "loss": 2.2273,
+      "step": 1457
+    },
+    {
+      "epoch": 0.03727828439723419,
+      "grad_norm": 1.9587653875350952,
+      "learning_rate": 4.875491480996068e-05,
+      "loss": 2.2128,
+      "step": 1488
+    },
+    {
+      "epoch": 0.03805491532217657,
+      "grad_norm": 1.8616324663162231,
+      "learning_rate": 4.977064220183487e-05,
+      "loss": 2.2045,
+      "step": 1519
+    },
+    {
+      "epoch": 0.03883154624711895,
+      "grad_norm": 2.7953391075134277,
+      "learning_rate": 4.9999915451558777e-05,
+      "loss": 2.1679,
+      "step": 1550
+    },
+    {
+      "epoch": 0.03960817717206133,
+      "grad_norm": 1.8601350784301758,
+      "learning_rate": 4.999955597496219e-05,
+      "loss": 2.1936,
+      "step": 1581
+    },
+    {
+      "epoch": 0.040384808097003705,
+      "grad_norm": 1.7780190706253052,
+      "learning_rate": 4.9998914381774255e-05,
+      "loss": 2.1935,
+      "step": 1612
+    },
+    {
+      "epoch": 0.041161439021946085,
+      "grad_norm": 1.6555310487747192,
+      "learning_rate": 4.999799067923527e-05,
+      "loss": 2.1441,
+      "step": 1643
+    },
+    {
+      "epoch": 0.041938069946888465,
+      "grad_norm": 3.3240256309509277,
+      "learning_rate": 4.999678487776908e-05,
+      "loss": 2.1462,
+      "step": 1674
+    },
+    {
+      "epoch": 0.042714700871830845,
+      "grad_norm": 1.7667723894119263,
+      "learning_rate": 4.9995296990983006e-05,
+      "loss": 2.1364,
+      "step": 1705
+    },
+    {
+      "epoch": 0.043491331796773225,
+      "grad_norm": 2.1140077114105225,
+      "learning_rate": 4.999352703566763e-05,
+      "loss": 2.1486,
+      "step": 1736
+    },
+    {
+      "epoch": 0.044267962721715605,
+      "grad_norm": 1.6703864336013794,
+      "learning_rate": 4.999147503179668e-05,
+      "loss": 2.1454,
+      "step": 1767
+    },
+    {
+      "epoch": 0.045044593646657985,
+      "grad_norm": 1.7508599758148193,
+      "learning_rate": 4.998914100252672e-05,
+      "loss": 2.132,
+      "step": 1798
+    },
+    {
+      "epoch": 0.04582122457160036,
+      "grad_norm": 1.6556063890457153,
+      "learning_rate": 4.998652497419696e-05,
+      "loss": 2.1009,
+      "step": 1829
+    },
+    {
+      "epoch": 0.04659785549654274,
+      "grad_norm": 2.0815255641937256,
+      "learning_rate": 4.9983626976328927e-05,
+      "loss": 2.0912,
+      "step": 1860
+    },
+    {
+      "epoch": 0.04737448642148512,
+      "grad_norm": 1.6446202993392944,
+      "learning_rate": 4.998044704162613e-05,
+      "loss": 2.0768,
+      "step": 1891
+    },
+    {
+      "epoch": 0.0481511173464275,
+      "grad_norm": 1.5588979721069336,
+      "learning_rate": 4.9976985205973705e-05,
+      "loss": 2.1137,
+      "step": 1922
+    },
+    {
+      "epoch": 0.04892774827136988,
+      "grad_norm": 2.4484736919403076,
+      "learning_rate": 4.997324150843799e-05,
+      "loss": 2.0816,
+      "step": 1953
+    },
+    {
+      "epoch": 0.04970437919631226,
+      "grad_norm": 1.626017451286316,
+      "learning_rate": 4.99692159912661e-05,
+      "loss": 2.0697,
+      "step": 1984
+    },
+    {
+      "epoch": 0.05048101012125464,
+      "grad_norm": 1.5020952224731445,
+      "learning_rate": 4.996490869988546e-05,
+      "loss": 2.0623,
+      "step": 2015
+    },
+    {
+      "epoch": 0.05125764104619701,
+      "grad_norm": 1.5254911184310913,
+      "learning_rate": 4.996031968290326e-05,
+      "loss": 2.0614,
+      "step": 2046
+    },
+    {
+      "epoch": 0.05203427197113939,
+      "grad_norm": 7.960453987121582,
+      "learning_rate": 4.995544899210594e-05,
+      "loss": 2.052,
+      "step": 2077
+    },
+    {
+      "epoch": 0.05281090289608177,
+      "grad_norm": 1.4704298973083496,
+      "learning_rate": 4.9950296682458583e-05,
+      "loss": 2.0536,
+      "step": 2108
+    },
+    {
+      "epoch": 0.05358753382102415,
+      "grad_norm": 1.520493984222412,
+      "learning_rate": 4.994486281210429e-05,
+      "loss": 2.0365,
+      "step": 2139
+    },
+    {
+      "epoch": 0.05436416474596653,
+      "grad_norm": 1.558133602142334,
+      "learning_rate": 4.9939147442363566e-05,
+      "loss": 2.0203,
+      "step": 2170
+    },
+    {
+      "epoch": 0.05514079567090891,
+      "grad_norm": 1.4657914638519287,
+      "learning_rate": 4.9933150637733574e-05,
+      "loss": 2.0153,
+      "step": 2201
+    },
+    {
+      "epoch": 0.05591742659585129,
+      "grad_norm": 1.4615249633789062,
+      "learning_rate": 4.992687246588743e-05,
+      "loss": 1.998,
+      "step": 2232
+    },
+    {
+      "epoch": 0.056694057520793664,
+      "grad_norm": 1.5175402164459229,
+      "learning_rate": 4.992031299767347e-05,
+      "loss": 1.9849,
+      "step": 2263
+    },
+    {
+      "epoch": 0.057470688445736044,
+      "grad_norm": 1.4774690866470337,
+      "learning_rate": 4.9913472307114386e-05,
+      "loss": 2.0004,
+      "step": 2294
+    },
+    {
+      "epoch": 0.058247319370678424,
+      "grad_norm": 1.4400330781936646,
+      "learning_rate": 4.9906350471406446e-05,
+      "loss": 1.997,
+      "step": 2325
+    },
+    {
+      "epoch": 0.059023950295620804,
+      "grad_norm": 1.4439043998718262,
+      "learning_rate": 4.989894757091861e-05,
+      "loss": 2.0006,
+      "step": 2356
+    },
+    {
+      "epoch": 0.059800581220563184,
+      "grad_norm": 1.4139447212219238,
+      "learning_rate": 4.989126368919158e-05,
+      "loss": 1.9874,
+      "step": 2387
+    },
+    {
+      "epoch": 0.060577212145505564,
+      "grad_norm": 1.501980185508728,
+      "learning_rate": 4.988329891293693e-05,
+      "loss": 1.9996,
+      "step": 2418
+    },
+    {
+      "epoch": 0.06135384307044794,
+      "grad_norm": 1.4147136211395264,
+      "learning_rate": 4.987505333203608e-05,
+      "loss": 2.011,
+      "step": 2449
+    },
+    {
+      "epoch": 0.06213047399539032,
+      "grad_norm": 1.8178402185440063,
+      "learning_rate": 4.9866527039539276e-05,
+      "loss": 1.9716,
+      "step": 2480
+    },
+    {
+      "epoch": 0.0629071049203327,
+      "grad_norm": 1.3381493091583252,
+      "learning_rate": 4.9857720131664594e-05,
+      "loss": 1.9703,
+      "step": 2511
+    },
+    {
+      "epoch": 0.06368373584527508,
+      "grad_norm": 1.4357298612594604,
+      "learning_rate": 4.9848632707796773e-05,
+      "loss": 1.9866,
+      "step": 2542
+    },
+    {
+      "epoch": 0.06446036677021745,
+      "grad_norm": 1.3444238901138306,
+      "learning_rate": 4.9839264870486155e-05,
+      "loss": 1.9693,
+      "step": 2573
+    },
+    {
+      "epoch": 0.06523699769515984,
+      "grad_norm": 1.3933724164962769,
+      "learning_rate": 4.9829616725447526e-05,
+      "loss": 1.9557,
+      "step": 2604
+    },
+    {
+      "epoch": 0.06601362862010221,
+      "grad_norm": 28.06955337524414,
+      "learning_rate": 4.981968838155888e-05,
+      "loss": 1.9641,
+      "step": 2635
+    },
+    {
+      "epoch": 0.0667902595450446,
+      "grad_norm": 1.4004745483398438,
+      "learning_rate": 4.980947995086024e-05,
+      "loss": 1.979,
+      "step": 2666
+    },
+    {
+      "epoch": 0.06756689046998697,
+      "grad_norm": 2.8693289756774902,
+      "learning_rate": 4.979899154855234e-05,
+      "loss": 1.9526,
+      "step": 2697
+    },
+    {
+      "epoch": 0.06834352139492936,
+      "grad_norm": 1.7132235765457153,
+      "learning_rate": 4.9788223292995386e-05,
+      "loss": 1.9425,
+      "step": 2728
+    },
+    {
+      "epoch": 0.06912015231987173,
+      "grad_norm": 1.3773481845855713,
+      "learning_rate": 4.977717530570768e-05,
+      "loss": 1.9852,
+      "step": 2759
+    },
+    {
+      "epoch": 0.0698967832448141,
+      "grad_norm": 1.337936282157898,
+      "learning_rate": 4.976584771136425e-05,
+      "loss": 1.9758,
+      "step": 2790
+    },
+    {
+      "epoch": 0.07067341416975649,
+      "grad_norm": 1.4963557720184326,
+      "learning_rate": 4.975424063779547e-05,
+      "loss": 1.9753,
+      "step": 2821
+    },
+    {
+      "epoch": 0.07145004509469886,
+      "grad_norm": 1.324593186378479,
+      "learning_rate": 4.974235421598557e-05,
+      "loss": 1.9486,
+      "step": 2852
+    },
+    {
+      "epoch": 0.07222667601964125,
+      "grad_norm": 2.7807044982910156,
+      "learning_rate": 4.973018858007122e-05,
+      "loss": 1.9234,
+      "step": 2883
+    },
+    {
+      "epoch": 0.07300330694458362,
+      "grad_norm": 9.35367488861084,
+      "learning_rate": 4.9717743867339963e-05,
+      "loss": 1.9714,
+      "step": 2914
+    },
+    {
+      "epoch": 0.07377993786952601,
+      "grad_norm": 7.89553689956665,
+      "learning_rate": 4.9705020218228695e-05,
+      "loss": 1.9319,
+      "step": 2945
+    },
+    {
+      "epoch": 0.07455656879446838,
+      "grad_norm": 1.3350094556808472,
+      "learning_rate": 4.969201777632205e-05,
+      "loss": 1.9251,
+      "step": 2976
+    },
+    {
+      "epoch": 0.07533319971941076,
+      "grad_norm": 1.3224897384643555,
+      "learning_rate": 4.9678736688350846e-05,
+      "loss": 1.9306,
+      "step": 3007
+    },
+    {
+      "epoch": 0.07610983064435314,
+      "grad_norm": 1.2839077711105347,
+      "learning_rate": 4.966517710419033e-05,
+      "loss": 1.9446,
+      "step": 3038
+    },
+    {
+      "epoch": 0.07688646156929552,
+      "grad_norm": 1.283780574798584,
+      "learning_rate": 4.965133917685858e-05,
+      "loss": 1.923,
+      "step": 3069
+    },
+    {
+      "epoch": 0.0776630924942379,
+      "grad_norm": 4.104744911193848,
+      "learning_rate": 4.9637223062514714e-05,
+      "loss": 1.9272,
+      "step": 3100
+    },
+    {
+      "epoch": 0.07843972341918028,
+      "grad_norm": 1.2491049766540527,
+      "learning_rate": 4.962282892045718e-05,
+      "loss": 1.9283,
+      "step": 3131
+    },
+    {
+      "epoch": 0.07921635434412266,
+      "grad_norm": 1.3095839023590088,
+      "learning_rate": 4.9608156913121904e-05,
+      "loss": 1.909,
+      "step": 3162
+    },
+    {
+      "epoch": 0.07999298526906504,
+      "grad_norm": 1.4149360656738281,
+      "learning_rate": 4.959320720608049e-05,
+      "loss": 1.9218,
+      "step": 3193
+    },
+    {
+      "epoch": 0.08076961619400741,
+      "grad_norm": 1.295985460281372,
+      "learning_rate": 4.9577979968038354e-05,
+      "loss": 1.9058,
+      "step": 3224
+    },
+    {
+      "epoch": 0.0815462471189498,
+      "grad_norm": 1.2115492820739746,
+      "learning_rate": 4.956247537083282e-05,
+      "loss": 1.9217,
+      "step": 3255
+    },
+    {
+      "epoch": 0.08232287804389217,
+      "grad_norm": 1.199232578277588,
+      "learning_rate": 4.9546693589431145e-05,
+      "loss": 1.9132,
+      "step": 3286
+    },
+    {
+      "epoch": 0.08309950896883456,
+      "grad_norm": 1.291377305984497,
+      "learning_rate": 4.9530634801928595e-05,
+      "loss": 1.9157,
+      "step": 3317
+    },
+    {
+      "epoch": 0.08387613989377693,
+      "grad_norm": 1.1573243141174316,
+      "learning_rate": 4.9514299189546395e-05,
+      "loss": 1.8986,
+      "step": 3348
+    },
+    {
+      "epoch": 0.08465277081871932,
+      "grad_norm": 1.1673740148544312,
+      "learning_rate": 4.949768693662973e-05,
+      "loss": 1.913,
+      "step": 3379
+    },
+    {
+      "epoch": 0.08542940174366169,
+      "grad_norm": 1.2315590381622314,
+      "learning_rate": 4.948079823064559e-05,
+      "loss": 1.9106,
+      "step": 3410
+    },
+    {
+      "epoch": 0.08620603266860406,
+      "grad_norm": 5.395792484283447,
+      "learning_rate": 4.946363326218074e-05,
+      "loss": 1.9167,
+      "step": 3441
+    },
+    {
+      "epoch": 0.08698266359354645,
+      "grad_norm": 2.3968420028686523,
+      "learning_rate": 4.9446192224939525e-05,
+      "loss": 1.8859,
+      "step": 3472
+    },
+    {
+      "epoch": 0.08775929451848882,
+      "grad_norm": 1.2546100616455078,
+      "learning_rate": 4.942847531574167e-05,
+      "loss": 1.9214,
+      "step": 3503
+    },
+    {
+      "epoch": 0.08853592544343121,
+      "grad_norm": 2.857767343521118,
+      "learning_rate": 4.941048273452008e-05,
+      "loss": 1.8984,
+      "step": 3534
+    },
+    {
+      "epoch": 0.08931255636837358,
+      "grad_norm": 1.1898611783981323,
+      "learning_rate": 4.9392214684318605e-05,
+      "loss": 1.8913,
+      "step": 3565
+    },
+    {
+      "epoch": 0.09008918729331597,
+      "grad_norm": 1.206209659576416,
+      "learning_rate": 4.93736713712897e-05,
+      "loss": 1.9059,
+      "step": 3596
+    },
+    {
+      "epoch": 0.09086581821825834,
+      "grad_norm": 1.1964539289474487,
+      "learning_rate": 4.9354853004692124e-05,
+      "loss": 1.8819,
+      "step": 3627
+    },
+    {
+      "epoch": 0.09164244914320072,
+      "grad_norm": 1.262360692024231,
+      "learning_rate": 4.93357597968886e-05,
+      "loss": 1.8778,
+      "step": 3658
+    },
+    {
+      "epoch": 0.0924190800681431,
+      "grad_norm": 1.1671788692474365,
+      "learning_rate": 4.931639196334338e-05,
+      "loss": 1.877,
+      "step": 3689
+    },
+    {
+      "epoch": 0.09319571099308548,
+      "grad_norm": 1.385147213935852,
+      "learning_rate": 4.9296749722619826e-05,
+      "loss": 1.8759,
+      "step": 3720
+    },
+    {
+      "epoch": 0.09397234191802786,
+      "grad_norm": 1.2124735116958618,
+      "learning_rate": 4.9276833296377966e-05,
+      "loss": 1.8614,
+      "step": 3751
+    },
+    {
+      "epoch": 0.09474897284297024,
+      "grad_norm": 1.1881778240203857,
+      "learning_rate": 4.925664290937196e-05,
+      "loss": 1.8644,
+      "step": 3782
+    },
+    {
+      "epoch": 0.09552560376791262,
+      "grad_norm": 1.1497128009796143,
+      "learning_rate": 4.9236178789447576e-05,
+      "loss": 1.856,
+      "step": 3813
+    },
+    {
+      "epoch": 0.096302234692855,
+      "grad_norm": 1.172032117843628,
+      "learning_rate": 4.921544116753962e-05,
+      "loss": 1.8611,
+      "step": 3844
+    },
+    {
+      "epoch": 0.09707886561779737,
+      "grad_norm": 1.2735902070999146,
+      "learning_rate": 4.919443027766935e-05,
+      "loss": 1.8567,
+      "step": 3875
+    },
+    {
+      "epoch": 0.09785549654273976,
+      "grad_norm": 1.1539477109909058,
+      "learning_rate": 4.91731463569418e-05,
+      "loss": 1.8672,
+      "step": 3906
+    },
+    {
+      "epoch": 0.09863212746768213,
+      "grad_norm": 1.1640018224716187,
+      "learning_rate": 4.915158964554312e-05,
+      "loss": 1.8498,
+      "step": 3937
+    },
+    {
+      "epoch": 0.09940875839262452,
+      "grad_norm": 1.169787883758545,
+      "learning_rate": 4.912976038673786e-05,
+      "loss": 1.8531,
+      "step": 3968
+    },
+    {
+      "epoch": 0.10018538931756689,
+      "grad_norm": 1.177053689956665,
+      "learning_rate": 4.9107658826866254e-05,
+      "loss": 1.8399,
+      "step": 3999
+    },
+    {
+      "epoch": 0.10096202024250928,
+      "grad_norm": 1.0724283456802368,
+      "learning_rate": 4.908528521534139e-05,
+      "loss": 1.8448,
+      "step": 4030
+    },
+    {
+      "epoch": 0.10173865116745165,
+      "grad_norm": 1.1025402545928955,
+      "learning_rate": 4.906263980464644e-05,
+      "loss": 1.8572,
+      "step": 4061
+    },
+    {
+      "epoch": 0.10251528209239402,
+      "grad_norm": 1.1584198474884033,
+      "learning_rate": 4.903972285033178e-05,
+      "loss": 1.8599,
+      "step": 4092
+    },
+    {
+      "epoch": 0.10329191301733641,
+      "grad_norm": 1.144680142402649,
+      "learning_rate": 4.901653461101213e-05,
+      "loss": 1.8362,
+      "step": 4123
+    },
+    {
+      "epoch": 0.10406854394227878,
+      "grad_norm": 1.136489748954773,
+      "learning_rate": 4.8993075348363626e-05,
+      "loss": 1.8446,
+      "step": 4154
+    },
+    {
+      "epoch": 0.10484517486722117,
+      "grad_norm": 1.1922993659973145,
+      "learning_rate": 4.896934532712084e-05,
+      "loss": 1.8731,
+      "step": 4185
+    },
+    {
+      "epoch": 0.10562180579216354,
+      "grad_norm": 1.229602336883545,
+      "learning_rate": 4.8945344815073846e-05,
+      "loss": 1.8356,
+      "step": 4216
+    },
+    {
+      "epoch": 0.10639843671710593,
+      "grad_norm": 1.223044514656067,
+      "learning_rate": 4.892107408306516e-05,
+      "loss": 1.826,
+      "step": 4247
+    },
+    {
+      "epoch": 0.1071750676420483,
+      "grad_norm": 2.0546789169311523,
+      "learning_rate": 4.889653340498669e-05,
+      "loss": 1.8433,
+      "step": 4278
+    },
+    {
+      "epoch": 0.10795169856699068,
+      "grad_norm": 1.132135272026062,
+      "learning_rate": 4.8871723057776664e-05,
+      "loss": 1.8349,
+      "step": 4309
+    },
+    {
+      "epoch": 0.10872832949193306,
+      "grad_norm": 1.141804575920105,
+      "learning_rate": 4.8846643321416476e-05,
+      "loss": 1.8588,
+      "step": 4340
+    },
+    {
+      "epoch": 0.10950496041687544,
+      "grad_norm": 1.2166588306427002,
+      "learning_rate": 4.882129447892753e-05,
+      "loss": 1.8221,
+      "step": 4371
+    },
+    {
+      "epoch": 0.11028159134181782,
+      "grad_norm": 1.1246964931488037,
+      "learning_rate": 4.8795676816368076e-05,
+      "loss": 1.8243,
+      "step": 4402
+    },
+    {
+      "epoch": 0.1110582222667602,
+      "grad_norm": 1.1591781377792358,
+      "learning_rate": 4.876979062282995e-05,
+      "loss": 1.8276,
+      "step": 4433
+    },
+    {
+      "epoch": 0.11183485319170258,
+      "grad_norm": 1.3351600170135498,
+      "learning_rate": 4.8743636190435325e-05,
+      "loss": 1.8396,
+      "step": 4464
+    },
+    {
+      "epoch": 0.11261148411664496,
+      "grad_norm": 1.1741563081741333,
+      "learning_rate": 4.871721381433344e-05,
+      "loss": 1.829,
+      "step": 4495
+    },
+    {
+      "epoch": 0.11338811504158733,
+      "grad_norm": 1.2419747114181519,
+      "learning_rate": 4.869052379269719e-05,
+      "loss": 1.8105,
+      "step": 4526
+    },
+    {
+      "epoch": 0.11416474596652972,
+      "grad_norm": 1.146235704421997,
+      "learning_rate": 4.866356642671985e-05,
+      "loss": 1.8284,
+      "step": 4557
+    },
+    {
+      "epoch": 0.11494137689147209,
+      "grad_norm": 1.0712112188339233,
+      "learning_rate": 4.8636342020611634e-05,
+      "loss": 1.8329,
+      "step": 4588
+    },
+    {
+      "epoch": 0.11571800781641448,
+      "grad_norm": 1.1644688844680786,
+      "learning_rate": 4.860885088159626e-05,
+      "loss": 1.8186,
+      "step": 4619
+    },
+    {
+      "epoch": 0.11649463874135685,
+      "grad_norm": 1.112260341644287,
+      "learning_rate": 4.858109331990751e-05,
+      "loss": 1.8385,
+      "step": 4650
+    },
+    {
+      "epoch": 0.11727126966629922,
+      "grad_norm": 1.1103355884552002,
+      "learning_rate": 4.855306964878567e-05,
+      "loss": 1.8172,
+      "step": 4681
+    },
+    {
+      "epoch": 0.11804790059124161,
+      "grad_norm": 1.1322386264801025,
+      "learning_rate": 4.8524780184474084e-05,
+      "loss": 1.8342,
+      "step": 4712
+    },
+    {
+      "epoch": 0.11882453151618398,
+      "grad_norm": 1.2622575759887695,
+      "learning_rate": 4.8496225246215496e-05,
+      "loss": 1.7997,
+      "step": 4743
+    },
+    {
+      "epoch": 0.11960116244112637,
+      "grad_norm": 1.0932341814041138,
+      "learning_rate": 4.8467405156248505e-05,
+      "loss": 1.8125,
+      "step": 4774
+    },
+    {
+      "epoch": 0.12037779336606874,
+      "grad_norm": 1.08563232421875,
+      "learning_rate": 4.843832023980392e-05,
+      "loss": 1.817,
+      "step": 4805
+    },
+    {
+      "epoch": 0.12115442429101113,
+      "grad_norm": 1.1124180555343628,
+      "learning_rate": 4.840897082510106e-05,
+      "loss": 1.8146,
+      "step": 4836
+    },
+    {
+      "epoch": 0.1219310552159535,
+      "grad_norm": 1.0874966382980347,
+      "learning_rate": 4.8379357243344084e-05,
+      "loss": 1.7897,
+      "step": 4867
+    },
+    {
+      "epoch": 0.12270768614089587,
+      "grad_norm": 1.1009818315505981,
+      "learning_rate": 4.8349479828718236e-05,
+      "loss": 1.8177,
+      "step": 4898
+    },
+    {
+      "epoch": 0.12348431706583826,
+      "grad_norm": 1.0934672355651855,
+      "learning_rate": 4.8319338918386075e-05,
+      "loss": 1.8121,
+      "step": 4929
+    },
+    {
+      "epoch": 0.12426094799078063,
+      "grad_norm": 1.0754870176315308,
+      "learning_rate": 4.828893485248369e-05,
+      "loss": 1.8298,
+      "step": 4960
+    },
+    {
+      "epoch": 0.125037578915723,
+      "grad_norm": 1.0648986101150513,
+      "learning_rate": 4.825826797411682e-05,
+      "loss": 1.7933,
+      "step": 4991
+    },
+    {
+      "epoch": 0.1258142098406654,
+      "grad_norm": 1.0323240756988525,
+      "learning_rate": 4.822733862935702e-05,
+      "loss": 1.818,
+      "step": 5022
+    },
+    {
+      "epoch": 0.12659084076560778,
+      "grad_norm": 1.1894131898880005,
+      "learning_rate": 4.819614716723775e-05,
+      "loss": 1.7932,
+      "step": 5053
+    },
+    {
+      "epoch": 0.12736747169055015,
+      "grad_norm": 1.047052025794983,
+      "learning_rate": 4.8164693939750425e-05,
+      "loss": 1.8382,
+      "step": 5084
+    },
+    {
+      "epoch": 0.12814410261549253,
+      "grad_norm": 1.1111092567443848,
+      "learning_rate": 4.813297930184042e-05,
+      "loss": 1.7988,
+      "step": 5115
+    },
+    {
+      "epoch": 0.1289207335404349,
+      "grad_norm": 1.0632669925689697,
+      "learning_rate": 4.810100361140314e-05,
+      "loss": 1.7971,
+      "step": 5146
+    },
+    {
+      "epoch": 0.1296973644653773,
+      "grad_norm": 1.1448324918746948,
+      "learning_rate": 4.8068767229279885e-05,
+      "loss": 1.7677,
+      "step": 5177
+    },
+    {
+      "epoch": 0.13047399539031967,
+      "grad_norm": 1.0809260606765747,
+      "learning_rate": 4.8036270519253854e-05,
+      "loss": 1.7829,
+      "step": 5208
+    },
+    {
+      "epoch": 0.13125062631526205,
+      "grad_norm": 1.0866914987564087,
+      "learning_rate": 4.8003513848046e-05,
+      "loss": 1.7746,
+      "step": 5239
+    },
+    {
+      "epoch": 0.13202725724020442,
+      "grad_norm": 1.021988034248352,
+      "learning_rate": 4.79704975853109e-05,
+      "loss": 1.8037,
+      "step": 5270
+    },
+    {
+      "epoch": 0.13280388816514682,
+      "grad_norm": 1.064077615737915,
+      "learning_rate": 4.793722210363262e-05,
+      "loss": 1.7969,
+      "step": 5301
+    },
+    {
+      "epoch": 0.1335805190900892,
+      "grad_norm": 1.087003469467163,
+      "learning_rate": 4.7903687778520414e-05,
+      "loss": 1.7823,
+      "step": 5332
+    },
+    {
+      "epoch": 0.13435715001503157,
+      "grad_norm": 1.0024127960205078,
+      "learning_rate": 4.7869894988404593e-05,
+      "loss": 1.7784,
+      "step": 5363
+    },
+    {
+      "epoch": 0.13513378093997394,
+      "grad_norm": 1.1223832368850708,
+      "learning_rate": 4.783584411463221e-05,
+      "loss": 1.7926,
+      "step": 5394
+    },
+    {
+      "epoch": 0.13591041186491631,
+      "grad_norm": 1.0648205280303955,
+      "learning_rate": 4.780153554146274e-05,
+      "loss": 1.795,
+      "step": 5425
+    },
+    {
+      "epoch": 0.13668704278985871,
+      "grad_norm": 1.0414562225341797,
+      "learning_rate": 4.7766969656063766e-05,
+      "loss": 1.7706,
+      "step": 5456
+    },
+    {
+      "epoch": 0.1374636737148011,
+      "grad_norm": 1.0693738460540771,
+      "learning_rate": 4.773214684850662e-05,
+      "loss": 1.7972,
+      "step": 5487
+    },
+    {
+      "epoch": 0.13824030463974346,
+      "grad_norm": 1.019546627998352,
+      "learning_rate": 4.769706751176193e-05,
+      "loss": 1.8109,
+      "step": 5518
+    },
+    {
+      "epoch": 0.13901693556468583,
+      "grad_norm": 1.0026518106460571,
+      "learning_rate": 4.7661732041695264e-05,
+      "loss": 1.7599,
+      "step": 5549
+    },
+    {
+      "epoch": 0.1397935664896282,
+      "grad_norm": 1.0571423768997192,
+      "learning_rate": 4.762614083706258e-05,
+      "loss": 1.8028,
+      "step": 5580
+    },
+    {
+      "epoch": 0.1405701974145706,
+      "grad_norm": 1.0544196367263794,
+      "learning_rate": 4.759029429950581e-05,
+      "loss": 1.7804,
+      "step": 5611
+    },
+    {
+      "epoch": 0.14134682833951298,
+      "grad_norm": 1.0218669176101685,
+      "learning_rate": 4.7554192833548235e-05,
+      "loss": 1.7707,
+      "step": 5642
+    },
+    {
+      "epoch": 0.14212345926445535,
+      "grad_norm": 1.1020311117172241,
+      "learning_rate": 4.751783684659e-05,
+      "loss": 1.7893,
+      "step": 5673
+    },
+    {
+      "epoch": 0.14290009018939773,
+      "grad_norm": 1.055177927017212,
+      "learning_rate": 4.748122674890348e-05,
+      "loss": 1.753,
+      "step": 5704
+    },
+    {
+      "epoch": 0.14367672111434013,
+      "grad_norm": 1.0327739715576172,
+      "learning_rate": 4.7444362953628654e-05,
+      "loss": 1.7662,
+      "step": 5735
+    },
+    {
+      "epoch": 0.1444533520392825,
+      "grad_norm": 1.0046563148498535,
+      "learning_rate": 4.7407245876768424e-05,
+      "loss": 1.7901,
+      "step": 5766
+    },
+    {
+      "epoch": 0.14522998296422487,
+      "grad_norm": 1.0504815578460693,
+      "learning_rate": 4.736987593718397e-05,
+      "loss": 1.7735,
+      "step": 5797
+    },
+    {
+      "epoch": 0.14600661388916725,
+      "grad_norm": 1.081047534942627,
+      "learning_rate": 4.733225355658999e-05,
+      "loss": 1.7837,
+      "step": 5828
+    },
+    {
+      "epoch": 0.14678324481410962,
+      "grad_norm": 1.012346863746643,
+      "learning_rate": 4.7294379159549926e-05,
+      "loss": 1.7705,
+      "step": 5859
+    },
+    {
+      "epoch": 0.14755987573905202,
+      "grad_norm": 0.9958217740058899,
+      "learning_rate": 4.725625317347119e-05,
+      "loss": 1.7649,
+      "step": 5890
+    },
+    {
+      "epoch": 0.1483365066639944,
+      "grad_norm": 1.0760499238967896,
+      "learning_rate": 4.7217876028600374e-05,
+      "loss": 1.7945,
+      "step": 5921
+    },
+    {
+      "epoch": 0.14911313758893677,
+      "grad_norm": 1.1010396480560303,
+      "learning_rate": 4.717924815801832e-05,
+      "loss": 1.7756,
+      "step": 5952
+    },
+    {
+      "epoch": 0.14988976851387914,
+      "grad_norm": 1.0379377603530884,
+      "learning_rate": 4.714036999763532e-05,
+      "loss": 1.771,
+      "step": 5983
+    },
+    {
+      "epoch": 0.1506663994388215,
+      "grad_norm": 1.0273206233978271,
+      "learning_rate": 4.7101241986186116e-05,
+      "loss": 1.7571,
+      "step": 6014
+    },
+    {
+      "epoch": 0.15144303036376391,
+      "grad_norm": 1.0275596380233765,
+      "learning_rate": 4.7061864565225e-05,
+      "loss": 1.7668,
+      "step": 6045
+    },
+    {
+      "epoch": 0.1522196612887063,
+      "grad_norm": 1.076374888420105,
+      "learning_rate": 4.702223817912081e-05,
+      "loss": 1.7819,
+      "step": 6076
+    },
+    {
+      "epoch": 0.15299629221364866,
+      "grad_norm": 1.0678372383117676,
+      "learning_rate": 4.698236327505195e-05,
+      "loss": 1.7811,
+      "step": 6107
+    },
+    {
+      "epoch": 0.15377292313859103,
+      "grad_norm": 1.017540693283081,
+      "learning_rate": 4.694224030300127e-05,
+      "loss": 1.7638,
+      "step": 6138
+    },
+    {
+      "epoch": 0.1545495540635334,
+      "grad_norm": 1.0062652826309204,
+      "learning_rate": 4.690186971575107e-05,
+      "loss": 1.7535,
+      "step": 6169
+    },
+    {
+      "epoch": 0.1553261849884758,
+      "grad_norm": 1.0092757940292358,
+      "learning_rate": 4.6861251968877916e-05,
+      "loss": 1.7671,
+      "step": 6200
+    },
+    {
+      "epoch": 0.15610281591341818,
+      "grad_norm": 0.9981346130371094,
+      "learning_rate": 4.68203875207476e-05,
+      "loss": 1.755,
+      "step": 6231
+    },
+    {
+      "epoch": 0.15687944683836055,
+      "grad_norm": 2.156689167022705,
+      "learning_rate": 4.677927683250983e-05,
+      "loss": 1.7679,
+      "step": 6262
+    },
+    {
+      "epoch": 0.15765607776330293,
+      "grad_norm": 0.9837804436683655,
+      "learning_rate": 4.6737920368093156e-05,
+      "loss": 1.7523,
+      "step": 6293
+    },
+    {
+      "epoch": 0.15843270868824533,
+      "grad_norm": 1.1614326238632202,
+      "learning_rate": 4.669631859419965e-05,
+      "loss": 1.7484,
+      "step": 6324
+    },
+    {
+      "epoch": 0.1592093396131877,
+      "grad_norm": 1.0423202514648438,
+      "learning_rate": 4.6654471980299676e-05,
+      "loss": 1.7707,
+      "step": 6355
+    },
+    {
+      "epoch": 0.15998597053813007,
+      "grad_norm": 0.9598566293716431,
+      "learning_rate": 4.661238099862658e-05,
+      "loss": 1.7544,
+      "step": 6386
+    },
+    {
+      "epoch": 0.16076260146307245,
+      "grad_norm": 1.053193211555481,
+      "learning_rate": 4.657004612417138e-05,
+      "loss": 1.7627,
+      "step": 6417
+    },
+    {
+      "epoch": 0.16153923238801482,
+      "grad_norm": 1.0258429050445557,
+      "learning_rate": 4.6527467834677374e-05,
+      "loss": 1.7397,
+      "step": 6448
+    },
+    {
+      "epoch": 0.16231586331295722,
+      "grad_norm": 1.0296120643615723,
+      "learning_rate": 4.648464661063478e-05,
+      "loss": 1.76,
+      "step": 6479
+    },
+    {
+      "epoch": 0.1630924942378996,
+      "grad_norm": 1.0015708208084106,
+      "learning_rate": 4.6441582935275264e-05,
+      "loss": 1.7542,
+      "step": 6510
+    },
+    {
+      "epoch": 0.16386912516284197,
+      "grad_norm": 0.982397198677063,
+      "learning_rate": 4.6398277294566586e-05,
+      "loss": 1.7278,
+      "step": 6541
+    },
+    {
+      "epoch": 0.16464575608778434,
+      "grad_norm": 0.9919869303703308,
+      "learning_rate": 4.6354730177207e-05,
+      "loss": 1.7419,
+      "step": 6572
+    },
+    {
+      "epoch": 0.1654223870127267,
+      "grad_norm": 1.0046809911727905,
+      "learning_rate": 4.6310942074619787e-05,
+      "loss": 1.7669,
+      "step": 6603
+    },
+    {
+      "epoch": 0.1661990179376691,
+      "grad_norm": 0.9675459265708923,
+      "learning_rate": 4.626691348094777e-05,
+      "loss": 1.7453,
+      "step": 6634
+    },
+    {
+      "epoch": 0.1669756488626115,
+      "grad_norm": 0.9798470735549927,
+      "learning_rate": 4.622264489304762e-05,
+      "loss": 1.7438,
+      "step": 6665
+    },
+    {
+      "epoch": 0.16775227978755386,
+      "grad_norm": 0.9938516616821289,
+      "learning_rate": 4.617813681048434e-05,
+      "loss": 1.7503,
+      "step": 6696
+    },
+    {
+      "epoch": 0.16852891071249623,
+      "grad_norm": 0.9619422554969788,
+      "learning_rate": 4.61333897355256e-05,
+      "loss": 1.7642,
+      "step": 6727
+    },
+    {
+      "epoch": 0.16930554163743863,
+      "grad_norm": 0.9402725100517273,
+      "learning_rate": 4.608840417313604e-05,
+      "loss": 1.7389,
+      "step": 6758
+    },
+    {
+      "epoch": 0.170082172562381,
+      "grad_norm": 0.9799749255180359,
+      "learning_rate": 4.6043180630971646e-05,
+      "loss": 1.7454,
+      "step": 6789
+    },
+    {
+      "epoch": 0.17085880348732338,
+      "grad_norm": 0.920802116394043,
+      "learning_rate": 4.599771961937391e-05,
+      "loss": 1.7384,
+      "step": 6820
+    },
+    {
+      "epoch": 0.17163543441226575,
+      "grad_norm": 0.9431501626968384,
+      "learning_rate": 4.5952021651364204e-05,
+      "loss": 1.7194,
+      "step": 6851
+    },
+    {
+      "epoch": 0.17241206533720813,
+      "grad_norm": 0.9482398629188538,
+      "learning_rate": 4.590608724263786e-05,
+      "loss": 1.7513,
+      "step": 6882
+    },
+    {
+      "epoch": 0.17318869626215053,
+      "grad_norm": 0.9813526272773743,
+      "learning_rate": 4.585991691155845e-05,
+      "loss": 1.7451,
+      "step": 6913
+    },
+    {
+      "epoch": 0.1739653271870929,
+      "grad_norm": 1.0157101154327393,
+      "learning_rate": 4.581351117915188e-05,
+      "loss": 1.7464,
+      "step": 6944
+    },
+    {
+      "epoch": 0.17474195811203527,
+      "grad_norm": 0.9638293981552124,
+      "learning_rate": 4.5766870569100534e-05,
+      "loss": 1.7313,
+      "step": 6975
+    },
+    {
+      "epoch": 0.17551858903697765,
+      "grad_norm": 0.9673603773117065,
+      "learning_rate": 4.571999560773736e-05,
+      "loss": 1.748,
+      "step": 7006
+    },
+    {
+      "epoch": 0.17629521996192002,
+      "grad_norm": 0.9882768392562866,
+      "learning_rate": 4.5672886824039915e-05,
+      "loss": 1.7384,
+      "step": 7037
+    },
+    {
+      "epoch": 0.17707185088686242,
+      "grad_norm": 0.9788867235183716,
+      "learning_rate": 4.5625544749624435e-05,
+      "loss": 1.7522,
+      "step": 7068
+    },
+    {
+      "epoch": 0.1778484818118048,
+      "grad_norm": 0.9794373512268066,
+      "learning_rate": 4.5577969918739794e-05,
+      "loss": 1.7077,
+      "step": 7099
+    },
+    {
+      "epoch": 0.17862511273674717,
+      "grad_norm": 1.0159693956375122,
+      "learning_rate": 4.5530162868261486e-05,
+      "loss": 1.7346,
+      "step": 7130
+    },
+    {
+      "epoch": 0.17940174366168954,
+      "grad_norm": 1.0156211853027344,
+      "learning_rate": 4.548212413768558e-05,
+      "loss": 1.7447,
+      "step": 7161
+    },
+    {
+      "epoch": 0.18017837458663194,
+      "grad_norm": 0.9733012318611145,
+      "learning_rate": 4.543385426912261e-05,
+      "loss": 1.7293,
+      "step": 7192
+    },
+    {
+      "epoch": 0.1809550055115743,
+      "grad_norm": 0.9807786345481873,
+      "learning_rate": 4.53853538072915e-05,
+      "loss": 1.7389,
+      "step": 7223
+    },
+    {
+      "epoch": 0.18173163643651669,
+      "grad_norm": 0.972074031829834,
+      "learning_rate": 4.533662329951336e-05,
+      "loss": 1.7218,
+      "step": 7254
+    },
+    {
+      "epoch": 0.18250826736145906,
+      "grad_norm": 0.9490823745727539,
+      "learning_rate": 4.528766329570536e-05,
+      "loss": 1.706,
+      "step": 7285
+    },
+    {
+      "epoch": 0.18328489828640143,
+      "grad_norm": 0.9884642958641052,
+      "learning_rate": 4.523847434837447e-05,
+      "loss": 1.7341,
+      "step": 7316
+    },
+    {
+      "epoch": 0.18406152921134383,
+      "grad_norm": 0.9563024640083313,
+      "learning_rate": 4.518905701261128e-05,
+      "loss": 1.718,
+      "step": 7347
+    },
+    {
+      "epoch": 0.1848381601362862,
+      "grad_norm": 1.177401065826416,
+      "learning_rate": 4.5139411846083715e-05,
+      "loss": 1.723,
+      "step": 7378
+    },
+    {
+      "epoch": 0.18561479106122858,
+      "grad_norm": 0.9618046879768372,
+      "learning_rate": 4.508953940903073e-05,
+      "loss": 1.7309,
+      "step": 7409
+    },
+    {
+      "epoch": 0.18639142198617095,
+      "grad_norm": 0.9730792045593262,
+      "learning_rate": 4.5039440264255994e-05,
+      "loss": 1.7236,
+      "step": 7440
+    },
+    {
+      "epoch": 0.18716805291111333,
+      "grad_norm": 0.9472179412841797,
+      "learning_rate": 4.498911497712155e-05,
+      "loss": 1.7142,
+      "step": 7471
+    },
+    {
+      "epoch": 0.18794468383605573,
+      "grad_norm": 0.9653865694999695,
+      "learning_rate": 4.493856411554142e-05,
+      "loss": 1.727,
+      "step": 7502
+    },
+    {
+      "epoch": 0.1887213147609981,
+      "grad_norm": 0.9445356726646423,
+      "learning_rate": 4.4887788249975206e-05,
+      "loss": 1.7154,
+      "step": 7533
+    },
+    {
+      "epoch": 0.18949794568594047,
+      "grad_norm": 1.1695305109024048,
+      "learning_rate": 4.4836787953421656e-05,
+      "loss": 1.7257,
+      "step": 7564
+    },
+    {
+      "epoch": 0.19027457661088285,
+      "grad_norm": 0.9815401434898376,
+      "learning_rate": 4.478556380141218e-05,
+      "loss": 1.7237,
+      "step": 7595
+    },
+    {
+      "epoch": 0.19105120753582525,
+      "grad_norm": 0.9725745916366577,
+      "learning_rate": 4.4734116372004375e-05,
+      "loss": 1.7362,
+      "step": 7626
+    },
+    {
+      "epoch": 0.19182783846076762,
+      "grad_norm": 0.9455934762954712,
+      "learning_rate": 4.4682446245775477e-05,
+      "loss": 1.7008,
+      "step": 7657
+    },
+    {
+      "epoch": 0.19260446938571,
+      "grad_norm": 1.056397557258606,
+      "learning_rate": 4.463055400581586e-05,
+      "loss": 1.7343,
+      "step": 7688
+    },
+    {
+      "epoch": 0.19338110031065237,
+      "grad_norm": 0.9315922856330872,
+      "learning_rate": 4.4578440237722374e-05,
+      "loss": 1.706,
+      "step": 7719
+    },
+    {
+      "epoch": 0.19415773123559474,
+      "grad_norm": 0.9606077075004578,
+      "learning_rate": 4.452610552959183e-05,
+      "loss": 1.7191,
+      "step": 7750
+    },
+    {
+      "epoch": 0.19493436216053714,
+      "grad_norm": 0.9894726872444153,
+      "learning_rate": 4.447355047201428e-05,
+      "loss": 1.7385,
+      "step": 7781
+    },
+    {
+      "epoch": 0.1957109930854795,
+      "grad_norm": 0.902044951915741,
+      "learning_rate": 4.4420775658066414e-05,
+      "loss": 1.6963,
+      "step": 7812
+    },
+    {
+      "epoch": 0.19648762401042189,
+      "grad_norm": 0.9528105854988098,
+      "learning_rate": 4.436778168330484e-05,
+      "loss": 1.7227,
+      "step": 7843
+    },
+    {
+      "epoch": 0.19726425493536426,
+      "grad_norm": 0.96247798204422,
+      "learning_rate": 4.4314569145759353e-05,
+      "loss": 1.7305,
+      "step": 7874
+    },
+    {
+      "epoch": 0.19804088586030663,
+      "grad_norm": 8.68586254119873,
+      "learning_rate": 4.42611386459262e-05,
+      "loss": 1.7203,
+      "step": 7905
+    },
+    {
+      "epoch": 0.19881751678524903,
+      "grad_norm": 0.9925842881202698,
+      "learning_rate": 4.420749078676133e-05,
+      "loss": 1.7008,
+      "step": 7936
+    },
+    {
+      "epoch": 0.1995941477101914,
+      "grad_norm": 0.9336928129196167,
+      "learning_rate": 4.4153626173673516e-05,
+      "loss": 1.6996,
+      "step": 7967
+    },
+    {
+      "epoch": 0.20037077863513378,
+      "grad_norm": 0.9617434740066528,
+      "learning_rate": 4.409954541451762e-05,
+      "loss": 1.7099,
+      "step": 7998
+    },
+    {
+      "epoch": 0.20114740956007615,
+      "grad_norm": 0.9902910590171814,
+      "learning_rate": 4.404524911958764e-05,
+      "loss": 1.7146,
+      "step": 8029
+    },
+    {
+      "epoch": 0.20192404048501855,
+      "grad_norm": 0.9573171734809875,
+      "learning_rate": 4.399073790160989e-05,
+      "loss": 1.7109,
+      "step": 8060
+    },
+    {
+      "epoch": 0.20270067140996093,
+      "grad_norm": 0.9622448682785034,
+      "learning_rate": 4.393601237573607e-05,
+      "loss": 1.7025,
+      "step": 8091
+    },
+    {
+      "epoch": 0.2034773023349033,
+      "grad_norm": 0.9422811269760132,
+      "learning_rate": 4.388107315953628e-05,
+      "loss": 1.7207,
+      "step": 8122
+    },
+    {
+      "epoch": 0.20425393325984567,
+      "grad_norm": 0.9360016584396362,
+      "learning_rate": 4.382592087299212e-05,
+      "loss": 1.7111,
+      "step": 8153
+    },
+    {
+      "epoch": 0.20503056418478804,
+      "grad_norm": 0.9532743096351624,
+      "learning_rate": 4.377055613848964e-05,
+      "loss": 1.7064,
+      "step": 8184
+    },
+    {
+      "epoch": 0.20580719510973045,
+      "grad_norm": 0.9463014006614685,
+      "learning_rate": 4.3714979580812355e-05,
+      "loss": 1.7107,
+      "step": 8215
+    },
+    {
+      "epoch": 0.20658382603467282,
+      "grad_norm": 3.2335429191589355,
+      "learning_rate": 4.365919182713416e-05,
+      "loss": 1.702,
+      "step": 8246
+    },
+    {
+      "epoch": 0.2073604569596152,
+      "grad_norm": 0.8885370492935181,
+      "learning_rate": 4.360319350701226e-05,
+      "loss": 1.7011,
+      "step": 8277
+    },
+    {
+      "epoch": 0.20813708788455756,
+      "grad_norm": 0.9680066704750061,
+      "learning_rate": 4.3546985252380115e-05,
+      "loss": 1.7313,
+      "step": 8308
+    },
+    {
+      "epoch": 0.20891371880949994,
+      "grad_norm": 0.9495968222618103,
+      "learning_rate": 4.349056769754021e-05,
+      "loss": 1.6877,
+      "step": 8339
+    },
+    {
+      "epoch": 0.20969034973444234,
+      "grad_norm": 0.9398072361946106,
+      "learning_rate": 4.3433941479156994e-05,
+      "loss": 1.6907,
+      "step": 8370
+    },
+    {
+      "epoch": 0.2104669806593847,
+      "grad_norm": 0.9507032036781311,
+      "learning_rate": 4.3377107236249647e-05,
+      "loss": 1.7275,
+      "step": 8401
+    },
+    {
+      "epoch": 0.21124361158432708,
+      "grad_norm": 0.9065993428230286,
+      "learning_rate": 4.332006561018488e-05,
+      "loss": 1.6821,
+      "step": 8432
+    },
+    {
+      "epoch": 0.21202024250926946,
+      "grad_norm": 0.9258121252059937,
+      "learning_rate": 4.3262817244669683e-05,
+      "loss": 1.7044,
+      "step": 8463
+    },
+    {
+      "epoch": 0.21279687343421186,
+      "grad_norm": 1.0351741313934326,
+      "learning_rate": 4.3205362785744083e-05,
+      "loss": 1.6997,
+      "step": 8494
+    },
+    {
+      "epoch": 0.21357350435915423,
+      "grad_norm": 0.9390230774879456,
+      "learning_rate": 4.314770288177384e-05,
+      "loss": 1.7038,
+      "step": 8525
+    },
+    {
+      "epoch": 0.2143501352840966,
+      "grad_norm": 0.8981362581253052,
+      "learning_rate": 4.308983818344313e-05,
+      "loss": 1.7026,
+      "step": 8556
+    },
+    {
+      "epoch": 0.21512676620903898,
+      "grad_norm": 0.9399337768554688,
+      "learning_rate": 4.3031769343747206e-05,
+      "loss": 1.7114,
+      "step": 8587
+    },
+    {
+      "epoch": 0.21590339713398135,
+      "grad_norm": 0.9432971477508545,
+      "learning_rate": 4.297349701798505e-05,
+      "loss": 1.6777,
+      "step": 8618
+    },
+    {
+      "epoch": 0.21668002805892375,
+      "grad_norm": 0.9247739315032959,
+      "learning_rate": 4.2915021863751916e-05,
+      "loss": 1.7159,
+      "step": 8649
+    },
+    {
+      "epoch": 0.21745665898386612,
+      "grad_norm": 1.0542418956756592,
+      "learning_rate": 4.285634454093198e-05,
+      "loss": 1.6872,
+      "step": 8680
+    },
+    {
+      "epoch": 0.2182332899088085,
+      "grad_norm": 0.9464339017868042,
+      "learning_rate": 4.279746571169086e-05,
+      "loss": 1.6933,
+      "step": 8711
+    },
+    {
+      "epoch": 0.21900992083375087,
+      "grad_norm": 0.9342502355575562,
+      "learning_rate": 4.2738386040468136e-05,
+      "loss": 1.701,
+      "step": 8742
+    },
+    {
+      "epoch": 0.21978655175869324,
+      "grad_norm": 0.935588538646698,
+      "learning_rate": 4.2679106193969866e-05,
+      "loss": 1.6874,
+      "step": 8773
+    },
+    {
+      "epoch": 0.22056318268363564,
+      "grad_norm": 0.9848533272743225,
+      "learning_rate": 4.261962684116106e-05,
+      "loss": 1.6834,
+      "step": 8804
+    },
+    {
+      "epoch": 0.22133981360857802,
+      "grad_norm": 0.9204187989234924,
+      "learning_rate": 4.2559948653258145e-05,
+      "loss": 1.7051,
+      "step": 8835
+    },
+    {
+      "epoch": 0.2221164445335204,
+      "grad_norm": 0.9236589670181274,
+      "learning_rate": 4.250007230372134e-05,
+      "loss": 1.7032,
+      "step": 8866
+    },
+    {
+      "epoch": 0.22289307545846276,
+      "grad_norm": 0.9299633502960205,
+      "learning_rate": 4.2439998468247126e-05,
+      "loss": 1.7007,
+      "step": 8897
+    },
+    {
+      "epoch": 0.22366970638340516,
+      "grad_norm": 0.9392068386077881,
+      "learning_rate": 4.2379727824760566e-05,
+      "loss": 1.6908,
+      "step": 8928
+    },
+    {
+      "epoch": 0.22444633730834754,
+      "grad_norm": 0.9339346289634705,
+      "learning_rate": 4.231926105340768e-05,
+      "loss": 1.712,
+      "step": 8959
+    },
+    {
+      "epoch": 0.2252229682332899,
+      "grad_norm": 0.9442049860954285,
+      "learning_rate": 4.225859883654776e-05,
+      "loss": 1.6876,
+      "step": 8990
+    },
+    {
+      "epoch": 0.22599959915823228,
+      "grad_norm": 0.9594300985336304,
+      "learning_rate": 4.219774185874569e-05,
+      "loss": 1.6933,
+      "step": 9021
+    },
+    {
+      "epoch": 0.22677623008317466,
+      "grad_norm": 0.9523814916610718,
+      "learning_rate": 4.213669080676418e-05,
+      "loss": 1.7117,
+      "step": 9052
+    },
+    {
+      "epoch": 0.22755286100811706,
+      "grad_norm": 0.9422123432159424,
+      "learning_rate": 4.2075446369556056e-05,
+      "loss": 1.6772,
+      "step": 9083
+    },
+    {
+      "epoch": 0.22832949193305943,
+      "grad_norm": 0.9157977104187012,
+      "learning_rate": 4.201400923825648e-05,
+      "loss": 1.6913,
+      "step": 9114
+    },
+    {
+      "epoch": 0.2291061228580018,
+      "grad_norm": 0.8751170039176941,
+      "learning_rate": 4.195238010617511e-05,
+      "loss": 1.6789,
+      "step": 9145
+    },
+    {
+      "epoch": 0.22988275378294418,
+      "grad_norm": 0.9077373743057251,
+      "learning_rate": 4.1890559668788344e-05,
+      "loss": 1.6984,
+      "step": 9176
+    },
+    {
+      "epoch": 0.23065938470788655,
+      "grad_norm": 0.9225157499313354,
+      "learning_rate": 4.1828548623731405e-05,
+      "loss": 1.6785,
+      "step": 9207
+    },
+    {
+      "epoch": 0.23143601563282895,
+      "grad_norm": 0.9228852987289429,
+      "learning_rate": 4.1766347670790506e-05,
+      "loss": 1.6813,
+      "step": 9238
+    },
+    {
+      "epoch": 0.23221264655777132,
+      "grad_norm": 0.9218633770942688,
+      "learning_rate": 4.170395751189495e-05,
+      "loss": 1.6733,
+      "step": 9269
+    },
+    {
+      "epoch": 0.2329892774827137,
+      "grad_norm": 0.9308573603630066,
+      "learning_rate": 4.164137885110921e-05,
+      "loss": 1.6838,
+      "step": 9300
+    },
+    {
+      "epoch": 0.23376590840765607,
+      "grad_norm": 0.9127876162528992,
+      "learning_rate": 4.157861239462495e-05,
+      "loss": 1.6878,
+      "step": 9331
+    },
+    {
+      "epoch": 0.23454253933259844,
+      "grad_norm": 0.9259180426597595,
+      "learning_rate": 4.1515658850753114e-05,
+      "loss": 1.7104,
+      "step": 9362
+    },
+    {
+      "epoch": 0.23531917025754084,
+      "grad_norm": 0.9242190718650818,
+      "learning_rate": 4.145251892991588e-05,
+      "loss": 1.7112,
+      "step": 9393
+    },
+    {
+      "epoch": 0.23609580118248322,
+      "grad_norm": 0.9331914186477661,
+      "learning_rate": 4.138919334463868e-05,
+      "loss": 1.6812,
+      "step": 9424
+    },
+    {
+      "epoch": 0.2368724321074256,
+      "grad_norm": 0.9196261167526245,
+      "learning_rate": 4.1325682809542124e-05,
+      "loss": 1.694,
+      "step": 9455
+    },
+    {
+      "epoch": 0.23764906303236796,
+      "grad_norm": 0.8865910172462463,
+      "learning_rate": 4.126198804133398e-05,
+      "loss": 1.6854,
+      "step": 9486
+    },
+    {
+      "epoch": 0.23842569395731036,
+      "grad_norm": 1.8956305980682373,
+      "learning_rate": 4.1198109758801055e-05,
+      "loss": 1.6977,
+      "step": 9517
+    },
+    {
+      "epoch": 0.23920232488225274,
+      "grad_norm": 0.8784962892532349,
+      "learning_rate": 4.113404868280107e-05,
+      "loss": 1.6683,
+      "step": 9548
+    },
+    {
+      "epoch": 0.2399789558071951,
+      "grad_norm": 0.8915697932243347,
+      "learning_rate": 4.106980553625457e-05,
+      "loss": 1.6781,
+      "step": 9579
+    },
+    {
+      "epoch": 0.24075558673213748,
+      "grad_norm": 0.8958423733711243,
+      "learning_rate": 4.100538104413674e-05,
+      "loss": 1.6904,
+      "step": 9610
+    },
+    {
+      "epoch": 0.24153221765707986,
+      "grad_norm": 0.8744849562644958,
+      "learning_rate": 4.09407759334692e-05,
+      "loss": 1.7014,
+      "step": 9641
+    },
+    {
+      "epoch": 0.24230884858202226,
+      "grad_norm": 0.8954976201057434,
+      "learning_rate": 4.087599093331186e-05,
+      "loss": 1.6656,
+      "step": 9672
+    },
+    {
+      "epoch": 0.24308547950696463,
+      "grad_norm": 0.9122741222381592,
+      "learning_rate": 4.081102677475462e-05,
+      "loss": 1.6588,
+      "step": 9703
+    },
+    {
+      "epoch": 0.243862110431907,
+      "grad_norm": 0.9107664227485657,
+      "learning_rate": 4.0745884190909194e-05,
+      "loss": 1.6705,
+      "step": 9734
+    },
+    {
+      "epoch": 0.24463874135684938,
+      "grad_norm": 0.9182100892066956,
+      "learning_rate": 4.0680563916900796e-05,
+      "loss": 1.67,
+      "step": 9765
+    },
+    {
+      "epoch": 0.24541537228179175,
+      "grad_norm": 0.9274264574050903,
+      "learning_rate": 4.0615066689859815e-05,
+      "loss": 1.6918,
+      "step": 9796
+    },
+    {
+      "epoch": 0.24619200320673415,
+      "grad_norm": 0.90735924243927,
+      "learning_rate": 4.0549393248913584e-05,
+      "loss": 1.6645,
+      "step": 9827
+    },
+    {
+      "epoch": 0.24696863413167652,
+      "grad_norm": 0.938662588596344,
+      "learning_rate": 4.048354433517794e-05,
+      "loss": 1.673,
+      "step": 9858
+    },
+    {
+      "epoch": 0.2477452650566189,
+      "grad_norm": 0.882673442363739,
+      "learning_rate": 4.0417520691748916e-05,
+      "loss": 1.6871,
+      "step": 9889
+    },
+    {
+      "epoch": 0.24852189598156127,
+      "grad_norm": 0.9797132015228271,
+      "learning_rate": 4.035132306369438e-05,
+      "loss": 1.6523,
+      "step": 9920
+    },
+    {
+      "epoch": 0.24929852690650367,
+      "grad_norm": 0.9515479207038879,
+      "learning_rate": 4.028495219804555e-05,
+      "loss": 1.6555,
+      "step": 9951
+    },
+    {
+      "epoch": 0.250075157831446,
+      "grad_norm": 0.922099769115448,
+      "learning_rate": 4.021840884378864e-05,
+      "loss": 1.7006,
+      "step": 9982
+    },
+    {
+      "epoch": 0.2508517887563884,
+      "grad_norm": 0.9409676790237427,
+      "learning_rate": 4.015169375185633e-05,
+      "loss": 1.675,
+      "step": 10013
+    },
+    {
+      "epoch": 0.2516284196813308,
+      "grad_norm": 0.9041734933853149,
+      "learning_rate": 4.0084807675119396e-05,
+      "loss": 1.6573,
+      "step": 10044
+    },
+    {
+      "epoch": 0.25240505060627316,
+      "grad_norm": 0.8927319049835205,
+      "learning_rate": 4.0017751368378106e-05,
+      "loss": 1.6989,
+      "step": 10075
+    },
+    {
+      "epoch": 0.25318168153121556,
+      "grad_norm": 0.9308295249938965,
+      "learning_rate": 3.995052558835377e-05,
+      "loss": 1.6719,
+      "step": 10106
+    },
+    {
+      "epoch": 0.2539583124561579,
+      "grad_norm": 0.9917974472045898,
+      "learning_rate": 3.988313109368017e-05,
+      "loss": 1.666,
+      "step": 10137
+    },
+    {
+      "epoch": 0.2547349433811003,
+      "grad_norm": 0.9043029546737671,
+      "learning_rate": 3.981556864489504e-05,
+      "loss": 1.6813,
+      "step": 10168
+    },
+    {
+      "epoch": 0.2555115743060427,
+      "grad_norm": 0.9117201566696167,
+      "learning_rate": 3.974783900443142e-05,
+      "loss": 1.6817,
+      "step": 10199
+    },
+    {
+      "epoch": 0.25628820523098506,
+      "grad_norm": 0.9044283628463745,
+      "learning_rate": 3.9679942936609095e-05,
+      "loss": 1.6696,
+      "step": 10230
+    },
+    {
+      "epoch": 0.25706483615592746,
+      "grad_norm": 0.9261301755905151,
+      "learning_rate": 3.961188120762596e-05,
+      "loss": 1.68,
+      "step": 10261
+    },
+    {
+      "epoch": 0.2578414670808698,
+      "grad_norm": 0.8816866874694824,
+      "learning_rate": 3.954365458554938e-05,
+      "loss": 1.6679,
+      "step": 10292
+    },
+    {
+      "epoch": 0.2586180980058122,
+      "grad_norm": 0.941526472568512,
+      "learning_rate": 3.947526384030751e-05,
+      "loss": 1.6603,
+      "step": 10323
+    },
+    {
+      "epoch": 0.2593947289307546,
+      "grad_norm": 0.9269728064537048,
+      "learning_rate": 3.9406709743680624e-05,
+      "loss": 1.6653,
+      "step": 10354
+    },
+    {
+      "epoch": 0.26017135985569695,
+      "grad_norm": 0.9419822096824646,
+      "learning_rate": 3.9337993069292366e-05,
+      "loss": 1.658,
+      "step": 10385
+    },
+    {
+      "epoch": 0.26094799078063935,
+      "grad_norm": 0.918933629989624,
+      "learning_rate": 3.926911459260109e-05,
+      "loss": 1.677,
+      "step": 10416
+    },
+    {
+      "epoch": 0.2617246217055817,
+      "grad_norm": 0.9213050603866577,
+      "learning_rate": 3.920007509089102e-05,
+      "loss": 1.658,
+      "step": 10447
+    },
+    {
+      "epoch": 0.2625012526305241,
+      "grad_norm": 0.9633814096450806,
+      "learning_rate": 3.913087534326357e-05,
+      "loss": 1.6772,
+      "step": 10478
+    },
+    {
+      "epoch": 0.2632778835554665,
+      "grad_norm": 0.9100768566131592,
+      "learning_rate": 3.9061516130628475e-05,
+      "loss": 1.706,
+      "step": 10509
+    },
+    {
+      "epoch": 0.26405451448040884,
+      "grad_norm": 1.0582828521728516,
+      "learning_rate": 3.8991998235695025e-05,
+      "loss": 1.7019,
+      "step": 10540
+    },
+    {
+      "epoch": 0.26483114540535124,
+      "grad_norm": 0.9042608141899109,
+      "learning_rate": 3.8922322442963224e-05,
+      "loss": 1.6524,
+      "step": 10571
+    },
+    {
+      "epoch": 0.26560777633029364,
+      "grad_norm": 0.919127881526947,
+      "learning_rate": 3.885248953871491e-05,
+      "loss": 1.6575,
+      "step": 10602
+    },
+    {
+      "epoch": 0.266384407255236,
+      "grad_norm": 0.9282626509666443,
+      "learning_rate": 3.8782500311004915e-05,
+      "loss": 1.6579,
+      "step": 10633
+    },
+    {
+      "epoch": 0.2671610381801784,
+      "grad_norm": 0.9151068925857544,
+      "learning_rate": 3.871235554965218e-05,
+      "loss": 1.6572,
+      "step": 10664
+    },
+    {
+      "epoch": 0.26793766910512073,
+      "grad_norm": 0.9543127417564392,
+      "learning_rate": 3.864205604623078e-05,
+      "loss": 1.6689,
+      "step": 10695
+    },
+    {
+      "epoch": 0.26871430003006314,
+      "grad_norm": 0.8784031271934509,
+      "learning_rate": 3.857160259406107e-05,
+      "loss": 1.6566,
+      "step": 10726
+    },
+    {
+      "epoch": 0.26949093095500554,
+      "grad_norm": 0.9115248918533325,
+      "learning_rate": 3.8500995988200674e-05,
+      "loss": 1.6721,
+      "step": 10757
+    },
+    {
+      "epoch": 0.2702675618799479,
+      "grad_norm": 0.9072122573852539,
+      "learning_rate": 3.843023702543556e-05,
+      "loss": 1.6499,
+      "step": 10788
+    },
+    {
+      "epoch": 0.2710441928048903,
+      "grad_norm": 0.8933644890785217,
+      "learning_rate": 3.8359326504270984e-05,
+      "loss": 1.6329,
+      "step": 10819
+    },
+    {
+      "epoch": 0.27182082372983263,
+      "grad_norm": 0.9022906422615051,
+      "learning_rate": 3.828826522492255e-05,
+      "loss": 1.671,
+      "step": 10850
+    },
+    {
+      "epoch": 0.27259745465477503,
+      "grad_norm": 0.8892773389816284,
+      "learning_rate": 3.821705398930713e-05,
+      "loss": 1.6631,
+      "step": 10881
+    },
+    {
+      "epoch": 0.27337408557971743,
+      "grad_norm": 0.9231569170951843,
+      "learning_rate": 3.814569360103385e-05,
+      "loss": 1.671,
+      "step": 10912
+    },
+    {
+      "epoch": 0.2741507165046598,
+      "grad_norm": 0.8740375638008118,
+      "learning_rate": 3.807418486539499e-05,
+      "loss": 1.6686,
+      "step": 10943
+    },
+    {
+      "epoch": 0.2749273474296022,
+      "grad_norm": 0.9009575843811035,
+      "learning_rate": 3.80025285893569e-05,
+      "loss": 1.6546,
+      "step": 10974
+    },
+    {
+      "epoch": 0.2757039783545445,
+      "grad_norm": 0.9032784700393677,
+      "learning_rate": 3.793072558155093e-05,
+      "loss": 1.6573,
+      "step": 11005
+    },
+    {
+      "epoch": 0.2764806092794869,
+      "grad_norm": 0.9191836714744568,
+      "learning_rate": 3.785877665226426e-05,
+      "loss": 1.6581,
+      "step": 11036
+    },
+    {
+      "epoch": 0.2772572402044293,
+      "grad_norm": 0.8953864574432373,
+      "learning_rate": 3.778668261343079e-05,
+      "loss": 1.6704,
+      "step": 11067
+    },
+    {
+      "epoch": 0.27803387112937167,
+      "grad_norm": 0.9173053503036499,
+      "learning_rate": 3.771444427862192e-05,
+      "loss": 1.6488,
+      "step": 11098
+    },
+    {
+      "epoch": 0.27881050205431407,
+      "grad_norm": 0.890015184879303,
+      "learning_rate": 3.7642062463037465e-05,
+      "loss": 1.6553,
+      "step": 11129
+    },
+    {
+      "epoch": 0.2795871329792564,
+      "grad_norm": 0.8968738317489624,
+      "learning_rate": 3.7569537983496373e-05,
+      "loss": 1.662,
+      "step": 11160
+    },
+    {
+      "epoch": 0.2803637639041988,
+      "grad_norm": 0.9387725591659546,
+      "learning_rate": 3.749687165842753e-05,
+      "loss": 1.6485,
+      "step": 11191
+    },
+    {
+      "epoch": 0.2811403948291412,
+      "grad_norm": 0.8963536024093628,
+      "learning_rate": 3.7424064307860536e-05,
+      "loss": 1.6513,
+      "step": 11222
+    },
+    {
+      "epoch": 0.28191702575408356,
+      "grad_norm": 0.9358355402946472,
+      "learning_rate": 3.735111675341645e-05,
+      "loss": 1.6532,
+      "step": 11253
+    },
+    {
+      "epoch": 0.28269365667902596,
+      "grad_norm": 0.9153929948806763,
+      "learning_rate": 3.7278029818298524e-05,
+      "loss": 1.6702,
+      "step": 11284
+    },
+    {
+      "epoch": 0.2834702876039683,
+      "grad_norm": 0.9129534959793091,
+      "learning_rate": 3.720480432728287e-05,
+      "loss": 1.6789,
+      "step": 11315
+    },
+    {
+      "epoch": 0.2842469185289107,
+      "grad_norm": 0.8935201168060303,
+      "learning_rate": 3.71314411067092e-05,
+      "loss": 1.6271,
+      "step": 11346
+    },
+    {
+      "epoch": 0.2850235494538531,
+      "grad_norm": 0.8974719047546387,
+      "learning_rate": 3.70579409844715e-05,
+      "loss": 1.6771,
+      "step": 11377
+    },
+    {
+      "epoch": 0.28580018037879545,
+      "grad_norm": 0.9133878350257874,
+      "learning_rate": 3.698430479000865e-05,
+      "loss": 1.6748,
+      "step": 11408
+    },
+    {
+      "epoch": 0.28657681130373786,
+      "grad_norm": 0.9137885570526123,
+      "learning_rate": 3.691053335429509e-05,
+      "loss": 1.6518,
+      "step": 11439
+    },
+    {
+      "epoch": 0.28735344222868026,
+      "grad_norm": 0.852199912071228,
+      "learning_rate": 3.683662750983147e-05,
+      "loss": 1.6636,
+      "step": 11470
+    },
+    {
+      "epoch": 0.2881300731536226,
+      "grad_norm": 0.834365963935852,
+      "learning_rate": 3.676258809063518e-05,
+      "loss": 1.6437,
+      "step": 11501
+    },
+    {
+      "epoch": 0.288906704078565,
+      "grad_norm": 1.05277681350708,
+      "learning_rate": 3.6688415932231004e-05,
+      "loss": 1.6308,
+      "step": 11532
+    },
+    {
+      "epoch": 0.28968333500350735,
+      "grad_norm": 0.8785567283630371,
+      "learning_rate": 3.661411187164166e-05,
+      "loss": 1.6608,
+      "step": 11563
+    },
+    {
+      "epoch": 0.29045996592844975,
+      "grad_norm": 0.8666633367538452,
+      "learning_rate": 3.65396767473784e-05,
+      "loss": 1.6747,
+      "step": 11594
+    },
+    {
+      "epoch": 0.29123659685339215,
+      "grad_norm": 0.8817443251609802,
+      "learning_rate": 3.6465111399431465e-05,
+      "loss": 1.6376,
+      "step": 11625
+    },
+    {
+      "epoch": 0.2920132277783345,
+      "grad_norm": 0.9468926191329956,
+      "learning_rate": 3.6390416669260674e-05,
+      "loss": 1.6668,
+      "step": 11656
+    },
+    {
+      "epoch": 0.2927898587032769,
+      "grad_norm": 0.8657639026641846,
+      "learning_rate": 3.63155933997859e-05,
+      "loss": 1.6332,
+      "step": 11687
+    },
+    {
+      "epoch": 0.29356648962821924,
+      "grad_norm": 1.0250166654586792,
+      "learning_rate": 3.624064243537758e-05,
+      "loss": 1.6577,
+      "step": 11718
+    },
+    {
+      "epoch": 0.29434312055316164,
+      "grad_norm": 0.885832667350769,
+      "learning_rate": 3.616556462184716e-05,
+      "loss": 1.6532,
+      "step": 11749
+    },
+    {
+      "epoch": 0.29511975147810404,
+      "grad_norm": 0.9171744585037231,
+      "learning_rate": 3.609036080643755e-05,
+      "loss": 1.648,
+      "step": 11780
+    },
+    {
+      "epoch": 0.2958963824030464,
+      "grad_norm": 0.9100250601768494,
+      "learning_rate": 3.60150318378136e-05,
+      "loss": 1.6495,
+      "step": 11811
+    },
+    {
+      "epoch": 0.2966730133279888,
+      "grad_norm": 0.876293957233429,
+      "learning_rate": 3.5939578566052465e-05,
+      "loss": 1.6677,
+      "step": 11842
+    },
+    {
+      "epoch": 0.29744964425293113,
+      "grad_norm": 0.9444800615310669,
+      "learning_rate": 3.586400184263408e-05,
+      "loss": 1.6341,
+      "step": 11873
+    },
+    {
+      "epoch": 0.29822627517787353,
+      "grad_norm": 0.8957802057266235,
+      "learning_rate": 3.578830252043148e-05,
+      "loss": 1.6327,
+      "step": 11904
+    },
+    {
+      "epoch": 0.29900290610281594,
+      "grad_norm": 0.8906880617141724,
+      "learning_rate": 3.571248145370125e-05,
+      "loss": 1.6347,
+      "step": 11935
+    },
+    {
+      "epoch": 0.2997795370277583,
+      "grad_norm": 0.931266188621521,
+      "learning_rate": 3.5636539498073794e-05,
+      "loss": 1.6623,
+      "step": 11966
+    },
+    {
+      "epoch": 0.3005561679527007,
+      "grad_norm": 0.9208303093910217,
+      "learning_rate": 3.556047751054378e-05,
+      "loss": 1.6598,
+      "step": 11997
+    },
+    {
+      "epoch": 0.301332798877643,
+      "grad_norm": 0.8807421922683716,
+      "learning_rate": 3.548429634946039e-05,
+      "loss": 1.6563,
+      "step": 12028
+    },
+    {
+      "epoch": 0.3021094298025854,
+      "grad_norm": 0.9276000261306763,
+      "learning_rate": 3.540799687451768e-05,
+      "loss": 1.6341,
+      "step": 12059
+    },
+    {
+      "epoch": 0.30288606072752783,
+      "grad_norm": 0.8768136501312256,
+      "learning_rate": 3.533157994674485e-05,
+      "loss": 1.6545,
+      "step": 12090
+    },
+    {
+      "epoch": 0.3036626916524702,
+      "grad_norm": 3.2070443630218506,
+      "learning_rate": 3.5255046428496546e-05,
+      "loss": 1.6377,
+      "step": 12121
+    },
+    {
+      "epoch": 0.3044393225774126,
+      "grad_norm": 0.9450961947441101,
+      "learning_rate": 3.517839718344311e-05,
+      "loss": 1.6235,
+      "step": 12152
+    },
+    {
+      "epoch": 0.3052159535023549,
+      "grad_norm": 0.8712328672409058,
+      "learning_rate": 3.510163307656086e-05,
+      "loss": 1.6238,
+      "step": 12183
+    },
+    {
+      "epoch": 0.3059925844272973,
+      "grad_norm": 0.8799425959587097,
+      "learning_rate": 3.5024754974122324e-05,
+      "loss": 1.6406,
+      "step": 12214
+    },
+    {
+      "epoch": 0.3067692153522397,
+      "grad_norm": 0.8861176371574402,
+      "learning_rate": 3.494776374368643e-05,
+      "loss": 1.6446,
+      "step": 12245
+    },
+    {
+      "epoch": 0.30754584627718207,
+      "grad_norm": 0.8820211887359619,
+      "learning_rate": 3.4870660254088724e-05,
+      "loss": 1.6449,
+      "step": 12276
+    },
+    {
+      "epoch": 0.30832247720212447,
+      "grad_norm": 0.8866766095161438,
+      "learning_rate": 3.479344537543164e-05,
+      "loss": 1.6505,
+      "step": 12307
+    },
+    {
+      "epoch": 0.3090991081270668,
+      "grad_norm": 0.8629591464996338,
+      "learning_rate": 3.4716119979074565e-05,
+      "loss": 1.6279,
+      "step": 12338
+    },
+    {
+      "epoch": 0.3098757390520092,
+      "grad_norm": 0.9161331057548523,
+      "learning_rate": 3.463868493762412e-05,
+      "loss": 1.6398,
+      "step": 12369
+    },
+    {
+      "epoch": 0.3106523699769516,
+      "grad_norm": 0.8870587944984436,
+      "learning_rate": 3.456114112492418e-05,
+      "loss": 1.6564,
+      "step": 12400
+    },
+    {
+      "epoch": 0.31142900090189396,
+      "grad_norm": 0.9587643146514893,
+      "learning_rate": 3.4483489416046164e-05,
+      "loss": 1.6406,
+      "step": 12431
+    },
+    {
+      "epoch": 0.31220563182683636,
+      "grad_norm": 0.8947318196296692,
+      "learning_rate": 3.440573068727905e-05,
+      "loss": 1.6483,
+      "step": 12462
+    },
+    {
+      "epoch": 0.31298226275177876,
+      "grad_norm": 0.9000875353813171,
+      "learning_rate": 3.4327865816119495e-05,
+      "loss": 1.6526,
+      "step": 12493
+    },
+    {
+      "epoch": 0.3137588936767211,
+      "grad_norm": 0.8959850072860718,
+      "learning_rate": 3.4249895681262025e-05,
+      "loss": 1.6443,
+      "step": 12524
+    },
+    {
+      "epoch": 0.3145355246016635,
+      "grad_norm": 0.9080119132995605,
+      "learning_rate": 3.417182116258899e-05,
+      "loss": 1.6469,
+      "step": 12555
+    },
+    {
+      "epoch": 0.31531215552660585,
+      "grad_norm": 0.8936130404472351,
+      "learning_rate": 3.409364314116074e-05,
+      "loss": 1.6353,
+      "step": 12586
+    },
+    {
+      "epoch": 0.31608878645154825,
+      "grad_norm": 0.8636283874511719,
+      "learning_rate": 3.401536249920559e-05,
+      "loss": 1.6426,
+      "step": 12617
+    },
+    {
+      "epoch": 0.31686541737649065,
+      "grad_norm": 0.8834002614021301,
+      "learning_rate": 3.393698012010998e-05,
+      "loss": 1.6294,
+      "step": 12648
+    },
+    {
+      "epoch": 0.317642048301433,
+      "grad_norm": 0.8980857133865356,
+      "learning_rate": 3.385849688840839e-05,
+      "loss": 1.6266,
+      "step": 12679
+    },
+    {
+      "epoch": 0.3184186792263754,
+      "grad_norm": 0.9084813594818115,
+      "learning_rate": 3.3779913689773414e-05,
+      "loss": 1.6354,
+      "step": 12710
+    },
+    {
+      "epoch": 0.31919531015131775,
+      "grad_norm": 0.8725720643997192,
+      "learning_rate": 3.370123141100578e-05,
+      "loss": 1.601,
+      "step": 12741
+    },
+    {
+      "epoch": 0.31997194107626015,
+      "grad_norm": 0.9047378301620483,
+      "learning_rate": 3.3622450940024305e-05,
+      "loss": 1.6413,
+      "step": 12772
+    },
+    {
+      "epoch": 0.32074857200120255,
+      "grad_norm": 0.8764498233795166,
+      "learning_rate": 3.35435731658559e-05,
+      "loss": 1.6401,
+      "step": 12803
+    },
+    {
+      "epoch": 0.3215252029261449,
+      "grad_norm": 0.8859270215034485,
+      "learning_rate": 3.346459897862552e-05,
+      "loss": 1.6382,
+      "step": 12834
+    },
+    {
+      "epoch": 0.3223018338510873,
+      "grad_norm": 0.9855139255523682,
+      "learning_rate": 3.338552926954613e-05,
+      "loss": 1.6595,
+      "step": 12865
+    },
+    {
+      "epoch": 0.32307846477602964,
+      "grad_norm": 0.8921546339988708,
+      "learning_rate": 3.330636493090868e-05,
+      "loss": 1.6389,
+      "step": 12896
+    },
+    {
+      "epoch": 0.32385509570097204,
+      "grad_norm": 0.9123478531837463,
+      "learning_rate": 3.322710685607193e-05,
+      "loss": 1.6365,
+      "step": 12927
+    },
+    {
+      "epoch": 0.32463172662591444,
+      "grad_norm": 0.8619018793106079,
+      "learning_rate": 3.314775593945251e-05,
+      "loss": 1.6286,
+      "step": 12958
+    },
+    {
+      "epoch": 0.3254083575508568,
+      "grad_norm": 0.927751898765564,
+      "learning_rate": 3.3068313076514714e-05,
+      "loss": 1.6252,
+      "step": 12989
+    },
+    {
+      "epoch": 0.3261849884757992,
+      "grad_norm": 0.8673629760742188,
+      "learning_rate": 3.298877916376047e-05,
+      "loss": 1.6235,
+      "step": 13020
+    },
+    {
+      "epoch": 0.32696161940074153,
+      "grad_norm": 0.8902471661567688,
+      "learning_rate": 3.290915509871915e-05,
+      "loss": 1.6466,
+      "step": 13051
+    },
+    {
+      "epoch": 0.32773825032568393,
+      "grad_norm": 0.8668501377105713,
+      "learning_rate": 3.282944177993753e-05,
+      "loss": 1.6274,
+      "step": 13082
+    },
+    {
+      "epoch": 0.32851488125062633,
+      "grad_norm": 0.9352705478668213,
+      "learning_rate": 3.274964010696957e-05,
+      "loss": 1.6329,
+      "step": 13113
+    },
+    {
+      "epoch": 0.3292915121755687,
+      "grad_norm": 0.8960979580879211,
+      "learning_rate": 3.266975098036629e-05,
+      "loss": 1.6291,
+      "step": 13144
+    },
+    {
+      "epoch": 0.3300681431005111,
+      "grad_norm": 0.903521716594696,
+      "learning_rate": 3.258977530166562e-05,
+      "loss": 1.6454,
+      "step": 13175
+    },
+    {
+      "epoch": 0.3308447740254534,
+      "grad_norm": 0.9136938452720642,
+      "learning_rate": 3.250971397338227e-05,
+      "loss": 1.6447,
+      "step": 13206
+    },
+    {
+      "epoch": 0.3316214049503958,
+      "grad_norm": 0.9085225462913513,
+      "learning_rate": 3.2429567898997404e-05,
+      "loss": 1.6452,
+      "step": 13237
+    },
+    {
+      "epoch": 0.3323980358753382,
+      "grad_norm": 0.8968013525009155,
+      "learning_rate": 3.234933798294859e-05,
+      "loss": 1.6281,
+      "step": 13268
+    },
+    {
+      "epoch": 0.33317466680028057,
+      "grad_norm": 0.9228054881095886,
+      "learning_rate": 3.2269025130619535e-05,
+      "loss": 1.6299,
+      "step": 13299
+    },
+    {
+      "epoch": 0.333951297725223,
+      "grad_norm": 0.886323869228363,
+      "learning_rate": 3.218863024832985e-05,
+      "loss": 1.6367,
+      "step": 13330
+    },
+    {
+      "epoch": 0.3347279286501654,
+      "grad_norm": 0.9017085433006287,
+      "learning_rate": 3.2108154243324864e-05,
+      "loss": 1.6383,
+      "step": 13361
+    },
+    {
+      "epoch": 0.3355045595751077,
+      "grad_norm": 0.8789897561073303,
+      "learning_rate": 3.2027598023765345e-05,
+      "loss": 1.6162,
+      "step": 13392
+    },
+    {
+      "epoch": 0.3362811905000501,
+      "grad_norm": 0.8568314909934998,
+      "learning_rate": 3.194696249871729e-05,
+      "loss": 1.6431,
+      "step": 13423
+    },
+    {
+      "epoch": 0.33705782142499247,
+      "grad_norm": 0.8744024634361267,
+      "learning_rate": 3.186624857814164e-05,
+      "loss": 1.6248,
+      "step": 13454
+    },
+    {
+      "epoch": 0.33783445234993487,
+      "grad_norm": 0.920241117477417,
+      "learning_rate": 3.178545717288401e-05,
+      "loss": 1.6213,
+      "step": 13485
+    },
+    {
+      "epoch": 0.33861108327487727,
+      "grad_norm": 0.853951096534729,
+      "learning_rate": 3.170458919466444e-05,
+      "loss": 1.6171,
+      "step": 13516
+    },
+    {
+      "epoch": 0.3393877141998196,
+      "grad_norm": 0.9314168095588684,
+      "learning_rate": 3.1623645556067063e-05,
+      "loss": 1.6319,
+      "step": 13547
+    },
+    {
+      "epoch": 0.340164345124762,
+      "grad_norm": 0.8591824173927307,
+      "learning_rate": 3.154262717052985e-05,
+      "loss": 1.6338,
+      "step": 13578
+    },
+    {
+      "epoch": 0.34094097604970436,
+      "grad_norm": 0.8584831357002258,
+      "learning_rate": 3.146153495233426e-05,
+      "loss": 1.6153,
+      "step": 13609
+    },
+    {
+      "epoch": 0.34171760697464676,
+      "grad_norm": 0.9188579320907593,
+      "learning_rate": 3.1380369816594944e-05,
+      "loss": 1.6341,
+      "step": 13640
+    },
+    {
+      "epoch": 0.34249423789958916,
+      "grad_norm": 0.8540886640548706,
+      "learning_rate": 3.129913267924946e-05,
+      "loss": 1.6237,
+      "step": 13671
+    },
+    {
+      "epoch": 0.3432708688245315,
+      "grad_norm": 0.8978856801986694,
+      "learning_rate": 3.121782445704782e-05,
+      "loss": 1.6333,
+      "step": 13702
+    },
+    {
+      "epoch": 0.3440474997494739,
+      "grad_norm": 0.9080212116241455,
+      "learning_rate": 3.11364460675423e-05,
+      "loss": 1.6107,
+      "step": 13733
+    },
+    {
+      "epoch": 0.34482413067441625,
+      "grad_norm": 0.8724472522735596,
+      "learning_rate": 3.1054998429076934e-05,
+      "loss": 1.6486,
+      "step": 13764
+    },
+    {
+      "epoch": 0.34560076159935865,
+      "grad_norm": 0.8924047350883484,
+      "learning_rate": 3.097348246077728e-05,
+      "loss": 1.6151,
+      "step": 13795
+    },
+    {
+      "epoch": 0.34637739252430105,
+      "grad_norm": 0.873216450214386,
+      "learning_rate": 3.0891899082539924e-05,
+      "loss": 1.6082,
+      "step": 13826
+    },
+    {
+      "epoch": 0.3471540234492434,
+      "grad_norm": 0.8874927759170532,
+      "learning_rate": 3.0810249215022233e-05,
+      "loss": 1.6198,
+      "step": 13857
+    },
+    {
+      "epoch": 0.3479306543741858,
+      "grad_norm": 0.8605682849884033,
+      "learning_rate": 3.0728533779631865e-05,
+      "loss": 1.6325,
+      "step": 13888
+    },
+    {
+      "epoch": 0.34870728529912814,
+      "grad_norm": 0.8890742063522339,
+      "learning_rate": 3.064675369851637e-05,
+      "loss": 1.6311,
+      "step": 13919
+    },
+    {
+      "epoch": 0.34948391622407055,
+      "grad_norm": 0.9032161235809326,
+      "learning_rate": 3.056490989455289e-05,
+      "loss": 1.6331,
+      "step": 13950
+    },
+    {
+      "epoch": 0.35026054714901295,
+      "grad_norm": 0.8931726813316345,
+      "learning_rate": 3.0483003291337596e-05,
+      "loss": 1.6353,
+      "step": 13981
+    },
+    {
+      "epoch": 0.3510371780739553,
+      "grad_norm": 0.9311168193817139,
+      "learning_rate": 3.040103481317539e-05,
+      "loss": 1.6269,
+      "step": 14012
+    },
+    {
+      "epoch": 0.3518138089988977,
+      "grad_norm": 0.875851035118103,
+      "learning_rate": 3.03190053850694e-05,
+      "loss": 1.6252,
+      "step": 14043
+    },
+    {
+      "epoch": 0.35259043992384004,
+      "grad_norm": 0.817078173160553,
+      "learning_rate": 3.0236915932710573e-05,
+      "loss": 1.6235,
+      "step": 14074
+    },
+    {
+      "epoch": 0.35336707084878244,
+      "grad_norm": 0.9579086899757385,
+      "learning_rate": 3.0154767382467232e-05,
+      "loss": 1.6186,
+      "step": 14105
+    },
+    {
+      "epoch": 0.35414370177372484,
+      "grad_norm": 0.8686785101890564,
+      "learning_rate": 3.0072560661374582e-05,
+      "loss": 1.6306,
+      "step": 14136
+    },
+    {
+      "epoch": 0.3549203326986672,
+      "grad_norm": 0.8935508728027344,
+      "learning_rate": 2.999029669712431e-05,
+      "loss": 1.6241,
+      "step": 14167
+    },
+    {
+      "epoch": 0.3556969636236096,
+      "grad_norm": 0.9693512320518494,
+      "learning_rate": 2.990797641805408e-05,
+      "loss": 1.599,
+      "step": 14198
+    },
+    {
+      "epoch": 0.356473594548552,
+      "grad_norm": 0.8598000407218933,
+      "learning_rate": 2.982560075313704e-05,
+      "loss": 1.6149,
+      "step": 14229
+    },
+    {
+      "epoch": 0.35725022547349433,
+      "grad_norm": 0.8287764191627502,
+      "learning_rate": 2.9743170631971368e-05,
+      "loss": 1.5822,
+      "step": 14260
+    },
+    {
+      "epoch": 0.35802685639843673,
+      "grad_norm": 0.8897140026092529,
+      "learning_rate": 2.9660686984769792e-05,
+      "loss": 1.5948,
+      "step": 14291
+    },
+    {
+      "epoch": 0.3588034873233791,
+      "grad_norm": 0.883812665939331,
+      "learning_rate": 2.9578150742349047e-05,
+      "loss": 1.6301,
+      "step": 14322
+    },
+    {
+      "epoch": 0.3595801182483215,
+      "grad_norm": 0.8680452704429626,
+      "learning_rate": 2.949556283611942e-05,
+      "loss": 1.6246,
+      "step": 14353
+    },
+    {
+      "epoch": 0.3603567491732639,
+      "grad_norm": 0.9011819362640381,
+      "learning_rate": 2.9412924198074206e-05,
+      "loss": 1.6075,
+      "step": 14384
+    },
+    {
+      "epoch": 0.3611333800982062,
+      "grad_norm": 0.8731768131256104,
+      "learning_rate": 2.9330235760779208e-05,
+      "loss": 1.6206,
+      "step": 14415
+    },
+    {
+      "epoch": 0.3619100110231486,
+      "grad_norm": 0.8883015513420105,
+      "learning_rate": 2.9247498457362188e-05,
+      "loss": 1.6127,
+      "step": 14446
+    },
+    {
+      "epoch": 0.36268664194809097,
+      "grad_norm": 0.9235113859176636,
+      "learning_rate": 2.9164713221502373e-05,
+      "loss": 1.6269,
+      "step": 14477
+    },
+    {
+      "epoch": 0.36346327287303337,
+      "grad_norm": 0.9029502272605896,
+      "learning_rate": 2.9081880987419912e-05,
+      "loss": 1.6181,
+      "step": 14508
+    },
+    {
+      "epoch": 0.3642399037979758,
+      "grad_norm": 0.8695984482765198,
+      "learning_rate": 2.8999002689865296e-05,
+      "loss": 1.6123,
+      "step": 14539
+    },
+    {
+      "epoch": 0.3650165347229181,
+      "grad_norm": 0.912814199924469,
+      "learning_rate": 2.8916079264108852e-05,
+      "loss": 1.6099,
+      "step": 14570
+    },
+    {
+      "epoch": 0.3657931656478605,
+      "grad_norm": 0.8456264138221741,
+      "learning_rate": 2.883311164593017e-05,
+      "loss": 1.6321,
+      "step": 14601
+    },
+    {
+      "epoch": 0.36656979657280286,
+      "grad_norm": 0.9382089376449585,
+      "learning_rate": 2.875010077160754e-05,
+      "loss": 1.6154,
+      "step": 14632
+    },
+    {
+      "epoch": 0.36734642749774526,
+      "grad_norm": 0.9086084961891174,
+      "learning_rate": 2.866704757790741e-05,
+      "loss": 1.637,
+      "step": 14663
+    },
+    {
+      "epoch": 0.36812305842268767,
+      "grad_norm": 0.8620360493659973,
+      "learning_rate": 2.858395300207376e-05,
+      "loss": 1.613,
+      "step": 14694
+    },
+    {
+      "epoch": 0.36889968934763,
+      "grad_norm": 0.845750629901886,
+      "learning_rate": 2.8500817981817607e-05,
+      "loss": 1.6099,
+      "step": 14725
+    },
+    {
+      "epoch": 0.3696763202725724,
+      "grad_norm": 0.8987907767295837,
+      "learning_rate": 2.8417643455306336e-05,
+      "loss": 1.617,
+      "step": 14756
+    },
+    {
+      "epoch": 0.37045295119751476,
+      "grad_norm": 0.8765819668769836,
+      "learning_rate": 2.8334430361153185e-05,
+      "loss": 1.6304,
+      "step": 14787
+    },
+    {
+      "epoch": 0.37122958212245716,
+      "grad_norm": 0.8677089214324951,
+      "learning_rate": 2.8251179638406612e-05,
+      "loss": 1.6359,
+      "step": 14818
+    },
+    {
+      "epoch": 0.37200621304739956,
+      "grad_norm": 0.8903617858886719,
+      "learning_rate": 2.8167892226539704e-05,
+      "loss": 1.6049,
+      "step": 14849
+    },
+    {
+      "epoch": 0.3727828439723419,
+      "grad_norm": 0.8951026797294617,
+      "learning_rate": 2.8084569065439588e-05,
+      "loss": 1.6486,
+      "step": 14880
+    },
+    {
+      "epoch": 0.3735594748972843,
+      "grad_norm": 0.9354173541069031,
+      "learning_rate": 2.8001211095396807e-05,
+      "loss": 1.6158,
+      "step": 14911
+    },
+    {
+      "epoch": 0.37433610582222665,
+      "grad_norm": 0.8839575052261353,
+      "learning_rate": 2.791781925709473e-05,
+      "loss": 1.6058,
+      "step": 14942
+    },
+    {
+      "epoch": 0.37511273674716905,
+      "grad_norm": 0.9768513441085815,
+      "learning_rate": 2.7834394491598908e-05,
+      "loss": 1.6282,
+      "step": 14973
+    },
+    {
+      "epoch": 0.37588936767211145,
+      "grad_norm": 0.8685802817344666,
+      "learning_rate": 2.7750937740346485e-05,
+      "loss": 1.6089,
+      "step": 15004
+    },
+    {
+      "epoch": 0.3766659985970538,
+      "grad_norm": 0.8572190999984741,
+      "learning_rate": 2.7667449945135564e-05,
+      "loss": 1.6108,
+      "step": 15035
+    },
+    {
+      "epoch": 0.3774426295219962,
+      "grad_norm": 0.8866077661514282,
+      "learning_rate": 2.7583932048114557e-05,
+      "loss": 1.5994,
+      "step": 15066
+    },
+    {
+      "epoch": 0.37821926044693854,
+      "grad_norm": 0.8617647886276245,
+      "learning_rate": 2.7500384991771587e-05,
+      "loss": 1.5863,
+      "step": 15097
+    },
+    {
+      "epoch": 0.37899589137188094,
+      "grad_norm": 0.859278678894043,
+      "learning_rate": 2.7416809718923825e-05,
+      "loss": 1.5955,
+      "step": 15128
+    },
+    {
+      "epoch": 0.37977252229682334,
+      "grad_norm": 0.8811472654342651,
+      "learning_rate": 2.7333207172706864e-05,
+      "loss": 1.6017,
+      "step": 15159
+    },
+    {
+      "epoch": 0.3805491532217657,
+      "grad_norm": 1.3888036012649536,
+      "learning_rate": 2.7249578296564088e-05,
+      "loss": 1.6213,
+      "step": 15190
+    },
+    {
+      "epoch": 0.3813257841467081,
+      "grad_norm": 0.8886885643005371,
+      "learning_rate": 2.7165924034235973e-05,
+      "loss": 1.6179,
+      "step": 15221
+    },
+    {
+      "epoch": 0.3821024150716505,
+      "grad_norm": 0.9003123044967651,
+      "learning_rate": 2.708224532974953e-05,
+      "loss": 1.6009,
+      "step": 15252
+    }
+  ],
+  "logging_steps": 31,
+  "max_steps": 30517,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 7630,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 1.065842709765292e+19,
+  "train_batch_size": 16,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-15260/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:465f927e21695e17129aa36be8b4481e804b293136634c25aeb3429480a9a6e5
+size 5304

checkpoint-22890/config.json

@@ -0,0 +1,29 @@
+{
+  "_name_or_path": "/mnt/parscratch/users/acp23ay/private/models/Qwen2.5-7B-my-madlad-mean/",
+  "architectures": [
+    "Qwen2ForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 151643,
+  "eos_token_id": 151643,
+  "hidden_act": "silu",
+  "hidden_size": 3584,
+  "initializer_range": 0.02,
+  "intermediate_size": 18944,
+  "max_position_embeddings": 131072,
+  "max_window_layers": 28,
+  "model_type": "qwen2",
+  "num_attention_heads": 28,
+  "num_hidden_layers": 28,
+  "num_key_value_heads": 4,
+  "rms_norm_eps": 1e-06,
+  "rope_theta": 1000000.0,
+  "sliding_window": null,
+  "tie_word_embeddings": false,
+  "torch_dtype": "float32",
+  "transformers_version": "4.44.0.dev0",
+  "use_cache": true,
+  "use_mrope": false,
+  "use_sliding_window": false,
+  "vocab_size": 161672
+}

checkpoint-22890/generation_config.json

@@ -0,0 +1,6 @@
+{
+  "bos_token_id": 151643,
+  "eos_token_id": 151643,
+  "max_new_tokens": 2048,
+  "transformers_version": "4.44.0.dev0"
+}

checkpoint-22890/model-00001-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f43d23833e7f3e1adf6fa01d8c399ddf65719a1ada6660f2d768f083dbed4260
+size 4842817304

checkpoint-22890/model-00002-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:855851c22a0213c322c44b2d933eed67bc9ff8215091c3eb491ec94b662ffadd
+size 4998852224

checkpoint-22890/model-00003-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0c748c56f7d09ef7934bc2e6d2bbb3a6b4fec8beeb1759acb8db07b32764bd38
+size 4984124304

checkpoint-22890/model-00004-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:089cf99962fd351ceb590220ea351f1d18937b35376077982b276416f768317a
+size 4932743992

checkpoint-22890/model-00005-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b61bee4a8b5e41f744ec0820def58c39c0c333e6787b7e24146468348f082a0
+size 4998852296

checkpoint-22890/model-00006-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:628d3159a71b880bfa279212c50c2771f6550b1b46a6f458a43987c369e3c6e2
+size 3662865184

checkpoint-22890/model-00007-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:117b861c7d172b0172e758265cc41293553e13eafd6684c3a707607af0b90514
+size 2317729920

checkpoint-22890/model.safetensors.index.json

@@ -0,0 +1,346 @@
+{
+  "metadata": {
+    "total_size": 30737946624
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00007-of-00007.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.self_attn.k_proj.bias": "model-00001-of-00007.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.self_attn.q_proj.bias": "model-00001-of-00007.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.self_attn.v_proj.bias": "model-00001-of-00007.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.self_attn.k_proj.bias": "model-00001-of-00007.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.self_attn.q_proj.bias": "model-00001-of-00007.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.self_attn.v_proj.bias": "model-00001-of-00007.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.self_attn.k_proj.bias": "model-00003-of-00007.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.self_attn.q_proj.bias": "model-00003-of-00007.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.self_attn.v_proj.bias": "model-00003-of-00007.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.self_attn.k_proj.bias": "model-00003-of-00007.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.self_attn.q_proj.bias": "model-00003-of-00007.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.self_attn.v_proj.bias": "model-00003-of-00007.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.self_attn.k_proj.bias": "model-00003-of-00007.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.self_attn.q_proj.bias": "model-00003-of-00007.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.self_attn.v_proj.bias": "model-00003-of-00007.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.13.self_attn.k_proj.bias": "model-00003-of-00007.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.self_attn.q_proj.bias": "model-00003-of-00007.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.self_attn.v_proj.bias": "model-00003-of-00007.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.self_attn.k_proj.bias": "model-00004-of-00007.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.self_attn.q_proj.bias": "model-00004-of-00007.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.self_attn.v_proj.bias": "model-00004-of-00007.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.self_attn.k_proj.bias": "model-00004-of-00007.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.self_attn.q_proj.bias": "model-00004-of-00007.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.self_attn.v_proj.bias": "model-00004-of-00007.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.self_attn.k_proj.bias": "model-00004-of-00007.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.self_attn.q_proj.bias": "model-00004-of-00007.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.self_attn.v_proj.bias": "model-00004-of-00007.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.self_attn.k_proj.bias": "model-00004-of-00007.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.self_attn.q_proj.bias": "model-00004-of-00007.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.self_attn.v_proj.bias": "model-00004-of-00007.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.18.self_attn.k_proj.bias": "model-00004-of-00007.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.self_attn.q_proj.bias": "model-00004-of-00007.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.self_attn.v_proj.bias": "model-00004-of-00007.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.19.self_attn.k_proj.bias": "model-00005-of-00007.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.19.self_attn.q_proj.bias": "model-00005-of-00007.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.19.self_attn.v_proj.bias": "model-00005-of-00007.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.2.self_attn.k_proj.bias": "model-00001-of-00007.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.self_attn.q_proj.bias": "model-00001-of-00007.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.self_attn.v_proj.bias": "model-00001-of-00007.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.20.self_attn.k_proj.bias": "model-00005-of-00007.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.20.self_attn.q_proj.bias": "model-00005-of-00007.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.20.self_attn.v_proj.bias": "model-00005-of-00007.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.self_attn.k_proj.bias": "model-00005-of-00007.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.self_attn.q_proj.bias": "model-00005-of-00007.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.self_attn.v_proj.bias": "model-00005-of-00007.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.self_attn.k_proj.bias": "model-00005-of-00007.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.self_attn.q_proj.bias": "model-00005-of-00007.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.self_attn.v_proj.bias": "model-00005-of-00007.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.self_attn.k_proj.bias": "model-00005-of-00007.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.self_attn.q_proj.bias": "model-00005-of-00007.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.self_attn.v_proj.bias": "model-00005-of-00007.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.24.self_attn.k_proj.bias": "model-00005-of-00007.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.24.self_attn.q_proj.bias": "model-00005-of-00007.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.self_attn.v_proj.bias": "model-00005-of-00007.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.25.self_attn.k_proj.bias": "model-00006-of-00007.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.25.self_attn.q_proj.bias": "model-00006-of-00007.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.25.self_attn.v_proj.bias": "model-00006-of-00007.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.self_attn.k_proj.bias": "model-00006-of-00007.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.self_attn.q_proj.bias": "model-00006-of-00007.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.self_attn.v_proj.bias": "model-00006-of-00007.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.self_attn.k_proj.bias": "model-00006-of-00007.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.self_attn.q_proj.bias": "model-00006-of-00007.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.self_attn.v_proj.bias": "model-00006-of-00007.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.self_attn.k_proj.bias": "model-00002-of-00007.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.self_attn.q_proj.bias": "model-00002-of-00007.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.self_attn.v_proj.bias": "model-00002-of-00007.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.self_attn.k_proj.bias": "model-00002-of-00007.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.self_attn.q_proj.bias": "model-00002-of-00007.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.self_attn.v_proj.bias": "model-00002-of-00007.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.self_attn.k_proj.bias": "model-00002-of-00007.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.self_attn.q_proj.bias": "model-00002-of-00007.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.self_attn.v_proj.bias": "model-00002-of-00007.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.self_attn.k_proj.bias": "model-00002-of-00007.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.self_attn.q_proj.bias": "model-00002-of-00007.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.self_attn.v_proj.bias": "model-00002-of-00007.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.self_attn.k_proj.bias": "model-00002-of-00007.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.self_attn.q_proj.bias": "model-00002-of-00007.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.self_attn.v_proj.bias": "model-00002-of-00007.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.8.self_attn.k_proj.bias": "model-00002-of-00007.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.8.self_attn.q_proj.bias": "model-00002-of-00007.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.8.self_attn.v_proj.bias": "model-00002-of-00007.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.self_attn.k_proj.bias": "model-00003-of-00007.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.self_attn.q_proj.bias": "model-00003-of-00007.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.self_attn.v_proj.bias": "model-00003-of-00007.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "model.norm.weight": "model-00006-of-00007.safetensors"
+  }
+}

checkpoint-22890/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d8ee29a59c76aea58adea3d45489e9fa932b35a672970af9b9065102792f39d2
+size 16728842070

checkpoint-22890/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9d9cd6a0487226e5bd30d1846894c82af483733ab4381b75bae9c0745e05d405
+size 14244

checkpoint-22890/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9cceae98d85860224f83926a9c2d0797a9f4411abc8a933e66f23226e9ba794a
+size 1064

checkpoint-22890/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:465f927e21695e17129aa36be8b4481e804b293136634c25aeb3429480a9a6e5
+size 5304

checkpoint-30517/config.json

@@ -0,0 +1,29 @@
+{
+  "_name_or_path": "/mnt/parscratch/users/acp23ay/private/models/Qwen2.5-7B-my-madlad-mean/",
+  "architectures": [
+    "Qwen2ForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 151643,
+  "eos_token_id": 151643,
+  "hidden_act": "silu",
+  "hidden_size": 3584,
+  "initializer_range": 0.02,
+  "intermediate_size": 18944,
+  "max_position_embeddings": 131072,
+  "max_window_layers": 28,
+  "model_type": "qwen2",
+  "num_attention_heads": 28,
+  "num_hidden_layers": 28,
+  "num_key_value_heads": 4,
+  "rms_norm_eps": 1e-06,
+  "rope_theta": 1000000.0,
+  "sliding_window": null,
+  "tie_word_embeddings": false,
+  "torch_dtype": "float32",
+  "transformers_version": "4.44.0.dev0",
+  "use_cache": true,
+  "use_mrope": false,
+  "use_sliding_window": false,
+  "vocab_size": 161672
+}

checkpoint-30517/generation_config.json

@@ -0,0 +1,6 @@
+{
+  "bos_token_id": 151643,
+  "eos_token_id": 151643,
+  "max_new_tokens": 2048,
+  "transformers_version": "4.44.0.dev0"
+}

checkpoint-30517/model-00001-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f9fb075512f466c494c69b3117f2976cd523d932bb583ff1874aa7afb804ec77
+size 4842817304

checkpoint-30517/model-00002-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:855851c22a0213c322c44b2d933eed67bc9ff8215091c3eb491ec94b662ffadd
+size 4998852224

checkpoint-30517/model-00003-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0c748c56f7d09ef7934bc2e6d2bbb3a6b4fec8beeb1759acb8db07b32764bd38
+size 4984124304

checkpoint-30517/model-00004-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:089cf99962fd351ceb590220ea351f1d18937b35376077982b276416f768317a
+size 4932743992

checkpoint-30517/model-00005-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b61bee4a8b5e41f744ec0820def58c39c0c333e6787b7e24146468348f082a0
+size 4998852296

checkpoint-30517/model-00006-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:baea304248903e05cfc92b8e4b43bbac43ac06e958bed3b0771381caefaa6e69
+size 3662865184

checkpoint-30517/model-00007-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7d030b42cb4575e2e8e809b40687fb605439d4980e0ec3cdbb593e4981fddc95
+size 2317729920

checkpoint-30517/model.safetensors.index.json

@@ -0,0 +1,346 @@
+{
+  "metadata": {
+    "total_size": 30737946624
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00007-of-00007.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.self_attn.k_proj.bias": "model-00001-of-00007.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.self_attn.q_proj.bias": "model-00001-of-00007.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.self_attn.v_proj.bias": "model-00001-of-00007.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.self_attn.k_proj.bias": "model-00001-of-00007.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.self_attn.q_proj.bias": "model-00001-of-00007.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.self_attn.v_proj.bias": "model-00001-of-00007.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.self_attn.k_proj.bias": "model-00003-of-00007.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.self_attn.q_proj.bias": "model-00003-of-00007.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.self_attn.v_proj.bias": "model-00003-of-00007.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.self_attn.k_proj.bias": "model-00003-of-00007.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.self_attn.q_proj.bias": "model-00003-of-00007.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.self_attn.v_proj.bias": "model-00003-of-00007.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.self_attn.k_proj.bias": "model-00003-of-00007.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.self_attn.q_proj.bias": "model-00003-of-00007.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.self_attn.v_proj.bias": "model-00003-of-00007.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.13.self_attn.k_proj.bias": "model-00003-of-00007.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.self_attn.q_proj.bias": "model-00003-of-00007.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.self_attn.v_proj.bias": "model-00003-of-00007.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.self_attn.k_proj.bias": "model-00004-of-00007.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.self_attn.q_proj.bias": "model-00004-of-00007.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.self_attn.v_proj.bias": "model-00004-of-00007.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.self_attn.k_proj.bias": "model-00004-of-00007.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.self_attn.q_proj.bias": "model-00004-of-00007.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.self_attn.v_proj.bias": "model-00004-of-00007.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.self_attn.k_proj.bias": "model-00004-of-00007.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.self_attn.q_proj.bias": "model-00004-of-00007.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.self_attn.v_proj.bias": "model-00004-of-00007.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.self_attn.k_proj.bias": "model-00004-of-00007.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.self_attn.q_proj.bias": "model-00004-of-00007.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.self_attn.v_proj.bias": "model-00004-of-00007.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.18.self_attn.k_proj.bias": "model-00004-of-00007.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.self_attn.q_proj.bias": "model-00004-of-00007.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.self_attn.v_proj.bias": "model-00004-of-00007.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.19.self_attn.k_proj.bias": "model-00005-of-00007.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.19.self_attn.q_proj.bias": "model-00005-of-00007.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.19.self_attn.v_proj.bias": "model-00005-of-00007.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.2.self_attn.k_proj.bias": "model-00001-of-00007.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.self_attn.q_proj.bias": "model-00001-of-00007.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.self_attn.v_proj.bias": "model-00001-of-00007.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.20.self_attn.k_proj.bias": "model-00005-of-00007.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.20.self_attn.q_proj.bias": "model-00005-of-00007.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.20.self_attn.v_proj.bias": "model-00005-of-00007.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.self_attn.k_proj.bias": "model-00005-of-00007.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.self_attn.q_proj.bias": "model-00005-of-00007.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.self_attn.v_proj.bias": "model-00005-of-00007.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.self_attn.k_proj.bias": "model-00005-of-00007.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.self_attn.q_proj.bias": "model-00005-of-00007.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.self_attn.v_proj.bias": "model-00005-of-00007.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.self_attn.k_proj.bias": "model-00005-of-00007.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.self_attn.q_proj.bias": "model-00005-of-00007.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.self_attn.v_proj.bias": "model-00005-of-00007.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.24.self_attn.k_proj.bias": "model-00005-of-00007.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.24.self_attn.q_proj.bias": "model-00005-of-00007.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.self_attn.v_proj.bias": "model-00005-of-00007.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.25.self_attn.k_proj.bias": "model-00006-of-00007.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.25.self_attn.q_proj.bias": "model-00006-of-00007.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.25.self_attn.v_proj.bias": "model-00006-of-00007.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.self_attn.k_proj.bias": "model-00006-of-00007.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.self_attn.q_proj.bias": "model-00006-of-00007.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.self_attn.v_proj.bias": "model-00006-of-00007.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.self_attn.k_proj.bias": "model-00006-of-00007.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.self_attn.q_proj.bias": "model-00006-of-00007.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.self_attn.v_proj.bias": "model-00006-of-00007.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.self_attn.k_proj.bias": "model-00002-of-00007.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.self_attn.q_proj.bias": "model-00002-of-00007.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.self_attn.v_proj.bias": "model-00002-of-00007.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.self_attn.k_proj.bias": "model-00002-of-00007.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.self_attn.q_proj.bias": "model-00002-of-00007.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.self_attn.v_proj.bias": "model-00002-of-00007.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.self_attn.k_proj.bias": "model-00002-of-00007.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.self_attn.q_proj.bias": "model-00002-of-00007.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.self_attn.v_proj.bias": "model-00002-of-00007.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.self_attn.k_proj.bias": "model-00002-of-00007.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.self_attn.q_proj.bias": "model-00002-of-00007.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.self_attn.v_proj.bias": "model-00002-of-00007.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.self_attn.k_proj.bias": "model-00002-of-00007.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.self_attn.q_proj.bias": "model-00002-of-00007.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.self_attn.v_proj.bias": "model-00002-of-00007.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.8.self_attn.k_proj.bias": "model-00002-of-00007.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.8.self_attn.q_proj.bias": "model-00002-of-00007.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.8.self_attn.v_proj.bias": "model-00002-of-00007.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.self_attn.k_proj.bias": "model-00003-of-00007.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.self_attn.q_proj.bias": "model-00003-of-00007.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.self_attn.v_proj.bias": "model-00003-of-00007.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "model.norm.weight": "model-00006-of-00007.safetensors"
+  }
+}

checkpoint-30517/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:80db2a120218c71d02d20b1f84b2d1576a6b08e3be7cf54ab8dcb9d6b2a46272
+size 16728842070

checkpoint-30517/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9d9cd6a0487226e5bd30d1846894c82af483733ab4381b75bae9c0745e05d405
+size 14244

checkpoint-30517/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4d91c597306f26392e8da0d33a09fdcca77cfdeb5ad42248045521772fa3d64e
+size 1064

checkpoint-30517/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:465f927e21695e17129aa36be8b4481e804b293136634c25aeb3429480a9a6e5
+size 5304

checkpoint-7630/config.json

@@ -0,0 +1,29 @@
+{
+  "_name_or_path": "/mnt/parscratch/users/acp23ay/private/models/Qwen2.5-7B-my-madlad-mean/",
+  "architectures": [
+    "Qwen2ForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 151643,
+  "eos_token_id": 151643,
+  "hidden_act": "silu",
+  "hidden_size": 3584,
+  "initializer_range": 0.02,
+  "intermediate_size": 18944,
+  "max_position_embeddings": 131072,
+  "max_window_layers": 28,
+  "model_type": "qwen2",
+  "num_attention_heads": 28,
+  "num_hidden_layers": 28,
+  "num_key_value_heads": 4,
+  "rms_norm_eps": 1e-06,
+  "rope_theta": 1000000.0,
+  "sliding_window": null,
+  "tie_word_embeddings": false,
+  "torch_dtype": "float32",
+  "transformers_version": "4.44.0.dev0",
+  "use_cache": true,
+  "use_mrope": false,
+  "use_sliding_window": false,
+  "vocab_size": 161672
+}

checkpoint-7630/generation_config.json

@@ -0,0 +1,6 @@
+{
+  "bos_token_id": 151643,
+  "eos_token_id": 151643,
+  "max_new_tokens": 2048,
+  "transformers_version": "4.44.0.dev0"
+}

checkpoint-7630/model-00001-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:90db5a32440c49c0f497058399f461e3639782ef98111836f911cc5ada4b628c
+size 4842817304

checkpoint-7630/model-00002-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:855851c22a0213c322c44b2d933eed67bc9ff8215091c3eb491ec94b662ffadd
+size 4998852224

checkpoint-7630/model-00003-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0c748c56f7d09ef7934bc2e6d2bbb3a6b4fec8beeb1759acb8db07b32764bd38
+size 4984124304