Upload folder using huggingface_hub

checkpoint-12208/config.json

@@ -0,0 +1,39 @@
+{
+  "_name_or_path": "/mnt/parscratch/users/acp23ay/private/models/Llama-3.1-8B-Instruct-ta-madlad-mean/",
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 128000,
+  "eos_token_id": [
+    128001,
+    128008,
+    128009
+  ],
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 131072,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 8,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": {
+    "factor": 8.0,
+    "high_freq_factor": 4.0,
+    "low_freq_factor": 1.0,
+    "original_max_position_embeddings": 8192,
+    "rope_type": "llama3"
+  },
+  "rope_theta": 500000.0,
+  "tie_word_embeddings": false,
+  "torch_dtype": "float32",
+  "transformers_version": "4.44.0.dev0",
+  "use_cache": true,
+  "vocab_size": 138256
+}

checkpoint-12208/generation_config.json

@@ -0,0 +1,12 @@
+{
+  "bos_token_id": 128000,
+  "do_sample": true,
+  "eos_token_id": [
+    128001,
+    128008,
+    128009
+  ],
+  "temperature": 0.6,
+  "top_p": 0.9,
+  "transformers_version": "4.44.0.dev0"
+}

checkpoint-12208/model-00001-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:42c35fcef8e1e04d32dea15a29c5e93a7f75e90fc909cfdd491e8842a0578dd4
+size 4983197184

checkpoint-12208/model-00002-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:168629c67732309d49f60a5ec48a6d160212bc987365e82e183dfbf74ba0c1f3
+size 4899116432

checkpoint-12208/model-00003-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1a223e87240e50dfb05634ea909cf1ec715d37a12b681b14a3a6e97f28688f2b
+size 4999813112

checkpoint-12208/model-00004-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3047d199c611c6d12b1cca780f033ad89df9c076f742674f7ac4a892ca9573b3
+size 4999813128

checkpoint-12208/model-00005-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0a2f91d749d664ecd22c71f3d54488966fc6899a03924b8fea3613c8f83398a8
+size 4832007496

checkpoint-12208/model-00006-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:db159783b9bec981ff0f9d655f42d8ecd197ca2a6cc0fb5224070bc07d606d4d
+size 4999813120

checkpoint-12208/model-00007-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b77c27004239a1bce35451437813db39c418bd8f08b406ad22af0b2cd9453374
+size 2734998184

checkpoint-12208/model.safetensors.index.json

@@ -0,0 +1,298 @@
+{
+  "metadata": {
+    "total_size": 32448724992
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00007-of-00007.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00007-of-00007.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00007-of-00007.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00007-of-00007.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00007-of-00007.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "model.norm.weight": "model-00007-of-00007.safetensors"
+  }
+}

checkpoint-12208/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5ba069faf91b22bc7301922d2796c42b030a97971b349e79197c9958e4d8d66e
+size 16040396334

checkpoint-12208/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9d9cd6a0487226e5bd30d1846894c82af483733ab4381b75bae9c0745e05d405
+size 14244

checkpoint-12208/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:abe644ed33a3c4139223f0857a985127f3e6fbaa8c89fa14b57671b49ca52c21
+size 1064

checkpoint-12208/trainer_state.json

@@ -0,0 +1,2784 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.4057431534166445,
+  "eval_steps": 500,
+  "global_step": 12208,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.001030311087476735,
+      "grad_norm": 60.25547409057617,
+      "learning_rate": 1.0157273918741808e-06,
+      "loss": 8.8455,
+      "step": 31
+    },
+    {
+      "epoch": 0.00206062217495347,
+      "grad_norm": 15.669363975524902,
+      "learning_rate": 2.0314547837483616e-06,
+      "loss": 7.1553,
+      "step": 62
+    },
+    {
+      "epoch": 0.003090933262430205,
+      "grad_norm": 15.366345405578613,
+      "learning_rate": 3.0471821756225426e-06,
+      "loss": 5.8784,
+      "step": 93
+    },
+    {
+      "epoch": 0.00412124434990694,
+      "grad_norm": 36.30561828613281,
+      "learning_rate": 4.062909567496723e-06,
+      "loss": 4.7708,
+      "step": 124
+    },
+    {
+      "epoch": 0.005151555437383675,
+      "grad_norm": 27.202678680419922,
+      "learning_rate": 5.078636959370905e-06,
+      "loss": 4.1629,
+      "step": 155
+    },
+    {
+      "epoch": 0.00618186652486041,
+      "grad_norm": 24.30484962463379,
+      "learning_rate": 6.094364351245085e-06,
+      "loss": 3.867,
+      "step": 186
+    },
+    {
+      "epoch": 0.007212177612337145,
+      "grad_norm": 19.916366577148438,
+      "learning_rate": 7.110091743119267e-06,
+      "loss": 3.6131,
+      "step": 217
+    },
+    {
+      "epoch": 0.00824248869981388,
+      "grad_norm": 17.577274322509766,
+      "learning_rate": 8.125819134993446e-06,
+      "loss": 3.4772,
+      "step": 248
+    },
+    {
+      "epoch": 0.009272799787290615,
+      "grad_norm": 12.133190155029297,
+      "learning_rate": 9.141546526867629e-06,
+      "loss": 3.3218,
+      "step": 279
+    },
+    {
+      "epoch": 0.01030311087476735,
+      "grad_norm": 19.79263687133789,
+      "learning_rate": 1.015727391874181e-05,
+      "loss": 3.2055,
+      "step": 310
+    },
+    {
+      "epoch": 0.011333421962244085,
+      "grad_norm": 16.38133430480957,
+      "learning_rate": 1.117300131061599e-05,
+      "loss": 3.1062,
+      "step": 341
+    },
+    {
+      "epoch": 0.01236373304972082,
+      "grad_norm": 12.638299942016602,
+      "learning_rate": 1.218872870249017e-05,
+      "loss": 3.0106,
+      "step": 372
+    },
+    {
+      "epoch": 0.013394044137197554,
+      "grad_norm": 9.46596908569336,
+      "learning_rate": 1.3204456094364351e-05,
+      "loss": 2.924,
+      "step": 403
+    },
+    {
+      "epoch": 0.01442435522467429,
+      "grad_norm": 10.945392608642578,
+      "learning_rate": 1.4220183486238533e-05,
+      "loss": 2.844,
+      "step": 434
+    },
+    {
+      "epoch": 0.015454666312151024,
+      "grad_norm": 8.474015235900879,
+      "learning_rate": 1.5235910878112714e-05,
+      "loss": 2.7892,
+      "step": 465
+    },
+    {
+      "epoch": 0.01648497739962776,
+      "grad_norm": 9.370804786682129,
+      "learning_rate": 1.6251638269986893e-05,
+      "loss": 2.7509,
+      "step": 496
+    },
+    {
+      "epoch": 0.017515288487104493,
+      "grad_norm": 11.63398551940918,
+      "learning_rate": 1.7267365661861077e-05,
+      "loss": 2.6999,
+      "step": 527
+    },
+    {
+      "epoch": 0.01854559957458123,
+      "grad_norm": 9.17713451385498,
+      "learning_rate": 1.8283093053735257e-05,
+      "loss": 2.6459,
+      "step": 558
+    },
+    {
+      "epoch": 0.019575910662057962,
+      "grad_norm": 7.119054794311523,
+      "learning_rate": 1.9298820445609438e-05,
+      "loss": 2.603,
+      "step": 589
+    },
+    {
+      "epoch": 0.0206062217495347,
+      "grad_norm": 6.653646945953369,
+      "learning_rate": 2.031454783748362e-05,
+      "loss": 2.5588,
+      "step": 620
+    },
+    {
+      "epoch": 0.021636532837011432,
+      "grad_norm": 8.332653045654297,
+      "learning_rate": 2.13302752293578e-05,
+      "loss": 2.5357,
+      "step": 651
+    },
+    {
+      "epoch": 0.02266684392448817,
+      "grad_norm": 6.4949116706848145,
+      "learning_rate": 2.234600262123198e-05,
+      "loss": 2.4967,
+      "step": 682
+    },
+    {
+      "epoch": 0.023697155011964902,
+      "grad_norm": 9.41009521484375,
+      "learning_rate": 2.336173001310616e-05,
+      "loss": 2.4563,
+      "step": 713
+    },
+    {
+      "epoch": 0.02472746609944164,
+      "grad_norm": 7.840345859527588,
+      "learning_rate": 2.437745740498034e-05,
+      "loss": 2.4383,
+      "step": 744
+    },
+    {
+      "epoch": 0.025757777186918372,
+      "grad_norm": 6.116458415985107,
+      "learning_rate": 2.5393184796854525e-05,
+      "loss": 2.3817,
+      "step": 775
+    },
+    {
+      "epoch": 0.02678808827439511,
+      "grad_norm": 5.938300609588623,
+      "learning_rate": 2.6408912188728702e-05,
+      "loss": 2.3508,
+      "step": 806
+    },
+    {
+      "epoch": 0.027818399361871842,
+      "grad_norm": 5.4408345222473145,
+      "learning_rate": 2.7424639580602886e-05,
+      "loss": 2.3325,
+      "step": 837
+    },
+    {
+      "epoch": 0.02884871044934858,
+      "grad_norm": 5.375136375427246,
+      "learning_rate": 2.8440366972477066e-05,
+      "loss": 2.3101,
+      "step": 868
+    },
+    {
+      "epoch": 0.029879021536825312,
+      "grad_norm": 5.149726867675781,
+      "learning_rate": 2.9456094364351244e-05,
+      "loss": 2.282,
+      "step": 899
+    },
+    {
+      "epoch": 0.03090933262430205,
+      "grad_norm": 4.591221332550049,
+      "learning_rate": 3.0471821756225428e-05,
+      "loss": 2.2427,
+      "step": 930
+    },
+    {
+      "epoch": 0.031939643711778785,
+      "grad_norm": 4.977034091949463,
+      "learning_rate": 3.148754914809961e-05,
+      "loss": 2.2218,
+      "step": 961
+    },
+    {
+      "epoch": 0.03296995479925552,
+      "grad_norm": 5.038781642913818,
+      "learning_rate": 3.2503276539973785e-05,
+      "loss": 2.2044,
+      "step": 992
+    },
+    {
+      "epoch": 0.03400026588673225,
+      "grad_norm": 4.872281551361084,
+      "learning_rate": 3.351900393184797e-05,
+      "loss": 2.1657,
+      "step": 1023
+    },
+    {
+      "epoch": 0.035030576974208985,
+      "grad_norm": 4.370841979980469,
+      "learning_rate": 3.453473132372215e-05,
+      "loss": 2.1365,
+      "step": 1054
+    },
+    {
+      "epoch": 0.036060888061685725,
+      "grad_norm": 4.087072849273682,
+      "learning_rate": 3.555045871559633e-05,
+      "loss": 2.1253,
+      "step": 1085
+    },
+    {
+      "epoch": 0.03709119914916246,
+      "grad_norm": 4.113957405090332,
+      "learning_rate": 3.6566186107470514e-05,
+      "loss": 2.0973,
+      "step": 1116
+    },
+    {
+      "epoch": 0.03812151023663919,
+      "grad_norm": 4.0119733810424805,
+      "learning_rate": 3.7581913499344695e-05,
+      "loss": 2.1024,
+      "step": 1147
+    },
+    {
+      "epoch": 0.039151821324115925,
+      "grad_norm": 4.247573375701904,
+      "learning_rate": 3.8597640891218876e-05,
+      "loss": 2.0722,
+      "step": 1178
+    },
+    {
+      "epoch": 0.04018213241159266,
+      "grad_norm": 3.5575129985809326,
+      "learning_rate": 3.9613368283093056e-05,
+      "loss": 2.056,
+      "step": 1209
+    },
+    {
+      "epoch": 0.0412124434990694,
+      "grad_norm": 3.8885862827301025,
+      "learning_rate": 4.062909567496724e-05,
+      "loss": 2.0389,
+      "step": 1240
+    },
+    {
+      "epoch": 0.04224275458654613,
+      "grad_norm": 3.680628538131714,
+      "learning_rate": 4.164482306684142e-05,
+      "loss": 2.0385,
+      "step": 1271
+    },
+    {
+      "epoch": 0.043273065674022865,
+      "grad_norm": 3.780876874923706,
+      "learning_rate": 4.26605504587156e-05,
+      "loss": 2.0097,
+      "step": 1302
+    },
+    {
+      "epoch": 0.0443033767614996,
+      "grad_norm": 4.235328674316406,
+      "learning_rate": 4.367627785058978e-05,
+      "loss": 2.0024,
+      "step": 1333
+    },
+    {
+      "epoch": 0.04533368784897634,
+      "grad_norm": 3.326941967010498,
+      "learning_rate": 4.469200524246396e-05,
+      "loss": 1.9953,
+      "step": 1364
+    },
+    {
+      "epoch": 0.04636399893645307,
+      "grad_norm": 3.28456449508667,
+      "learning_rate": 4.570773263433814e-05,
+      "loss": 1.9579,
+      "step": 1395
+    },
+    {
+      "epoch": 0.047394310023929805,
+      "grad_norm": 16.107433319091797,
+      "learning_rate": 4.672346002621232e-05,
+      "loss": 1.9701,
+      "step": 1426
+    },
+    {
+      "epoch": 0.04842462111140654,
+      "grad_norm": 3.5708224773406982,
+      "learning_rate": 4.77391874180865e-05,
+      "loss": 1.9621,
+      "step": 1457
+    },
+    {
+      "epoch": 0.04945493219888328,
+      "grad_norm": 2.9053499698638916,
+      "learning_rate": 4.875491480996068e-05,
+      "loss": 1.9458,
+      "step": 1488
+    },
+    {
+      "epoch": 0.05048524328636001,
+      "grad_norm": 3.0863258838653564,
+      "learning_rate": 4.977064220183487e-05,
+      "loss": 1.9483,
+      "step": 1519
+    },
+    {
+      "epoch": 0.051515554373836744,
+      "grad_norm": 2.9012269973754883,
+      "learning_rate": 4.9999915451558777e-05,
+      "loss": 1.928,
+      "step": 1550
+    },
+    {
+      "epoch": 0.05254586546131348,
+      "grad_norm": 3.0949041843414307,
+      "learning_rate": 4.999955597496219e-05,
+      "loss": 1.9229,
+      "step": 1581
+    },
+    {
+      "epoch": 0.05357617654879022,
+      "grad_norm": 2.8687901496887207,
+      "learning_rate": 4.9998914381774255e-05,
+      "loss": 1.915,
+      "step": 1612
+    },
+    {
+      "epoch": 0.05460648763626695,
+      "grad_norm": 3.2136878967285156,
+      "learning_rate": 4.999799067923527e-05,
+      "loss": 1.9197,
+      "step": 1643
+    },
+    {
+      "epoch": 0.055636798723743684,
+      "grad_norm": 2.590843677520752,
+      "learning_rate": 4.999678487776908e-05,
+      "loss": 1.8756,
+      "step": 1674
+    },
+    {
+      "epoch": 0.05666710981122042,
+      "grad_norm": 2.64634108543396,
+      "learning_rate": 4.9995296990983006e-05,
+      "loss": 1.9033,
+      "step": 1705
+    },
+    {
+      "epoch": 0.05769742089869716,
+      "grad_norm": 3.0151331424713135,
+      "learning_rate": 4.999352703566763e-05,
+      "loss": 1.8883,
+      "step": 1736
+    },
+    {
+      "epoch": 0.05872773198617389,
+      "grad_norm": 2.526806354522705,
+      "learning_rate": 4.999147503179668e-05,
+      "loss": 1.8666,
+      "step": 1767
+    },
+    {
+      "epoch": 0.059758043073650624,
+      "grad_norm": 2.510300397872925,
+      "learning_rate": 4.998914100252672e-05,
+      "loss": 1.854,
+      "step": 1798
+    },
+    {
+      "epoch": 0.06078835416112736,
+      "grad_norm": 2.4867682456970215,
+      "learning_rate": 4.998652497419696e-05,
+      "loss": 1.8548,
+      "step": 1829
+    },
+    {
+      "epoch": 0.0618186652486041,
+      "grad_norm": 2.3920586109161377,
+      "learning_rate": 4.9983626976328927e-05,
+      "loss": 1.8495,
+      "step": 1860
+    },
+    {
+      "epoch": 0.06284897633608083,
+      "grad_norm": 2.714177370071411,
+      "learning_rate": 4.998044704162613e-05,
+      "loss": 1.8433,
+      "step": 1891
+    },
+    {
+      "epoch": 0.06387928742355757,
+      "grad_norm": 2.3094465732574463,
+      "learning_rate": 4.9976985205973705e-05,
+      "loss": 1.8382,
+      "step": 1922
+    },
+    {
+      "epoch": 0.0649095985110343,
+      "grad_norm": 2.47184419631958,
+      "learning_rate": 4.997324150843799e-05,
+      "loss": 1.8464,
+      "step": 1953
+    },
+    {
+      "epoch": 0.06593990959851104,
+      "grad_norm": 2.391841411590576,
+      "learning_rate": 4.99692159912661e-05,
+      "loss": 1.8179,
+      "step": 1984
+    },
+    {
+      "epoch": 0.06697022068598776,
+      "grad_norm": 2.2471864223480225,
+      "learning_rate": 4.996490869988546e-05,
+      "loss": 1.8149,
+      "step": 2015
+    },
+    {
+      "epoch": 0.0680005317734645,
+      "grad_norm": 2.5497376918792725,
+      "learning_rate": 4.996031968290326e-05,
+      "loss": 1.8099,
+      "step": 2046
+    },
+    {
+      "epoch": 0.06903084286094124,
+      "grad_norm": 2.330463409423828,
+      "learning_rate": 4.995544899210594e-05,
+      "loss": 1.8267,
+      "step": 2077
+    },
+    {
+      "epoch": 0.07006115394841797,
+      "grad_norm": 2.3259341716766357,
+      "learning_rate": 4.9950296682458583e-05,
+      "loss": 1.7801,
+      "step": 2108
+    },
+    {
+      "epoch": 0.07109146503589471,
+      "grad_norm": 2.1711952686309814,
+      "learning_rate": 4.994486281210429e-05,
+      "loss": 1.7961,
+      "step": 2139
+    },
+    {
+      "epoch": 0.07212177612337145,
+      "grad_norm": 2.1808884143829346,
+      "learning_rate": 4.9939147442363566e-05,
+      "loss": 1.8109,
+      "step": 2170
+    },
+    {
+      "epoch": 0.07315208721084818,
+      "grad_norm": 2.089256525039673,
+      "learning_rate": 4.9933150637733574e-05,
+      "loss": 1.8026,
+      "step": 2201
+    },
+    {
+      "epoch": 0.07418239829832492,
+      "grad_norm": 2.0864951610565186,
+      "learning_rate": 4.992687246588743e-05,
+      "loss": 1.7753,
+      "step": 2232
+    },
+    {
+      "epoch": 0.07521270938580164,
+      "grad_norm": 2.36157488822937,
+      "learning_rate": 4.992031299767347e-05,
+      "loss": 1.7746,
+      "step": 2263
+    },
+    {
+      "epoch": 0.07624302047327838,
+      "grad_norm": 2.5334439277648926,
+      "learning_rate": 4.9913472307114386e-05,
+      "loss": 1.7927,
+      "step": 2294
+    },
+    {
+      "epoch": 0.07727333156075512,
+      "grad_norm": 2.2565715312957764,
+      "learning_rate": 4.9906350471406446e-05,
+      "loss": 1.7668,
+      "step": 2325
+    },
+    {
+      "epoch": 0.07830364264823185,
+      "grad_norm": 2.1043128967285156,
+      "learning_rate": 4.989894757091861e-05,
+      "loss": 1.7771,
+      "step": 2356
+    },
+    {
+      "epoch": 0.07933395373570859,
+      "grad_norm": 1.9659819602966309,
+      "learning_rate": 4.989126368919158e-05,
+      "loss": 1.7666,
+      "step": 2387
+    },
+    {
+      "epoch": 0.08036426482318532,
+      "grad_norm": 2.0778403282165527,
+      "learning_rate": 4.988329891293693e-05,
+      "loss": 1.7405,
+      "step": 2418
+    },
+    {
+      "epoch": 0.08139457591066206,
+      "grad_norm": 2.1767923831939697,
+      "learning_rate": 4.987505333203608e-05,
+      "loss": 1.7495,
+      "step": 2449
+    },
+    {
+      "epoch": 0.0824248869981388,
+      "grad_norm": 2.260143280029297,
+      "learning_rate": 4.9866527039539276e-05,
+      "loss": 1.7504,
+      "step": 2480
+    },
+    {
+      "epoch": 0.08345519808561552,
+      "grad_norm": 2.18271803855896,
+      "learning_rate": 4.9857720131664594e-05,
+      "loss": 1.7456,
+      "step": 2511
+    },
+    {
+      "epoch": 0.08448550917309226,
+      "grad_norm": 2.209594964981079,
+      "learning_rate": 4.9848632707796773e-05,
+      "loss": 1.7528,
+      "step": 2542
+    },
+    {
+      "epoch": 0.085515820260569,
+      "grad_norm": 2.0666229724884033,
+      "learning_rate": 4.9839264870486155e-05,
+      "loss": 1.7517,
+      "step": 2573
+    },
+    {
+      "epoch": 0.08654613134804573,
+      "grad_norm": 2.1070454120635986,
+      "learning_rate": 4.9829616725447526e-05,
+      "loss": 1.7474,
+      "step": 2604
+    },
+    {
+      "epoch": 0.08757644243552247,
+      "grad_norm": 1.9430303573608398,
+      "learning_rate": 4.981968838155888e-05,
+      "loss": 1.7348,
+      "step": 2635
+    },
+    {
+      "epoch": 0.0886067535229992,
+      "grad_norm": 1.9638925790786743,
+      "learning_rate": 4.980947995086024e-05,
+      "loss": 1.7202,
+      "step": 2666
+    },
+    {
+      "epoch": 0.08963706461047594,
+      "grad_norm": 1.8845652341842651,
+      "learning_rate": 4.979899154855234e-05,
+      "loss": 1.7375,
+      "step": 2697
+    },
+    {
+      "epoch": 0.09066737569795268,
+      "grad_norm": 5.712058067321777,
+      "learning_rate": 4.9788223292995386e-05,
+      "loss": 1.7379,
+      "step": 2728
+    },
+    {
+      "epoch": 0.0916976867854294,
+      "grad_norm": 1.9520670175552368,
+      "learning_rate": 4.977717530570768e-05,
+      "loss": 1.7302,
+      "step": 2759
+    },
+    {
+      "epoch": 0.09272799787290614,
+      "grad_norm": 1.8802224397659302,
+      "learning_rate": 4.976584771136425e-05,
+      "loss": 1.74,
+      "step": 2790
+    },
+    {
+      "epoch": 0.09375830896038288,
+      "grad_norm": 2.1098153591156006,
+      "learning_rate": 4.975424063779547e-05,
+      "loss": 1.7024,
+      "step": 2821
+    },
+    {
+      "epoch": 0.09478862004785961,
+      "grad_norm": 2.1568291187286377,
+      "learning_rate": 4.974235421598557e-05,
+      "loss": 1.7131,
+      "step": 2852
+    },
+    {
+      "epoch": 0.09581893113533635,
+      "grad_norm": 1.8769980669021606,
+      "learning_rate": 4.973018858007122e-05,
+      "loss": 1.7008,
+      "step": 2883
+    },
+    {
+      "epoch": 0.09684924222281308,
+      "grad_norm": 1.8325533866882324,
+      "learning_rate": 4.9717743867339963e-05,
+      "loss": 1.7058,
+      "step": 2914
+    },
+    {
+      "epoch": 0.09787955331028982,
+      "grad_norm": 2.086416721343994,
+      "learning_rate": 4.9705020218228695e-05,
+      "loss": 1.711,
+      "step": 2945
+    },
+    {
+      "epoch": 0.09890986439776656,
+      "grad_norm": 1.8294793367385864,
+      "learning_rate": 4.969201777632205e-05,
+      "loss": 1.6998,
+      "step": 2976
+    },
+    {
+      "epoch": 0.09994017548524328,
+      "grad_norm": 2.0608153343200684,
+      "learning_rate": 4.9678736688350846e-05,
+      "loss": 1.6948,
+      "step": 3007
+    },
+    {
+      "epoch": 0.10097048657272002,
+      "grad_norm": 3.2166008949279785,
+      "learning_rate": 4.966517710419033e-05,
+      "loss": 1.6788,
+      "step": 3038
+    },
+    {
+      "epoch": 0.10200079766019676,
+      "grad_norm": 1.9431313276290894,
+      "learning_rate": 4.965133917685858e-05,
+      "loss": 1.7115,
+      "step": 3069
+    },
+    {
+      "epoch": 0.10303110874767349,
+      "grad_norm": 1.967512845993042,
+      "learning_rate": 4.9637223062514714e-05,
+      "loss": 1.7033,
+      "step": 3100
+    },
+    {
+      "epoch": 0.10406141983515023,
+      "grad_norm": 1.9253389835357666,
+      "learning_rate": 4.962282892045718e-05,
+      "loss": 1.6856,
+      "step": 3131
+    },
+    {
+      "epoch": 0.10509173092262696,
+      "grad_norm": 1.986840009689331,
+      "learning_rate": 4.9608156913121904e-05,
+      "loss": 1.723,
+      "step": 3162
+    },
+    {
+      "epoch": 0.1061220420101037,
+      "grad_norm": 1.83523690700531,
+      "learning_rate": 4.959320720608049e-05,
+      "loss": 1.6912,
+      "step": 3193
+    },
+    {
+      "epoch": 0.10715235309758044,
+      "grad_norm": 2.1271955966949463,
+      "learning_rate": 4.9577979968038354e-05,
+      "loss": 1.7032,
+      "step": 3224
+    },
+    {
+      "epoch": 0.10818266418505716,
+      "grad_norm": 1.8383768796920776,
+      "learning_rate": 4.956247537083282e-05,
+      "loss": 1.6726,
+      "step": 3255
+    },
+    {
+      "epoch": 0.1092129752725339,
+      "grad_norm": 1.8806651830673218,
+      "learning_rate": 4.9546693589431145e-05,
+      "loss": 1.6817,
+      "step": 3286
+    },
+    {
+      "epoch": 0.11024328636001064,
+      "grad_norm": 1.7535260915756226,
+      "learning_rate": 4.9530634801928595e-05,
+      "loss": 1.6875,
+      "step": 3317
+    },
+    {
+      "epoch": 0.11127359744748737,
+      "grad_norm": 1.765906810760498,
+      "learning_rate": 4.9514299189546395e-05,
+      "loss": 1.6859,
+      "step": 3348
+    },
+    {
+      "epoch": 0.11230390853496411,
+      "grad_norm": 1.869828462600708,
+      "learning_rate": 4.949768693662973e-05,
+      "loss": 1.6915,
+      "step": 3379
+    },
+    {
+      "epoch": 0.11333421962244083,
+      "grad_norm": 1.8347504138946533,
+      "learning_rate": 4.948079823064559e-05,
+      "loss": 1.6859,
+      "step": 3410
+    },
+    {
+      "epoch": 0.11436453070991758,
+      "grad_norm": 1.7692474126815796,
+      "learning_rate": 4.946363326218074e-05,
+      "loss": 1.6565,
+      "step": 3441
+    },
+    {
+      "epoch": 0.11539484179739432,
+      "grad_norm": 1.8231885433197021,
+      "learning_rate": 4.9446192224939525e-05,
+      "loss": 1.686,
+      "step": 3472
+    },
+    {
+      "epoch": 0.11642515288487104,
+      "grad_norm": 1.7155958414077759,
+      "learning_rate": 4.942847531574167e-05,
+      "loss": 1.6538,
+      "step": 3503
+    },
+    {
+      "epoch": 0.11745546397234778,
+      "grad_norm": 1.787183403968811,
+      "learning_rate": 4.941048273452008e-05,
+      "loss": 1.6776,
+      "step": 3534
+    },
+    {
+      "epoch": 0.11848577505982451,
+      "grad_norm": 1.741213083267212,
+      "learning_rate": 4.9392214684318605e-05,
+      "loss": 1.6784,
+      "step": 3565
+    },
+    {
+      "epoch": 0.11951608614730125,
+      "grad_norm": 1.7836824655532837,
+      "learning_rate": 4.93736713712897e-05,
+      "loss": 1.6557,
+      "step": 3596
+    },
+    {
+      "epoch": 0.12054639723477799,
+      "grad_norm": 1.7103859186172485,
+      "learning_rate": 4.9354853004692124e-05,
+      "loss": 1.6606,
+      "step": 3627
+    },
+    {
+      "epoch": 0.12157670832225471,
+      "grad_norm": 1.7865506410598755,
+      "learning_rate": 4.93357597968886e-05,
+      "loss": 1.6409,
+      "step": 3658
+    },
+    {
+      "epoch": 0.12260701940973145,
+      "grad_norm": 1.7770143747329712,
+      "learning_rate": 4.931639196334338e-05,
+      "loss": 1.6574,
+      "step": 3689
+    },
+    {
+      "epoch": 0.1236373304972082,
+      "grad_norm": 1.857575535774231,
+      "learning_rate": 4.9296749722619826e-05,
+      "loss": 1.6724,
+      "step": 3720
+    },
+    {
+      "epoch": 0.12466764158468492,
+      "grad_norm": 1.8742581605911255,
+      "learning_rate": 4.9276833296377966e-05,
+      "loss": 1.6506,
+      "step": 3751
+    },
+    {
+      "epoch": 0.12569795267216166,
+      "grad_norm": 1.827668309211731,
+      "learning_rate": 4.925664290937196e-05,
+      "loss": 1.6523,
+      "step": 3782
+    },
+    {
+      "epoch": 0.1267282637596384,
+      "grad_norm": 1.7517486810684204,
+      "learning_rate": 4.9236178789447576e-05,
+      "loss": 1.6459,
+      "step": 3813
+    },
+    {
+      "epoch": 0.12775857484711514,
+      "grad_norm": 1.8109570741653442,
+      "learning_rate": 4.921544116753962e-05,
+      "loss": 1.6614,
+      "step": 3844
+    },
+    {
+      "epoch": 0.12878888593459187,
+      "grad_norm": 1.692597508430481,
+      "learning_rate": 4.919443027766935e-05,
+      "loss": 1.6431,
+      "step": 3875
+    },
+    {
+      "epoch": 0.1298191970220686,
+      "grad_norm": 1.8650025129318237,
+      "learning_rate": 4.91731463569418e-05,
+      "loss": 1.6466,
+      "step": 3906
+    },
+    {
+      "epoch": 0.13084950810954532,
+      "grad_norm": 1.6794081926345825,
+      "learning_rate": 4.915158964554312e-05,
+      "loss": 1.6504,
+      "step": 3937
+    },
+    {
+      "epoch": 0.13187981919702207,
+      "grad_norm": 1.7685374021530151,
+      "learning_rate": 4.912976038673786e-05,
+      "loss": 1.6446,
+      "step": 3968
+    },
+    {
+      "epoch": 0.1329101302844988,
+      "grad_norm": 1.7601110935211182,
+      "learning_rate": 4.9107658826866254e-05,
+      "loss": 1.631,
+      "step": 3999
+    },
+    {
+      "epoch": 0.13394044137197553,
+      "grad_norm": 2.0616064071655273,
+      "learning_rate": 4.908528521534139e-05,
+      "loss": 1.6476,
+      "step": 4030
+    },
+    {
+      "epoch": 0.13497075245945228,
+      "grad_norm": 1.8973504304885864,
+      "learning_rate": 4.906263980464644e-05,
+      "loss": 1.6582,
+      "step": 4061
+    },
+    {
+      "epoch": 0.136001063546929,
+      "grad_norm": 1.7768895626068115,
+      "learning_rate": 4.903972285033178e-05,
+      "loss": 1.6159,
+      "step": 4092
+    },
+    {
+      "epoch": 0.13703137463440573,
+      "grad_norm": 1.8264424800872803,
+      "learning_rate": 4.901653461101213e-05,
+      "loss": 1.6289,
+      "step": 4123
+    },
+    {
+      "epoch": 0.1380616857218825,
+      "grad_norm": 1.7140119075775146,
+      "learning_rate": 4.8993075348363626e-05,
+      "loss": 1.6357,
+      "step": 4154
+    },
+    {
+      "epoch": 0.13909199680935921,
+      "grad_norm": 1.6964486837387085,
+      "learning_rate": 4.896934532712084e-05,
+      "loss": 1.6233,
+      "step": 4185
+    },
+    {
+      "epoch": 0.14012230789683594,
+      "grad_norm": 1.8008025884628296,
+      "learning_rate": 4.8945344815073846e-05,
+      "loss": 1.637,
+      "step": 4216
+    },
+    {
+      "epoch": 0.1411526189843127,
+      "grad_norm": 1.562730073928833,
+      "learning_rate": 4.892107408306516e-05,
+      "loss": 1.6379,
+      "step": 4247
+    },
+    {
+      "epoch": 0.14218293007178942,
+      "grad_norm": 1.8273371458053589,
+      "learning_rate": 4.889653340498669e-05,
+      "loss": 1.6246,
+      "step": 4278
+    },
+    {
+      "epoch": 0.14321324115926615,
+      "grad_norm": 56.33716583251953,
+      "learning_rate": 4.8871723057776664e-05,
+      "loss": 1.6457,
+      "step": 4309
+    },
+    {
+      "epoch": 0.1442435522467429,
+      "grad_norm": 1.746523380279541,
+      "learning_rate": 4.8846643321416476e-05,
+      "loss": 1.6343,
+      "step": 4340
+    },
+    {
+      "epoch": 0.14527386333421963,
+      "grad_norm": 1.7737531661987305,
+      "learning_rate": 4.882129447892753e-05,
+      "loss": 1.6447,
+      "step": 4371
+    },
+    {
+      "epoch": 0.14630417442169635,
+      "grad_norm": 1.660485863685608,
+      "learning_rate": 4.8795676816368076e-05,
+      "loss": 1.6192,
+      "step": 4402
+    },
+    {
+      "epoch": 0.14733448550917308,
+      "grad_norm": 1.6823406219482422,
+      "learning_rate": 4.876979062282995e-05,
+      "loss": 1.6253,
+      "step": 4433
+    },
+    {
+      "epoch": 0.14836479659664983,
+      "grad_norm": 7.78139066696167,
+      "learning_rate": 4.8743636190435325e-05,
+      "loss": 1.6234,
+      "step": 4464
+    },
+    {
+      "epoch": 0.14939510768412656,
+      "grad_norm": 1.7426058053970337,
+      "learning_rate": 4.871721381433344e-05,
+      "loss": 1.6337,
+      "step": 4495
+    },
+    {
+      "epoch": 0.1504254187716033,
+      "grad_norm": 1.6294783353805542,
+      "learning_rate": 4.869052379269719e-05,
+      "loss": 1.6217,
+      "step": 4526
+    },
+    {
+      "epoch": 0.15145572985908004,
+      "grad_norm": 1.6523306369781494,
+      "learning_rate": 4.866356642671985e-05,
+      "loss": 1.605,
+      "step": 4557
+    },
+    {
+      "epoch": 0.15248604094655677,
+      "grad_norm": 1.8571300506591797,
+      "learning_rate": 4.8636342020611634e-05,
+      "loss": 1.6218,
+      "step": 4588
+    },
+    {
+      "epoch": 0.1535163520340335,
+      "grad_norm": 1.7754936218261719,
+      "learning_rate": 4.860885088159626e-05,
+      "loss": 1.6171,
+      "step": 4619
+    },
+    {
+      "epoch": 0.15454666312151025,
+      "grad_norm": 1.91987943649292,
+      "learning_rate": 4.858109331990751e-05,
+      "loss": 1.6167,
+      "step": 4650
+    },
+    {
+      "epoch": 0.15557697420898697,
+      "grad_norm": 1.5994452238082886,
+      "learning_rate": 4.855306964878567e-05,
+      "loss": 1.5951,
+      "step": 4681
+    },
+    {
+      "epoch": 0.1566072852964637,
+      "grad_norm": 1.6490916013717651,
+      "learning_rate": 4.8524780184474084e-05,
+      "loss": 1.616,
+      "step": 4712
+    },
+    {
+      "epoch": 0.15763759638394045,
+      "grad_norm": 1.5921640396118164,
+      "learning_rate": 4.8496225246215496e-05,
+      "loss": 1.6346,
+      "step": 4743
+    },
+    {
+      "epoch": 0.15866790747141718,
+      "grad_norm": 1.6729261875152588,
+      "learning_rate": 4.8467405156248505e-05,
+      "loss": 1.6165,
+      "step": 4774
+    },
+    {
+      "epoch": 0.1596982185588939,
+      "grad_norm": 1.628113031387329,
+      "learning_rate": 4.843832023980392e-05,
+      "loss": 1.6119,
+      "step": 4805
+    },
+    {
+      "epoch": 0.16072852964637063,
+      "grad_norm": 1.651647925376892,
+      "learning_rate": 4.840897082510106e-05,
+      "loss": 1.5997,
+      "step": 4836
+    },
+    {
+      "epoch": 0.1617588407338474,
+      "grad_norm": 1.5297720432281494,
+      "learning_rate": 4.8379357243344084e-05,
+      "loss": 1.6242,
+      "step": 4867
+    },
+    {
+      "epoch": 0.1627891518213241,
+      "grad_norm": 1.5779869556427002,
+      "learning_rate": 4.8349479828718236e-05,
+      "loss": 1.6149,
+      "step": 4898
+    },
+    {
+      "epoch": 0.16381946290880084,
+      "grad_norm": 1.5843939781188965,
+      "learning_rate": 4.8319338918386075e-05,
+      "loss": 1.5926,
+      "step": 4929
+    },
+    {
+      "epoch": 0.1648497739962776,
+      "grad_norm": 2.3762106895446777,
+      "learning_rate": 4.828893485248369e-05,
+      "loss": 1.6108,
+      "step": 4960
+    },
+    {
+      "epoch": 0.16588008508375432,
+      "grad_norm": 1.5871953964233398,
+      "learning_rate": 4.825826797411682e-05,
+      "loss": 1.6103,
+      "step": 4991
+    },
+    {
+      "epoch": 0.16691039617123105,
+      "grad_norm": 1.5934125185012817,
+      "learning_rate": 4.822733862935702e-05,
+      "loss": 1.6091,
+      "step": 5022
+    },
+    {
+      "epoch": 0.1679407072587078,
+      "grad_norm": 1.6997628211975098,
+      "learning_rate": 4.819614716723775e-05,
+      "loss": 1.6098,
+      "step": 5053
+    },
+    {
+      "epoch": 0.16897101834618453,
+      "grad_norm": 1.682849645614624,
+      "learning_rate": 4.8164693939750425e-05,
+      "loss": 1.599,
+      "step": 5084
+    },
+    {
+      "epoch": 0.17000132943366125,
+      "grad_norm": 1.709743857383728,
+      "learning_rate": 4.813297930184042e-05,
+      "loss": 1.6194,
+      "step": 5115
+    },
+    {
+      "epoch": 0.171031640521138,
+      "grad_norm": 1.725879430770874,
+      "learning_rate": 4.810100361140314e-05,
+      "loss": 1.6115,
+      "step": 5146
+    },
+    {
+      "epoch": 0.17206195160861473,
+      "grad_norm": 1.6710290908813477,
+      "learning_rate": 4.8068767229279885e-05,
+      "loss": 1.6032,
+      "step": 5177
+    },
+    {
+      "epoch": 0.17309226269609146,
+      "grad_norm": 1.6156634092330933,
+      "learning_rate": 4.8036270519253854e-05,
+      "loss": 1.5973,
+      "step": 5208
+    },
+    {
+      "epoch": 0.1741225737835682,
+      "grad_norm": 1.5654059648513794,
+      "learning_rate": 4.8003513848046e-05,
+      "loss": 1.5817,
+      "step": 5239
+    },
+    {
+      "epoch": 0.17515288487104494,
+      "grad_norm": 1.5789822340011597,
+      "learning_rate": 4.79704975853109e-05,
+      "loss": 1.6138,
+      "step": 5270
+    },
+    {
+      "epoch": 0.17618319595852167,
+      "grad_norm": 1.6022037267684937,
+      "learning_rate": 4.793722210363262e-05,
+      "loss": 1.5998,
+      "step": 5301
+    },
+    {
+      "epoch": 0.1772135070459984,
+      "grad_norm": 1.5142741203308105,
+      "learning_rate": 4.7903687778520414e-05,
+      "loss": 1.6061,
+      "step": 5332
+    },
+    {
+      "epoch": 0.17824381813347515,
+      "grad_norm": 1.6454212665557861,
+      "learning_rate": 4.7869894988404593e-05,
+      "loss": 1.6063,
+      "step": 5363
+    },
+    {
+      "epoch": 0.17927412922095187,
+      "grad_norm": 1.5250823497772217,
+      "learning_rate": 4.783584411463221e-05,
+      "loss": 1.6038,
+      "step": 5394
+    },
+    {
+      "epoch": 0.1803044403084286,
+      "grad_norm": 1.5829335451126099,
+      "learning_rate": 4.780153554146274e-05,
+      "loss": 1.5949,
+      "step": 5425
+    },
+    {
+      "epoch": 0.18133475139590535,
+      "grad_norm": 1.5342432260513306,
+      "learning_rate": 4.7766969656063766e-05,
+      "loss": 1.5913,
+      "step": 5456
+    },
+    {
+      "epoch": 0.18236506248338208,
+      "grad_norm": 1.6397250890731812,
+      "learning_rate": 4.773214684850662e-05,
+      "loss": 1.6102,
+      "step": 5487
+    },
+    {
+      "epoch": 0.1833953735708588,
+      "grad_norm": 1.5228471755981445,
+      "learning_rate": 4.769706751176193e-05,
+      "loss": 1.5885,
+      "step": 5518
+    },
+    {
+      "epoch": 0.18442568465833556,
+      "grad_norm": 1.6186103820800781,
+      "learning_rate": 4.7661732041695264e-05,
+      "loss": 1.6086,
+      "step": 5549
+    },
+    {
+      "epoch": 0.18545599574581229,
+      "grad_norm": 1.6024582386016846,
+      "learning_rate": 4.762614083706258e-05,
+      "loss": 1.6004,
+      "step": 5580
+    },
+    {
+      "epoch": 0.186486306833289,
+      "grad_norm": 1.5443711280822754,
+      "learning_rate": 4.759029429950581e-05,
+      "loss": 1.6048,
+      "step": 5611
+    },
+    {
+      "epoch": 0.18751661792076577,
+      "grad_norm": 1.4831629991531372,
+      "learning_rate": 4.7554192833548235e-05,
+      "loss": 1.5841,
+      "step": 5642
+    },
+    {
+      "epoch": 0.1885469290082425,
+      "grad_norm": 1.6426068544387817,
+      "learning_rate": 4.751783684659e-05,
+      "loss": 1.587,
+      "step": 5673
+    },
+    {
+      "epoch": 0.18957724009571922,
+      "grad_norm": 1.4609078168869019,
+      "learning_rate": 4.748122674890348e-05,
+      "loss": 1.5945,
+      "step": 5704
+    },
+    {
+      "epoch": 0.19060755118319597,
+      "grad_norm": 1.5365614891052246,
+      "learning_rate": 4.7444362953628654e-05,
+      "loss": 1.5737,
+      "step": 5735
+    },
+    {
+      "epoch": 0.1916378622706727,
+      "grad_norm": 1.5755670070648193,
+      "learning_rate": 4.7407245876768424e-05,
+      "loss": 1.5862,
+      "step": 5766
+    },
+    {
+      "epoch": 0.19266817335814942,
+      "grad_norm": 1.6469846963882446,
+      "learning_rate": 4.736987593718397e-05,
+      "loss": 1.5663,
+      "step": 5797
+    },
+    {
+      "epoch": 0.19369848444562615,
+      "grad_norm": 1.5927278995513916,
+      "learning_rate": 4.733225355658999e-05,
+      "loss": 1.5776,
+      "step": 5828
+    },
+    {
+      "epoch": 0.1947287955331029,
+      "grad_norm": 1.5593287944793701,
+      "learning_rate": 4.7294379159549926e-05,
+      "loss": 1.579,
+      "step": 5859
+    },
+    {
+      "epoch": 0.19575910662057963,
+      "grad_norm": 1.534055233001709,
+      "learning_rate": 4.725625317347119e-05,
+      "loss": 1.6017,
+      "step": 5890
+    },
+    {
+      "epoch": 0.19678941770805636,
+      "grad_norm": 1.5846387147903442,
+      "learning_rate": 4.7217876028600374e-05,
+      "loss": 1.5739,
+      "step": 5921
+    },
+    {
+      "epoch": 0.1978197287955331,
+      "grad_norm": 1.5377682447433472,
+      "learning_rate": 4.717924815801832e-05,
+      "loss": 1.57,
+      "step": 5952
+    },
+    {
+      "epoch": 0.19885003988300984,
+      "grad_norm": 1.467956781387329,
+      "learning_rate": 4.714036999763532e-05,
+      "loss": 1.5736,
+      "step": 5983
+    },
+    {
+      "epoch": 0.19988035097048656,
+      "grad_norm": 1.601070523262024,
+      "learning_rate": 4.7101241986186116e-05,
+      "loss": 1.5861,
+      "step": 6014
+    },
+    {
+      "epoch": 0.20091066205796332,
+      "grad_norm": 1.5051921606063843,
+      "learning_rate": 4.7061864565225e-05,
+      "loss": 1.5735,
+      "step": 6045
+    },
+    {
+      "epoch": 0.20194097314544004,
+      "grad_norm": 1.462843418121338,
+      "learning_rate": 4.702223817912081e-05,
+      "loss": 1.582,
+      "step": 6076
+    },
+    {
+      "epoch": 0.20297128423291677,
+      "grad_norm": 1.5698682069778442,
+      "learning_rate": 4.698236327505195e-05,
+      "loss": 1.5647,
+      "step": 6107
+    },
+    {
+      "epoch": 0.20400159532039353,
+      "grad_norm": 1.5633916854858398,
+      "learning_rate": 4.694224030300127e-05,
+      "loss": 1.5741,
+      "step": 6138
+    },
+    {
+      "epoch": 0.20503190640787025,
+      "grad_norm": 1.6174733638763428,
+      "learning_rate": 4.690186971575107e-05,
+      "loss": 1.5634,
+      "step": 6169
+    },
+    {
+      "epoch": 0.20606221749534698,
+      "grad_norm": 1.4957518577575684,
+      "learning_rate": 4.6861251968877916e-05,
+      "loss": 1.575,
+      "step": 6200
+    },
+    {
+      "epoch": 0.2070925285828237,
+      "grad_norm": 1.670933485031128,
+      "learning_rate": 4.68203875207476e-05,
+      "loss": 1.5792,
+      "step": 6231
+    },
+    {
+      "epoch": 0.20812283967030046,
+      "grad_norm": 1.5676430463790894,
+      "learning_rate": 4.677927683250983e-05,
+      "loss": 1.5689,
+      "step": 6262
+    },
+    {
+      "epoch": 0.20915315075777718,
+      "grad_norm": 1.5753976106643677,
+      "learning_rate": 4.6737920368093156e-05,
+      "loss": 1.5594,
+      "step": 6293
+    },
+    {
+      "epoch": 0.2101834618452539,
+      "grad_norm": 1.4973617792129517,
+      "learning_rate": 4.669631859419965e-05,
+      "loss": 1.5593,
+      "step": 6324
+    },
+    {
+      "epoch": 0.21121377293273066,
+      "grad_norm": 1.4691433906555176,
+      "learning_rate": 4.6654471980299676e-05,
+      "loss": 1.5711,
+      "step": 6355
+    },
+    {
+      "epoch": 0.2122440840202074,
+      "grad_norm": 1.407630443572998,
+      "learning_rate": 4.661238099862658e-05,
+      "loss": 1.5787,
+      "step": 6386
+    },
+    {
+      "epoch": 0.21327439510768412,
+      "grad_norm": 1.5011677742004395,
+      "learning_rate": 4.657004612417138e-05,
+      "loss": 1.5751,
+      "step": 6417
+    },
+    {
+      "epoch": 0.21430470619516087,
+      "grad_norm": 1.509750485420227,
+      "learning_rate": 4.6527467834677374e-05,
+      "loss": 1.5583,
+      "step": 6448
+    },
+    {
+      "epoch": 0.2153350172826376,
+      "grad_norm": 1.3919882774353027,
+      "learning_rate": 4.648464661063478e-05,
+      "loss": 1.5712,
+      "step": 6479
+    },
+    {
+      "epoch": 0.21636532837011432,
+      "grad_norm": 1.4854936599731445,
+      "learning_rate": 4.6441582935275264e-05,
+      "loss": 1.5637,
+      "step": 6510
+    },
+    {
+      "epoch": 0.21739563945759108,
+      "grad_norm": 1.4413583278656006,
+      "learning_rate": 4.6398277294566586e-05,
+      "loss": 1.56,
+      "step": 6541
+    },
+    {
+      "epoch": 0.2184259505450678,
+      "grad_norm": 1.5063883066177368,
+      "learning_rate": 4.6354730177207e-05,
+      "loss": 1.5525,
+      "step": 6572
+    },
+    {
+      "epoch": 0.21945626163254453,
+      "grad_norm": 1.4899688959121704,
+      "learning_rate": 4.6310942074619787e-05,
+      "loss": 1.5817,
+      "step": 6603
+    },
+    {
+      "epoch": 0.22048657272002128,
+      "grad_norm": 1.3927967548370361,
+      "learning_rate": 4.626691348094777e-05,
+      "loss": 1.5407,
+      "step": 6634
+    },
+    {
+      "epoch": 0.221516883807498,
+      "grad_norm": 1.5378398895263672,
+      "learning_rate": 4.622264489304762e-05,
+      "loss": 1.5561,
+      "step": 6665
+    },
+    {
+      "epoch": 0.22254719489497474,
+      "grad_norm": 1.554624319076538,
+      "learning_rate": 4.617813681048434e-05,
+      "loss": 1.5859,
+      "step": 6696
+    },
+    {
+      "epoch": 0.22357750598245146,
+      "grad_norm": 1.5356658697128296,
+      "learning_rate": 4.61333897355256e-05,
+      "loss": 1.5531,
+      "step": 6727
+    },
+    {
+      "epoch": 0.22460781706992822,
+      "grad_norm": 1.5534918308258057,
+      "learning_rate": 4.608840417313604e-05,
+      "loss": 1.5774,
+      "step": 6758
+    },
+    {
+      "epoch": 0.22563812815740494,
+      "grad_norm": 1.5660988092422485,
+      "learning_rate": 4.6043180630971646e-05,
+      "loss": 1.5763,
+      "step": 6789
+    },
+    {
+      "epoch": 0.22666843924488167,
+      "grad_norm": 1.4993386268615723,
+      "learning_rate": 4.599771961937391e-05,
+      "loss": 1.5615,
+      "step": 6820
+    },
+    {
+      "epoch": 0.22769875033235842,
+      "grad_norm": 1.4630553722381592,
+      "learning_rate": 4.5952021651364204e-05,
+      "loss": 1.543,
+      "step": 6851
+    },
+    {
+      "epoch": 0.22872906141983515,
+      "grad_norm": 1.470173954963684,
+      "learning_rate": 4.590608724263786e-05,
+      "loss": 1.5674,
+      "step": 6882
+    },
+    {
+      "epoch": 0.22975937250731188,
+      "grad_norm": 1.5867971181869507,
+      "learning_rate": 4.585991691155845e-05,
+      "loss": 1.5702,
+      "step": 6913
+    },
+    {
+      "epoch": 0.23078968359478863,
+      "grad_norm": 1.44207763671875,
+      "learning_rate": 4.581351117915188e-05,
+      "loss": 1.5436,
+      "step": 6944
+    },
+    {
+      "epoch": 0.23181999468226536,
+      "grad_norm": 1.4691039323806763,
+      "learning_rate": 4.5766870569100534e-05,
+      "loss": 1.5465,
+      "step": 6975
+    },
+    {
+      "epoch": 0.23285030576974208,
+      "grad_norm": 1.4807918071746826,
+      "learning_rate": 4.571999560773736e-05,
+      "loss": 1.5564,
+      "step": 7006
+    },
+    {
+      "epoch": 0.23388061685721884,
+      "grad_norm": 1.481487512588501,
+      "learning_rate": 4.5672886824039915e-05,
+      "loss": 1.5466,
+      "step": 7037
+    },
+    {
+      "epoch": 0.23491092794469556,
+      "grad_norm": 1.4518013000488281,
+      "learning_rate": 4.5625544749624435e-05,
+      "loss": 1.5618,
+      "step": 7068
+    },
+    {
+      "epoch": 0.2359412390321723,
+      "grad_norm": 1.4186676740646362,
+      "learning_rate": 4.5577969918739794e-05,
+      "loss": 1.5528,
+      "step": 7099
+    },
+    {
+      "epoch": 0.23697155011964902,
+      "grad_norm": 1.5287110805511475,
+      "learning_rate": 4.5530162868261486e-05,
+      "loss": 1.5457,
+      "step": 7130
+    },
+    {
+      "epoch": 0.23800186120712577,
+      "grad_norm": 1.5516417026519775,
+      "learning_rate": 4.548212413768558e-05,
+      "loss": 1.5467,
+      "step": 7161
+    },
+    {
+      "epoch": 0.2390321722946025,
+      "grad_norm": 1.4710053205490112,
+      "learning_rate": 4.543385426912261e-05,
+      "loss": 1.5431,
+      "step": 7192
+    },
+    {
+      "epoch": 0.24006248338207922,
+      "grad_norm": 1.5005567073822021,
+      "learning_rate": 4.53853538072915e-05,
+      "loss": 1.5592,
+      "step": 7223
+    },
+    {
+      "epoch": 0.24109279446955598,
+      "grad_norm": 1.5864965915679932,
+      "learning_rate": 4.533662329951336e-05,
+      "loss": 1.5694,
+      "step": 7254
+    },
+    {
+      "epoch": 0.2421231055570327,
+      "grad_norm": 1.4661896228790283,
+      "learning_rate": 4.528766329570536e-05,
+      "loss": 1.545,
+      "step": 7285
+    },
+    {
+      "epoch": 0.24315341664450943,
+      "grad_norm": 1.5157560110092163,
+      "learning_rate": 4.523847434837447e-05,
+      "loss": 1.5458,
+      "step": 7316
+    },
+    {
+      "epoch": 0.24418372773198618,
+      "grad_norm": 1.4033585786819458,
+      "learning_rate": 4.518905701261128e-05,
+      "loss": 1.5464,
+      "step": 7347
+    },
+    {
+      "epoch": 0.2452140388194629,
+      "grad_norm": 1.5357593297958374,
+      "learning_rate": 4.5139411846083715e-05,
+      "loss": 1.5497,
+      "step": 7378
+    },
+    {
+      "epoch": 0.24624434990693964,
+      "grad_norm": 1.419507384300232,
+      "learning_rate": 4.508953940903073e-05,
+      "loss": 1.5414,
+      "step": 7409
+    },
+    {
+      "epoch": 0.2472746609944164,
+      "grad_norm": 1.5201773643493652,
+      "learning_rate": 4.5039440264255994e-05,
+      "loss": 1.5503,
+      "step": 7440
+    },
+    {
+      "epoch": 0.24830497208189312,
+      "grad_norm": 1.8000444173812866,
+      "learning_rate": 4.498911497712155e-05,
+      "loss": 1.5448,
+      "step": 7471
+    },
+    {
+      "epoch": 0.24933528316936984,
+      "grad_norm": 1.4876810312271118,
+      "learning_rate": 4.493856411554142e-05,
+      "loss": 1.5524,
+      "step": 7502
+    },
+    {
+      "epoch": 0.25036559425684657,
+      "grad_norm": 1.5130078792572021,
+      "learning_rate": 4.4887788249975206e-05,
+      "loss": 1.5454,
+      "step": 7533
+    },
+    {
+      "epoch": 0.2513959053443233,
+      "grad_norm": 1.4829351902008057,
+      "learning_rate": 4.4836787953421656e-05,
+      "loss": 1.5407,
+      "step": 7564
+    },
+    {
+      "epoch": 0.2524262164318001,
+      "grad_norm": 1.521550178527832,
+      "learning_rate": 4.478556380141218e-05,
+      "loss": 1.5727,
+      "step": 7595
+    },
+    {
+      "epoch": 0.2534565275192768,
+      "grad_norm": 1.4377928972244263,
+      "learning_rate": 4.4734116372004375e-05,
+      "loss": 1.5432,
+      "step": 7626
+    },
+    {
+      "epoch": 0.25448683860675353,
+      "grad_norm": 1.4101744890213013,
+      "learning_rate": 4.4682446245775477e-05,
+      "loss": 1.547,
+      "step": 7657
+    },
+    {
+      "epoch": 0.2555171496942303,
+      "grad_norm": 1.522524356842041,
+      "learning_rate": 4.463055400581586e-05,
+      "loss": 1.5418,
+      "step": 7688
+    },
+    {
+      "epoch": 0.256547460781707,
+      "grad_norm": 1.4160797595977783,
+      "learning_rate": 4.4578440237722374e-05,
+      "loss": 1.5457,
+      "step": 7719
+    },
+    {
+      "epoch": 0.25757777186918374,
+      "grad_norm": 1.4106636047363281,
+      "learning_rate": 4.452610552959183e-05,
+      "loss": 1.5405,
+      "step": 7750
+    },
+    {
+      "epoch": 0.2586080829566605,
+      "grad_norm": 1.422723650932312,
+      "learning_rate": 4.447355047201428e-05,
+      "loss": 1.5423,
+      "step": 7781
+    },
+    {
+      "epoch": 0.2596383940441372,
+      "grad_norm": 1.4362592697143555,
+      "learning_rate": 4.4420775658066414e-05,
+      "loss": 1.5372,
+      "step": 7812
+    },
+    {
+      "epoch": 0.26066870513161394,
+      "grad_norm": 1.4319696426391602,
+      "learning_rate": 4.436778168330484e-05,
+      "loss": 1.5451,
+      "step": 7843
+    },
+    {
+      "epoch": 0.26169901621909064,
+      "grad_norm": 1.4069257974624634,
+      "learning_rate": 4.4314569145759353e-05,
+      "loss": 1.5221,
+      "step": 7874
+    },
+    {
+      "epoch": 0.2627293273065674,
+      "grad_norm": 1.4424949884414673,
+      "learning_rate": 4.42611386459262e-05,
+      "loss": 1.5419,
+      "step": 7905
+    },
+    {
+      "epoch": 0.26375963839404415,
+      "grad_norm": 1.4579105377197266,
+      "learning_rate": 4.420749078676133e-05,
+      "loss": 1.5116,
+      "step": 7936
+    },
+    {
+      "epoch": 0.26478994948152085,
+      "grad_norm": 1.4563167095184326,
+      "learning_rate": 4.4153626173673516e-05,
+      "loss": 1.5296,
+      "step": 7967
+    },
+    {
+      "epoch": 0.2658202605689976,
+      "grad_norm": 1.4440968036651611,
+      "learning_rate": 4.409954541451762e-05,
+      "loss": 1.5548,
+      "step": 7998
+    },
+    {
+      "epoch": 0.26685057165647436,
+      "grad_norm": 1.5711034536361694,
+      "learning_rate": 4.404524911958764e-05,
+      "loss": 1.535,
+      "step": 8029
+    },
+    {
+      "epoch": 0.26788088274395105,
+      "grad_norm": 1.5221564769744873,
+      "learning_rate": 4.399073790160989e-05,
+      "loss": 1.5495,
+      "step": 8060
+    },
+    {
+      "epoch": 0.2689111938314278,
+      "grad_norm": 1.392699956893921,
+      "learning_rate": 4.393601237573607e-05,
+      "loss": 1.546,
+      "step": 8091
+    },
+    {
+      "epoch": 0.26994150491890456,
+      "grad_norm": 1.5343137979507446,
+      "learning_rate": 4.388107315953628e-05,
+      "loss": 1.549,
+      "step": 8122
+    },
+    {
+      "epoch": 0.27097181600638126,
+      "grad_norm": 1.4483468532562256,
+      "learning_rate": 4.382592087299212e-05,
+      "loss": 1.5424,
+      "step": 8153
+    },
+    {
+      "epoch": 0.272002127093858,
+      "grad_norm": 1.4963489770889282,
+      "learning_rate": 4.377055613848964e-05,
+      "loss": 1.508,
+      "step": 8184
+    },
+    {
+      "epoch": 0.27303243818133477,
+      "grad_norm": 1.4839162826538086,
+      "learning_rate": 4.3714979580812355e-05,
+      "loss": 1.5203,
+      "step": 8215
+    },
+    {
+      "epoch": 0.27406274926881147,
+      "grad_norm": 1.4272018671035767,
+      "learning_rate": 4.365919182713416e-05,
+      "loss": 1.5264,
+      "step": 8246
+    },
+    {
+      "epoch": 0.2750930603562882,
+      "grad_norm": 1.3808270692825317,
+      "learning_rate": 4.360319350701226e-05,
+      "loss": 1.5255,
+      "step": 8277
+    },
+    {
+      "epoch": 0.276123371443765,
+      "grad_norm": 1.4179162979125977,
+      "learning_rate": 4.3546985252380115e-05,
+      "loss": 1.535,
+      "step": 8308
+    },
+    {
+      "epoch": 0.2771536825312417,
+      "grad_norm": 1.3617374897003174,
+      "learning_rate": 4.349056769754021e-05,
+      "loss": 1.5295,
+      "step": 8339
+    },
+    {
+      "epoch": 0.27818399361871843,
+      "grad_norm": 1.4745615720748901,
+      "learning_rate": 4.3433941479156994e-05,
+      "loss": 1.5438,
+      "step": 8370
+    },
+    {
+      "epoch": 0.2792143047061952,
+      "grad_norm": 1.3661375045776367,
+      "learning_rate": 4.3377107236249647e-05,
+      "loss": 1.5134,
+      "step": 8401
+    },
+    {
+      "epoch": 0.2802446157936719,
+      "grad_norm": 1.3907949924468994,
+      "learning_rate": 4.332006561018488e-05,
+      "loss": 1.5237,
+      "step": 8432
+    },
+    {
+      "epoch": 0.28127492688114863,
+      "grad_norm": 1.3575704097747803,
+      "learning_rate": 4.3262817244669683e-05,
+      "loss": 1.5226,
+      "step": 8463
+    },
+    {
+      "epoch": 0.2823052379686254,
+      "grad_norm": 1.3836462497711182,
+      "learning_rate": 4.3205362785744083e-05,
+      "loss": 1.5433,
+      "step": 8494
+    },
+    {
+      "epoch": 0.2833355490561021,
+      "grad_norm": 1.6108276844024658,
+      "learning_rate": 4.314770288177384e-05,
+      "loss": 1.5324,
+      "step": 8525
+    },
+    {
+      "epoch": 0.28436586014357884,
+      "grad_norm": 1.4650689363479614,
+      "learning_rate": 4.308983818344313e-05,
+      "loss": 1.535,
+      "step": 8556
+    },
+    {
+      "epoch": 0.2853961712310556,
+      "grad_norm": 1.5836583375930786,
+      "learning_rate": 4.3031769343747206e-05,
+      "loss": 1.5313,
+      "step": 8587
+    },
+    {
+      "epoch": 0.2864264823185323,
+      "grad_norm": 1.5348492860794067,
+      "learning_rate": 4.297349701798505e-05,
+      "loss": 1.5106,
+      "step": 8618
+    },
+    {
+      "epoch": 0.28745679340600905,
+      "grad_norm": 1.4060319662094116,
+      "learning_rate": 4.2915021863751916e-05,
+      "loss": 1.5283,
+      "step": 8649
+    },
+    {
+      "epoch": 0.2884871044934858,
+      "grad_norm": 1.531657099723816,
+      "learning_rate": 4.285634454093198e-05,
+      "loss": 1.5087,
+      "step": 8680
+    },
+    {
+      "epoch": 0.2895174155809625,
+      "grad_norm": 1.4756299257278442,
+      "learning_rate": 4.279746571169086e-05,
+      "loss": 1.5042,
+      "step": 8711
+    },
+    {
+      "epoch": 0.29054772666843925,
+      "grad_norm": 1.3221153020858765,
+      "learning_rate": 4.2738386040468136e-05,
+      "loss": 1.5244,
+      "step": 8742
+    },
+    {
+      "epoch": 0.29157803775591595,
+      "grad_norm": 1.4067268371582031,
+      "learning_rate": 4.2679106193969866e-05,
+      "loss": 1.5012,
+      "step": 8773
+    },
+    {
+      "epoch": 0.2926083488433927,
+      "grad_norm": 1.5192064046859741,
+      "learning_rate": 4.261962684116106e-05,
+      "loss": 1.521,
+      "step": 8804
+    },
+    {
+      "epoch": 0.29363865993086946,
+      "grad_norm": 1.3847788572311401,
+      "learning_rate": 4.2559948653258145e-05,
+      "loss": 1.5128,
+      "step": 8835
+    },
+    {
+      "epoch": 0.29466897101834616,
+      "grad_norm": 1.4612780809402466,
+      "learning_rate": 4.250007230372134e-05,
+      "loss": 1.5371,
+      "step": 8866
+    },
+    {
+      "epoch": 0.2956992821058229,
+      "grad_norm": 1.468971610069275,
+      "learning_rate": 4.2439998468247126e-05,
+      "loss": 1.5199,
+      "step": 8897
+    },
+    {
+      "epoch": 0.29672959319329967,
+      "grad_norm": 1.386236310005188,
+      "learning_rate": 4.2379727824760566e-05,
+      "loss": 1.5273,
+      "step": 8928
+    },
+    {
+      "epoch": 0.29775990428077637,
+      "grad_norm": 1.3843929767608643,
+      "learning_rate": 4.231926105340768e-05,
+      "loss": 1.5011,
+      "step": 8959
+    },
+    {
+      "epoch": 0.2987902153682531,
+      "grad_norm": 1.4554557800292969,
+      "learning_rate": 4.225859883654776e-05,
+      "loss": 1.5311,
+      "step": 8990
+    },
+    {
+      "epoch": 0.2998205264557299,
+      "grad_norm": 1.3674421310424805,
+      "learning_rate": 4.219774185874569e-05,
+      "loss": 1.5302,
+      "step": 9021
+    },
+    {
+      "epoch": 0.3008508375432066,
+      "grad_norm": 1.3804330825805664,
+      "learning_rate": 4.213669080676418e-05,
+      "loss": 1.538,
+      "step": 9052
+    },
+    {
+      "epoch": 0.3018811486306833,
+      "grad_norm": 1.4643255472183228,
+      "learning_rate": 4.2075446369556056e-05,
+      "loss": 1.5172,
+      "step": 9083
+    },
+    {
+      "epoch": 0.3029114597181601,
+      "grad_norm": 1.3375928401947021,
+      "learning_rate": 4.201400923825648e-05,
+      "loss": 1.5123,
+      "step": 9114
+    },
+    {
+      "epoch": 0.3039417708056368,
+      "grad_norm": 1.4321980476379395,
+      "learning_rate": 4.195238010617511e-05,
+      "loss": 1.5196,
+      "step": 9145
+    },
+    {
+      "epoch": 0.30497208189311353,
+      "grad_norm": 1.4312376976013184,
+      "learning_rate": 4.1890559668788344e-05,
+      "loss": 1.5138,
+      "step": 9176
+    },
+    {
+      "epoch": 0.3060023929805903,
+      "grad_norm": 1.3089646100997925,
+      "learning_rate": 4.1828548623731405e-05,
+      "loss": 1.5027,
+      "step": 9207
+    },
+    {
+      "epoch": 0.307032704068067,
+      "grad_norm": 1.4863250255584717,
+      "learning_rate": 4.1766347670790506e-05,
+      "loss": 1.5091,
+      "step": 9238
+    },
+    {
+      "epoch": 0.30806301515554374,
+      "grad_norm": 1.373666763305664,
+      "learning_rate": 4.170395751189495e-05,
+      "loss": 1.5256,
+      "step": 9269
+    },
+    {
+      "epoch": 0.3090933262430205,
+      "grad_norm": 1.4160584211349487,
+      "learning_rate": 4.164137885110921e-05,
+      "loss": 1.4938,
+      "step": 9300
+    },
+    {
+      "epoch": 0.3101236373304972,
+      "grad_norm": 2.112110137939453,
+      "learning_rate": 4.157861239462495e-05,
+      "loss": 1.5106,
+      "step": 9331
+    },
+    {
+      "epoch": 0.31115394841797395,
+      "grad_norm": 1.337058663368225,
+      "learning_rate": 4.1515658850753114e-05,
+      "loss": 1.4999,
+      "step": 9362
+    },
+    {
+      "epoch": 0.3121842595054507,
+      "grad_norm": 1.3625296354293823,
+      "learning_rate": 4.145251892991588e-05,
+      "loss": 1.5136,
+      "step": 9393
+    },
+    {
+      "epoch": 0.3132145705929274,
+      "grad_norm": 1.399491548538208,
+      "learning_rate": 4.138919334463868e-05,
+      "loss": 1.499,
+      "step": 9424
+    },
+    {
+      "epoch": 0.31424488168040415,
+      "grad_norm": 1.4202344417572021,
+      "learning_rate": 4.1325682809542124e-05,
+      "loss": 1.5049,
+      "step": 9455
+    },
+    {
+      "epoch": 0.3152751927678809,
+      "grad_norm": 1.392248272895813,
+      "learning_rate": 4.126198804133398e-05,
+      "loss": 1.5287,
+      "step": 9486
+    },
+    {
+      "epoch": 0.3163055038553576,
+      "grad_norm": 1.3807618618011475,
+      "learning_rate": 4.1198109758801055e-05,
+      "loss": 1.5309,
+      "step": 9517
+    },
+    {
+      "epoch": 0.31733581494283436,
+      "grad_norm": 1.3117905855178833,
+      "learning_rate": 4.113404868280107e-05,
+      "loss": 1.4933,
+      "step": 9548
+    },
+    {
+      "epoch": 0.3183661260303111,
+      "grad_norm": 1.452086091041565,
+      "learning_rate": 4.106980553625457e-05,
+      "loss": 1.5221,
+      "step": 9579
+    },
+    {
+      "epoch": 0.3193964371177878,
+      "grad_norm": 1.477364182472229,
+      "learning_rate": 4.100538104413674e-05,
+      "loss": 1.4904,
+      "step": 9610
+    },
+    {
+      "epoch": 0.32042674820526457,
+      "grad_norm": 1.3584345579147339,
+      "learning_rate": 4.09407759334692e-05,
+      "loss": 1.4953,
+      "step": 9641
+    },
+    {
+      "epoch": 0.32145705929274127,
+      "grad_norm": 1.3619811534881592,
+      "learning_rate": 4.087599093331186e-05,
+      "loss": 1.4956,
+      "step": 9672
+    },
+    {
+      "epoch": 0.322487370380218,
+      "grad_norm": 1.4507052898406982,
+      "learning_rate": 4.081102677475462e-05,
+      "loss": 1.5197,
+      "step": 9703
+    },
+    {
+      "epoch": 0.3235176814676948,
+      "grad_norm": 1.4229698181152344,
+      "learning_rate": 4.0745884190909194e-05,
+      "loss": 1.498,
+      "step": 9734
+    },
+    {
+      "epoch": 0.32454799255517147,
+      "grad_norm": 1.3074679374694824,
+      "learning_rate": 4.0680563916900796e-05,
+      "loss": 1.5146,
+      "step": 9765
+    },
+    {
+      "epoch": 0.3255783036426482,
+      "grad_norm": 1.397815465927124,
+      "learning_rate": 4.0615066689859815e-05,
+      "loss": 1.5291,
+      "step": 9796
+    },
+    {
+      "epoch": 0.326608614730125,
+      "grad_norm": 1.3196336030960083,
+      "learning_rate": 4.0549393248913584e-05,
+      "loss": 1.5077,
+      "step": 9827
+    },
+    {
+      "epoch": 0.3276389258176017,
+      "grad_norm": 1.3129957914352417,
+      "learning_rate": 4.048354433517794e-05,
+      "loss": 1.4965,
+      "step": 9858
+    },
+    {
+      "epoch": 0.32866923690507843,
+      "grad_norm": 1.4380089044570923,
+      "learning_rate": 4.0417520691748916e-05,
+      "loss": 1.5115,
+      "step": 9889
+    },
+    {
+      "epoch": 0.3296995479925552,
+      "grad_norm": 1.3162370920181274,
+      "learning_rate": 4.035132306369438e-05,
+      "loss": 1.5029,
+      "step": 9920
+    },
+    {
+      "epoch": 0.3307298590800319,
+      "grad_norm": 1.3739668130874634,
+      "learning_rate": 4.028495219804555e-05,
+      "loss": 1.5083,
+      "step": 9951
+    },
+    {
+      "epoch": 0.33176017016750864,
+      "grad_norm": 1.3673723936080933,
+      "learning_rate": 4.021840884378864e-05,
+      "loss": 1.5223,
+      "step": 9982
+    },
+    {
+      "epoch": 0.3327904812549854,
+      "grad_norm": 1.3970317840576172,
+      "learning_rate": 4.015169375185633e-05,
+      "loss": 1.5003,
+      "step": 10013
+    },
+    {
+      "epoch": 0.3338207923424621,
+      "grad_norm": 1.2982394695281982,
+      "learning_rate": 4.0084807675119396e-05,
+      "loss": 1.5066,
+      "step": 10044
+    },
+    {
+      "epoch": 0.33485110342993885,
+      "grad_norm": 1.4548689126968384,
+      "learning_rate": 4.0017751368378106e-05,
+      "loss": 1.4993,
+      "step": 10075
+    },
+    {
+      "epoch": 0.3358814145174156,
+      "grad_norm": 1.3693586587905884,
+      "learning_rate": 3.995052558835377e-05,
+      "loss": 1.4987,
+      "step": 10106
+    },
+    {
+      "epoch": 0.3369117256048923,
+      "grad_norm": 1.4046767950057983,
+      "learning_rate": 3.988313109368017e-05,
+      "loss": 1.5098,
+      "step": 10137
+    },
+    {
+      "epoch": 0.33794203669236905,
+      "grad_norm": 1.3772069215774536,
+      "learning_rate": 3.981556864489504e-05,
+      "loss": 1.5165,
+      "step": 10168
+    },
+    {
+      "epoch": 0.3389723477798458,
+      "grad_norm": 1.471211314201355,
+      "learning_rate": 3.974783900443142e-05,
+      "loss": 1.5037,
+      "step": 10199
+    },
+    {
+      "epoch": 0.3400026588673225,
+      "grad_norm": 1.3990979194641113,
+      "learning_rate": 3.9679942936609095e-05,
+      "loss": 1.5096,
+      "step": 10230
+    },
+    {
+      "epoch": 0.34103296995479926,
+      "grad_norm": 1.3779234886169434,
+      "learning_rate": 3.961188120762596e-05,
+      "loss": 1.4914,
+      "step": 10261
+    },
+    {
+      "epoch": 0.342063281042276,
+      "grad_norm": 1.2866768836975098,
+      "learning_rate": 3.954365458554938e-05,
+      "loss": 1.5026,
+      "step": 10292
+    },
+    {
+      "epoch": 0.3430935921297527,
+      "grad_norm": 1.353468894958496,
+      "learning_rate": 3.947526384030751e-05,
+      "loss": 1.5063,
+      "step": 10323
+    },
+    {
+      "epoch": 0.34412390321722947,
+      "grad_norm": 1.3264256715774536,
+      "learning_rate": 3.9406709743680624e-05,
+      "loss": 1.4911,
+      "step": 10354
+    },
+    {
+      "epoch": 0.3451542143047062,
+      "grad_norm": 1.3496876955032349,
+      "learning_rate": 3.9337993069292366e-05,
+      "loss": 1.4921,
+      "step": 10385
+    },
+    {
+      "epoch": 0.3461845253921829,
+      "grad_norm": 1.3812434673309326,
+      "learning_rate": 3.926911459260109e-05,
+      "loss": 1.4826,
+      "step": 10416
+    },
+    {
+      "epoch": 0.34721483647965967,
+      "grad_norm": 1.4926965236663818,
+      "learning_rate": 3.920007509089102e-05,
+      "loss": 1.4994,
+      "step": 10447
+    },
+    {
+      "epoch": 0.3482451475671364,
+      "grad_norm": 1.3446170091629028,
+      "learning_rate": 3.913087534326357e-05,
+      "loss": 1.5114,
+      "step": 10478
+    },
+    {
+      "epoch": 0.3492754586546131,
+      "grad_norm": 1.3100495338439941,
+      "learning_rate": 3.9061516130628475e-05,
+      "loss": 1.5066,
+      "step": 10509
+    },
+    {
+      "epoch": 0.3503057697420899,
+      "grad_norm": 1.395874261856079,
+      "learning_rate": 3.8991998235695025e-05,
+      "loss": 1.4999,
+      "step": 10540
+    },
+    {
+      "epoch": 0.3513360808295666,
+      "grad_norm": 1.3682137727737427,
+      "learning_rate": 3.8922322442963224e-05,
+      "loss": 1.4778,
+      "step": 10571
+    },
+    {
+      "epoch": 0.35236639191704333,
+      "grad_norm": 1.4196573495864868,
+      "learning_rate": 3.885248953871491e-05,
+      "loss": 1.4909,
+      "step": 10602
+    },
+    {
+      "epoch": 0.3533967030045201,
+      "grad_norm": 1.4299864768981934,
+      "learning_rate": 3.8782500311004915e-05,
+      "loss": 1.5025,
+      "step": 10633
+    },
+    {
+      "epoch": 0.3544270140919968,
+      "grad_norm": 1.39677095413208,
+      "learning_rate": 3.871235554965218e-05,
+      "loss": 1.4932,
+      "step": 10664
+    },
+    {
+      "epoch": 0.35545732517947354,
+      "grad_norm": 1.3219736814498901,
+      "learning_rate": 3.864205604623078e-05,
+      "loss": 1.4795,
+      "step": 10695
+    },
+    {
+      "epoch": 0.3564876362669503,
+      "grad_norm": 1.3649324178695679,
+      "learning_rate": 3.857160259406107e-05,
+      "loss": 1.4838,
+      "step": 10726
+    },
+    {
+      "epoch": 0.357517947354427,
+      "grad_norm": 1.4109989404678345,
+      "learning_rate": 3.8500995988200674e-05,
+      "loss": 1.5058,
+      "step": 10757
+    },
+    {
+      "epoch": 0.35854825844190374,
+      "grad_norm": 1.3625038862228394,
+      "learning_rate": 3.843023702543556e-05,
+      "loss": 1.4912,
+      "step": 10788
+    },
+    {
+      "epoch": 0.3595785695293805,
+      "grad_norm": 1.4725775718688965,
+      "learning_rate": 3.8359326504270984e-05,
+      "loss": 1.5012,
+      "step": 10819
+    },
+    {
+      "epoch": 0.3606088806168572,
+      "grad_norm": 1.4126085042953491,
+      "learning_rate": 3.828826522492255e-05,
+      "loss": 1.4977,
+      "step": 10850
+    },
+    {
+      "epoch": 0.36163919170433395,
+      "grad_norm": 1.3949086666107178,
+      "learning_rate": 3.821705398930713e-05,
+      "loss": 1.4903,
+      "step": 10881
+    },
+    {
+      "epoch": 0.3626695027918107,
+      "grad_norm": 1.286792516708374,
+      "learning_rate": 3.814569360103385e-05,
+      "loss": 1.5067,
+      "step": 10912
+    },
+    {
+      "epoch": 0.3636998138792874,
+      "grad_norm": 1.274703025817871,
+      "learning_rate": 3.807418486539499e-05,
+      "loss": 1.4583,
+      "step": 10943
+    },
+    {
+      "epoch": 0.36473012496676416,
+      "grad_norm": 1.401455283164978,
+      "learning_rate": 3.80025285893569e-05,
+      "loss": 1.4834,
+      "step": 10974
+    },
+    {
+      "epoch": 0.3657604360542409,
+      "grad_norm": 1.308361530303955,
+      "learning_rate": 3.793072558155093e-05,
+      "loss": 1.4832,
+      "step": 11005
+    },
+    {
+      "epoch": 0.3667907471417176,
+      "grad_norm": 1.654733419418335,
+      "learning_rate": 3.785877665226426e-05,
+      "loss": 1.4867,
+      "step": 11036
+    },
+    {
+      "epoch": 0.36782105822919436,
+      "grad_norm": 1.3530856370925903,
+      "learning_rate": 3.778668261343079e-05,
+      "loss": 1.4873,
+      "step": 11067
+    },
+    {
+      "epoch": 0.3688513693166711,
+      "grad_norm": 1.3567407131195068,
+      "learning_rate": 3.771444427862192e-05,
+      "loss": 1.4935,
+      "step": 11098
+    },
+    {
+      "epoch": 0.3698816804041478,
+      "grad_norm": 1.3184572458267212,
+      "learning_rate": 3.7642062463037465e-05,
+      "loss": 1.4891,
+      "step": 11129
+    },
+    {
+      "epoch": 0.37091199149162457,
+      "grad_norm": 1.366489291191101,
+      "learning_rate": 3.7569537983496373e-05,
+      "loss": 1.5159,
+      "step": 11160
+    },
+    {
+      "epoch": 0.3719423025791013,
+      "grad_norm": 1.423258662223816,
+      "learning_rate": 3.749687165842753e-05,
+      "loss": 1.4938,
+      "step": 11191
+    },
+    {
+      "epoch": 0.372972613666578,
+      "grad_norm": 1.3226194381713867,
+      "learning_rate": 3.7424064307860536e-05,
+      "loss": 1.499,
+      "step": 11222
+    },
+    {
+      "epoch": 0.3740029247540548,
+      "grad_norm": 1.350500464439392,
+      "learning_rate": 3.735111675341645e-05,
+      "loss": 1.4952,
+      "step": 11253
+    },
+    {
+      "epoch": 0.37503323584153153,
+      "grad_norm": 1.3667839765548706,
+      "learning_rate": 3.7278029818298524e-05,
+      "loss": 1.4763,
+      "step": 11284
+    },
+    {
+      "epoch": 0.37606354692900823,
+      "grad_norm": 1.4876132011413574,
+      "learning_rate": 3.720480432728287e-05,
+      "loss": 1.4913,
+      "step": 11315
+    },
+    {
+      "epoch": 0.377093858016485,
+      "grad_norm": 1.3927743434906006,
+      "learning_rate": 3.71314411067092e-05,
+      "loss": 1.4948,
+      "step": 11346
+    },
+    {
+      "epoch": 0.37812416910396174,
+      "grad_norm": 1.3752413988113403,
+      "learning_rate": 3.70579409844715e-05,
+      "loss": 1.4763,
+      "step": 11377
+    },
+    {
+      "epoch": 0.37915448019143844,
+      "grad_norm": 1.3530951738357544,
+      "learning_rate": 3.698430479000865e-05,
+      "loss": 1.5077,
+      "step": 11408
+    },
+    {
+      "epoch": 0.3801847912789152,
+      "grad_norm": 1.4309345483779907,
+      "learning_rate": 3.691053335429509e-05,
+      "loss": 1.4945,
+      "step": 11439
+    },
+    {
+      "epoch": 0.38121510236639194,
+      "grad_norm": 1.2874380350112915,
+      "learning_rate": 3.683662750983147e-05,
+      "loss": 1.4698,
+      "step": 11470
+    },
+    {
+      "epoch": 0.38224541345386864,
+      "grad_norm": 1.3356250524520874,
+      "learning_rate": 3.676258809063518e-05,
+      "loss": 1.4924,
+      "step": 11501
+    },
+    {
+      "epoch": 0.3832757245413454,
+      "grad_norm": 1.304559588432312,
+      "learning_rate": 3.6688415932231004e-05,
+      "loss": 1.4682,
+      "step": 11532
+    },
+    {
+      "epoch": 0.3843060356288221,
+      "grad_norm": 1.4153447151184082,
+      "learning_rate": 3.661411187164166e-05,
+      "loss": 1.4989,
+      "step": 11563
+    },
+    {
+      "epoch": 0.38533634671629885,
+      "grad_norm": 1.356992244720459,
+      "learning_rate": 3.65396767473784e-05,
+      "loss": 1.4854,
+      "step": 11594
+    },
+    {
+      "epoch": 0.3863666578037756,
+      "grad_norm": 1.322449803352356,
+      "learning_rate": 3.6465111399431465e-05,
+      "loss": 1.4877,
+      "step": 11625
+    },
+    {
+      "epoch": 0.3873969688912523,
+      "grad_norm": 1.3981350660324097,
+      "learning_rate": 3.6390416669260674e-05,
+      "loss": 1.499,
+      "step": 11656
+    },
+    {
+      "epoch": 0.38842727997872906,
+      "grad_norm": 1.324871301651001,
+      "learning_rate": 3.63155933997859e-05,
+      "loss": 1.4814,
+      "step": 11687
+    },
+    {
+      "epoch": 0.3894575910662058,
+      "grad_norm": 1.3940790891647339,
+      "learning_rate": 3.624064243537758e-05,
+      "loss": 1.4754,
+      "step": 11718
+    },
+    {
+      "epoch": 0.3904879021536825,
+      "grad_norm": 1.2880780696868896,
+      "learning_rate": 3.616556462184716e-05,
+      "loss": 1.4832,
+      "step": 11749
+    },
+    {
+      "epoch": 0.39151821324115926,
+      "grad_norm": 1.315329670906067,
+      "learning_rate": 3.609036080643755e-05,
+      "loss": 1.4853,
+      "step": 11780
+    },
+    {
+      "epoch": 0.392548524328636,
+      "grad_norm": 1.4093523025512695,
+      "learning_rate": 3.60150318378136e-05,
+      "loss": 1.4978,
+      "step": 11811
+    },
+    {
+      "epoch": 0.3935788354161127,
+      "grad_norm": 1.271151065826416,
+      "learning_rate": 3.5939578566052465e-05,
+      "loss": 1.4933,
+      "step": 11842
+    },
+    {
+      "epoch": 0.39460914650358947,
+      "grad_norm": 1.2910923957824707,
+      "learning_rate": 3.586400184263408e-05,
+      "loss": 1.4853,
+      "step": 11873
+    },
+    {
+      "epoch": 0.3956394575910662,
+      "grad_norm": 1.2480064630508423,
+      "learning_rate": 3.578830252043148e-05,
+      "loss": 1.4642,
+      "step": 11904
+    },
+    {
+      "epoch": 0.3966697686785429,
+      "grad_norm": 1.263197422027588,
+      "learning_rate": 3.571248145370125e-05,
+      "loss": 1.4812,
+      "step": 11935
+    },
+    {
+      "epoch": 0.3977000797660197,
+      "grad_norm": 1.3231288194656372,
+      "learning_rate": 3.5636539498073794e-05,
+      "loss": 1.4744,
+      "step": 11966
+    },
+    {
+      "epoch": 0.39873039085349643,
+      "grad_norm": 1.3933110237121582,
+      "learning_rate": 3.556047751054378e-05,
+      "loss": 1.4849,
+      "step": 11997
+    },
+    {
+      "epoch": 0.39976070194097313,
+      "grad_norm": 1.3615801334381104,
+      "learning_rate": 3.548429634946039e-05,
+      "loss": 1.4866,
+      "step": 12028
+    },
+    {
+      "epoch": 0.4007910130284499,
+      "grad_norm": 1.298638939857483,
+      "learning_rate": 3.540799687451768e-05,
+      "loss": 1.4664,
+      "step": 12059
+    },
+    {
+      "epoch": 0.40182132411592664,
+      "grad_norm": 1.29216468334198,
+      "learning_rate": 3.533157994674485e-05,
+      "loss": 1.4697,
+      "step": 12090
+    },
+    {
+      "epoch": 0.40285163520340334,
+      "grad_norm": 1.3759845495224,
+      "learning_rate": 3.5255046428496546e-05,
+      "loss": 1.4854,
+      "step": 12121
+    },
+    {
+      "epoch": 0.4038819462908801,
+      "grad_norm": 1.4045615196228027,
+      "learning_rate": 3.517839718344311e-05,
+      "loss": 1.4622,
+      "step": 12152
+    },
+    {
+      "epoch": 0.40491225737835684,
+      "grad_norm": 1.2979034185409546,
+      "learning_rate": 3.510163307656086e-05,
+      "loss": 1.4797,
+      "step": 12183
+    }
+  ],
+  "logging_steps": 31,
+  "max_steps": 30517,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 2,
+  "save_steps": 3052,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 9.055780216089084e+18,
+  "train_batch_size": 8,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-12208/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a5a23be0ff07d6d3142f7c0980f91dddba845519c24fcb411cbb4b9ddb1513ff
+size 5304

checkpoint-15260/config.json

@@ -0,0 +1,39 @@
+{
+  "_name_or_path": "/mnt/parscratch/users/acp23ay/private/models/Llama-3.1-8B-Instruct-ta-madlad-mean/",
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 128000,
+  "eos_token_id": [
+    128001,
+    128008,
+    128009
+  ],
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 131072,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 8,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": {
+    "factor": 8.0,
+    "high_freq_factor": 4.0,
+    "low_freq_factor": 1.0,
+    "original_max_position_embeddings": 8192,
+    "rope_type": "llama3"
+  },
+  "rope_theta": 500000.0,
+  "tie_word_embeddings": false,
+  "torch_dtype": "float32",
+  "transformers_version": "4.44.0.dev0",
+  "use_cache": true,
+  "vocab_size": 138256
+}

checkpoint-15260/generation_config.json

@@ -0,0 +1,12 @@
+{
+  "bos_token_id": 128000,
+  "do_sample": true,
+  "eos_token_id": [
+    128001,
+    128008,
+    128009
+  ],
+  "temperature": 0.6,
+  "top_p": 0.9,
+  "transformers_version": "4.44.0.dev0"
+}

checkpoint-15260/model-00001-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ea0f35225868c2f9f8770492cc681c525266286582cc49af99ef751c57491ac0
+size 4983197184

checkpoint-15260/model-00002-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:168629c67732309d49f60a5ec48a6d160212bc987365e82e183dfbf74ba0c1f3
+size 4899116432

checkpoint-15260/model-00003-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1a223e87240e50dfb05634ea909cf1ec715d37a12b681b14a3a6e97f28688f2b
+size 4999813112

checkpoint-15260/model-00004-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3047d199c611c6d12b1cca780f033ad89df9c076f742674f7ac4a892ca9573b3
+size 4999813128

checkpoint-15260/model-00005-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0a2f91d749d664ecd22c71f3d54488966fc6899a03924b8fea3613c8f83398a8
+size 4832007496

checkpoint-15260/model-00006-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:df9d6dc5fc81adef766e05947ef61bc406443da64661faffeeb59936c5d5d820
+size 4999813120

checkpoint-15260/model-00007-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3e0b06bfc5416ea4594646dde5eb4b46407b14a7966270db1b3615f59746734c
+size 2734998184

checkpoint-15260/model.safetensors.index.json

@@ -0,0 +1,298 @@
+{
+  "metadata": {
+    "total_size": 32448724992
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00007-of-00007.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00007-of-00007.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00007-of-00007.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00007-of-00007.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00007-of-00007.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "model.norm.weight": "model-00007-of-00007.safetensors"
+  }
+}

checkpoint-15260/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ebb046bf713a5a9daa84654fa90f5a2c50f9ed6b428513e61d24fad0c1c30fd4
+size 16040396334

checkpoint-15260/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9d9cd6a0487226e5bd30d1846894c82af483733ab4381b75bae9c0745e05d405
+size 14244

checkpoint-15260/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2ce335347efaebf038b8915bc0a8d2d587a4a1aa08ad0e015b4bc7cc4fba634e
+size 1064

checkpoint-15260/trainer_state.json

@@ -0,0 +1,3477 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.5071789417708057,
+  "eval_steps": 500,
+  "global_step": 15260,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.001030311087476735,
+      "grad_norm": 60.25547409057617,
+      "learning_rate": 1.0157273918741808e-06,
+      "loss": 8.8455,
+      "step": 31
+    },
+    {
+      "epoch": 0.00206062217495347,
+      "grad_norm": 15.669363975524902,
+      "learning_rate": 2.0314547837483616e-06,
+      "loss": 7.1553,
+      "step": 62
+    },
+    {
+      "epoch": 0.003090933262430205,
+      "grad_norm": 15.366345405578613,
+      "learning_rate": 3.0471821756225426e-06,
+      "loss": 5.8784,
+      "step": 93
+    },
+    {
+      "epoch": 0.00412124434990694,
+      "grad_norm": 36.30561828613281,
+      "learning_rate": 4.062909567496723e-06,
+      "loss": 4.7708,
+      "step": 124
+    },
+    {
+      "epoch": 0.005151555437383675,
+      "grad_norm": 27.202678680419922,
+      "learning_rate": 5.078636959370905e-06,
+      "loss": 4.1629,
+      "step": 155
+    },
+    {
+      "epoch": 0.00618186652486041,
+      "grad_norm": 24.30484962463379,
+      "learning_rate": 6.094364351245085e-06,
+      "loss": 3.867,
+      "step": 186
+    },
+    {
+      "epoch": 0.007212177612337145,
+      "grad_norm": 19.916366577148438,
+      "learning_rate": 7.110091743119267e-06,
+      "loss": 3.6131,
+      "step": 217
+    },
+    {
+      "epoch": 0.00824248869981388,
+      "grad_norm": 17.577274322509766,
+      "learning_rate": 8.125819134993446e-06,
+      "loss": 3.4772,
+      "step": 248
+    },
+    {
+      "epoch": 0.009272799787290615,
+      "grad_norm": 12.133190155029297,
+      "learning_rate": 9.141546526867629e-06,
+      "loss": 3.3218,
+      "step": 279
+    },
+    {
+      "epoch": 0.01030311087476735,
+      "grad_norm": 19.79263687133789,
+      "learning_rate": 1.015727391874181e-05,
+      "loss": 3.2055,
+      "step": 310
+    },
+    {
+      "epoch": 0.011333421962244085,
+      "grad_norm": 16.38133430480957,
+      "learning_rate": 1.117300131061599e-05,
+      "loss": 3.1062,
+      "step": 341
+    },
+    {
+      "epoch": 0.01236373304972082,
+      "grad_norm": 12.638299942016602,
+      "learning_rate": 1.218872870249017e-05,
+      "loss": 3.0106,
+      "step": 372
+    },
+    {
+      "epoch": 0.013394044137197554,
+      "grad_norm": 9.46596908569336,
+      "learning_rate": 1.3204456094364351e-05,
+      "loss": 2.924,
+      "step": 403
+    },
+    {
+      "epoch": 0.01442435522467429,
+      "grad_norm": 10.945392608642578,
+      "learning_rate": 1.4220183486238533e-05,
+      "loss": 2.844,
+      "step": 434
+    },
+    {
+      "epoch": 0.015454666312151024,
+      "grad_norm": 8.474015235900879,
+      "learning_rate": 1.5235910878112714e-05,
+      "loss": 2.7892,
+      "step": 465
+    },
+    {
+      "epoch": 0.01648497739962776,
+      "grad_norm": 9.370804786682129,
+      "learning_rate": 1.6251638269986893e-05,
+      "loss": 2.7509,
+      "step": 496
+    },
+    {
+      "epoch": 0.017515288487104493,
+      "grad_norm": 11.63398551940918,
+      "learning_rate": 1.7267365661861077e-05,
+      "loss": 2.6999,
+      "step": 527
+    },
+    {
+      "epoch": 0.01854559957458123,
+      "grad_norm": 9.17713451385498,
+      "learning_rate": 1.8283093053735257e-05,
+      "loss": 2.6459,
+      "step": 558
+    },
+    {
+      "epoch": 0.019575910662057962,
+      "grad_norm": 7.119054794311523,
+      "learning_rate": 1.9298820445609438e-05,
+      "loss": 2.603,
+      "step": 589
+    },
+    {
+      "epoch": 0.0206062217495347,
+      "grad_norm": 6.653646945953369,
+      "learning_rate": 2.031454783748362e-05,
+      "loss": 2.5588,
+      "step": 620
+    },
+    {
+      "epoch": 0.021636532837011432,
+      "grad_norm": 8.332653045654297,
+      "learning_rate": 2.13302752293578e-05,
+      "loss": 2.5357,
+      "step": 651
+    },
+    {
+      "epoch": 0.02266684392448817,
+      "grad_norm": 6.4949116706848145,
+      "learning_rate": 2.234600262123198e-05,
+      "loss": 2.4967,
+      "step": 682
+    },
+    {
+      "epoch": 0.023697155011964902,
+      "grad_norm": 9.41009521484375,
+      "learning_rate": 2.336173001310616e-05,
+      "loss": 2.4563,
+      "step": 713
+    },
+    {
+      "epoch": 0.02472746609944164,
+      "grad_norm": 7.840345859527588,
+      "learning_rate": 2.437745740498034e-05,
+      "loss": 2.4383,
+      "step": 744
+    },
+    {
+      "epoch": 0.025757777186918372,
+      "grad_norm": 6.116458415985107,
+      "learning_rate": 2.5393184796854525e-05,
+      "loss": 2.3817,
+      "step": 775
+    },
+    {
+      "epoch": 0.02678808827439511,
+      "grad_norm": 5.938300609588623,
+      "learning_rate": 2.6408912188728702e-05,
+      "loss": 2.3508,
+      "step": 806
+    },
+    {
+      "epoch": 0.027818399361871842,
+      "grad_norm": 5.4408345222473145,
+      "learning_rate": 2.7424639580602886e-05,
+      "loss": 2.3325,
+      "step": 837
+    },
+    {
+      "epoch": 0.02884871044934858,
+      "grad_norm": 5.375136375427246,
+      "learning_rate": 2.8440366972477066e-05,
+      "loss": 2.3101,
+      "step": 868
+    },
+    {
+      "epoch": 0.029879021536825312,
+      "grad_norm": 5.149726867675781,
+      "learning_rate": 2.9456094364351244e-05,
+      "loss": 2.282,
+      "step": 899
+    },
+    {
+      "epoch": 0.03090933262430205,
+      "grad_norm": 4.591221332550049,
+      "learning_rate": 3.0471821756225428e-05,
+      "loss": 2.2427,
+      "step": 930
+    },
+    {
+      "epoch": 0.031939643711778785,
+      "grad_norm": 4.977034091949463,
+      "learning_rate": 3.148754914809961e-05,
+      "loss": 2.2218,
+      "step": 961
+    },
+    {
+      "epoch": 0.03296995479925552,
+      "grad_norm": 5.038781642913818,
+      "learning_rate": 3.2503276539973785e-05,
+      "loss": 2.2044,
+      "step": 992
+    },
+    {
+      "epoch": 0.03400026588673225,
+      "grad_norm": 4.872281551361084,
+      "learning_rate": 3.351900393184797e-05,
+      "loss": 2.1657,
+      "step": 1023
+    },
+    {
+      "epoch": 0.035030576974208985,
+      "grad_norm": 4.370841979980469,
+      "learning_rate": 3.453473132372215e-05,
+      "loss": 2.1365,
+      "step": 1054
+    },
+    {
+      "epoch": 0.036060888061685725,
+      "grad_norm": 4.087072849273682,
+      "learning_rate": 3.555045871559633e-05,
+      "loss": 2.1253,
+      "step": 1085
+    },
+    {
+      "epoch": 0.03709119914916246,
+      "grad_norm": 4.113957405090332,
+      "learning_rate": 3.6566186107470514e-05,
+      "loss": 2.0973,
+      "step": 1116
+    },
+    {
+      "epoch": 0.03812151023663919,
+      "grad_norm": 4.0119733810424805,
+      "learning_rate": 3.7581913499344695e-05,
+      "loss": 2.1024,
+      "step": 1147
+    },
+    {
+      "epoch": 0.039151821324115925,
+      "grad_norm": 4.247573375701904,
+      "learning_rate": 3.8597640891218876e-05,
+      "loss": 2.0722,
+      "step": 1178
+    },
+    {
+      "epoch": 0.04018213241159266,
+      "grad_norm": 3.5575129985809326,
+      "learning_rate": 3.9613368283093056e-05,
+      "loss": 2.056,
+      "step": 1209
+    },
+    {
+      "epoch": 0.0412124434990694,
+      "grad_norm": 3.8885862827301025,
+      "learning_rate": 4.062909567496724e-05,
+      "loss": 2.0389,
+      "step": 1240
+    },
+    {
+      "epoch": 0.04224275458654613,
+      "grad_norm": 3.680628538131714,
+      "learning_rate": 4.164482306684142e-05,
+      "loss": 2.0385,
+      "step": 1271
+    },
+    {
+      "epoch": 0.043273065674022865,
+      "grad_norm": 3.780876874923706,
+      "learning_rate": 4.26605504587156e-05,
+      "loss": 2.0097,
+      "step": 1302
+    },
+    {
+      "epoch": 0.0443033767614996,
+      "grad_norm": 4.235328674316406,
+      "learning_rate": 4.367627785058978e-05,
+      "loss": 2.0024,
+      "step": 1333
+    },
+    {
+      "epoch": 0.04533368784897634,
+      "grad_norm": 3.326941967010498,
+      "learning_rate": 4.469200524246396e-05,
+      "loss": 1.9953,
+      "step": 1364
+    },
+    {
+      "epoch": 0.04636399893645307,
+      "grad_norm": 3.28456449508667,
+      "learning_rate": 4.570773263433814e-05,
+      "loss": 1.9579,
+      "step": 1395
+    },
+    {
+      "epoch": 0.047394310023929805,
+      "grad_norm": 16.107433319091797,
+      "learning_rate": 4.672346002621232e-05,
+      "loss": 1.9701,
+      "step": 1426
+    },
+    {
+      "epoch": 0.04842462111140654,
+      "grad_norm": 3.5708224773406982,
+      "learning_rate": 4.77391874180865e-05,
+      "loss": 1.9621,
+      "step": 1457
+    },
+    {
+      "epoch": 0.04945493219888328,
+      "grad_norm": 2.9053499698638916,
+      "learning_rate": 4.875491480996068e-05,
+      "loss": 1.9458,
+      "step": 1488
+    },
+    {
+      "epoch": 0.05048524328636001,
+      "grad_norm": 3.0863258838653564,
+      "learning_rate": 4.977064220183487e-05,
+      "loss": 1.9483,
+      "step": 1519
+    },
+    {
+      "epoch": 0.051515554373836744,
+      "grad_norm": 2.9012269973754883,
+      "learning_rate": 4.9999915451558777e-05,
+      "loss": 1.928,
+      "step": 1550
+    },
+    {
+      "epoch": 0.05254586546131348,
+      "grad_norm": 3.0949041843414307,
+      "learning_rate": 4.999955597496219e-05,
+      "loss": 1.9229,
+      "step": 1581
+    },
+    {
+      "epoch": 0.05357617654879022,
+      "grad_norm": 2.8687901496887207,
+      "learning_rate": 4.9998914381774255e-05,
+      "loss": 1.915,
+      "step": 1612
+    },
+    {
+      "epoch": 0.05460648763626695,
+      "grad_norm": 3.2136878967285156,
+      "learning_rate": 4.999799067923527e-05,
+      "loss": 1.9197,
+      "step": 1643
+    },
+    {
+      "epoch": 0.055636798723743684,
+      "grad_norm": 2.590843677520752,
+      "learning_rate": 4.999678487776908e-05,
+      "loss": 1.8756,
+      "step": 1674
+    },
+    {
+      "epoch": 0.05666710981122042,
+      "grad_norm": 2.64634108543396,
+      "learning_rate": 4.9995296990983006e-05,
+      "loss": 1.9033,
+      "step": 1705
+    },
+    {
+      "epoch": 0.05769742089869716,
+      "grad_norm": 3.0151331424713135,
+      "learning_rate": 4.999352703566763e-05,
+      "loss": 1.8883,
+      "step": 1736
+    },
+    {
+      "epoch": 0.05872773198617389,
+      "grad_norm": 2.526806354522705,
+      "learning_rate": 4.999147503179668e-05,
+      "loss": 1.8666,
+      "step": 1767
+    },
+    {
+      "epoch": 0.059758043073650624,
+      "grad_norm": 2.510300397872925,
+      "learning_rate": 4.998914100252672e-05,
+      "loss": 1.854,
+      "step": 1798
+    },
+    {
+      "epoch": 0.06078835416112736,
+      "grad_norm": 2.4867682456970215,
+      "learning_rate": 4.998652497419696e-05,
+      "loss": 1.8548,
+      "step": 1829
+    },
+    {
+      "epoch": 0.0618186652486041,
+      "grad_norm": 2.3920586109161377,
+      "learning_rate": 4.9983626976328927e-05,
+      "loss": 1.8495,
+      "step": 1860
+    },
+    {
+      "epoch": 0.06284897633608083,
+      "grad_norm": 2.714177370071411,
+      "learning_rate": 4.998044704162613e-05,
+      "loss": 1.8433,
+      "step": 1891
+    },
+    {
+      "epoch": 0.06387928742355757,
+      "grad_norm": 2.3094465732574463,
+      "learning_rate": 4.9976985205973705e-05,
+      "loss": 1.8382,
+      "step": 1922
+    },
+    {
+      "epoch": 0.0649095985110343,
+      "grad_norm": 2.47184419631958,
+      "learning_rate": 4.997324150843799e-05,
+      "loss": 1.8464,
+      "step": 1953
+    },
+    {
+      "epoch": 0.06593990959851104,
+      "grad_norm": 2.391841411590576,
+      "learning_rate": 4.99692159912661e-05,
+      "loss": 1.8179,
+      "step": 1984
+    },
+    {
+      "epoch": 0.06697022068598776,
+      "grad_norm": 2.2471864223480225,
+      "learning_rate": 4.996490869988546e-05,
+      "loss": 1.8149,
+      "step": 2015
+    },
+    {
+      "epoch": 0.0680005317734645,
+      "grad_norm": 2.5497376918792725,
+      "learning_rate": 4.996031968290326e-05,
+      "loss": 1.8099,
+      "step": 2046
+    },
+    {
+      "epoch": 0.06903084286094124,
+      "grad_norm": 2.330463409423828,
+      "learning_rate": 4.995544899210594e-05,
+      "loss": 1.8267,
+      "step": 2077
+    },
+    {
+      "epoch": 0.07006115394841797,
+      "grad_norm": 2.3259341716766357,
+      "learning_rate": 4.9950296682458583e-05,
+      "loss": 1.7801,
+      "step": 2108
+    },
+    {
+      "epoch": 0.07109146503589471,
+      "grad_norm": 2.1711952686309814,
+      "learning_rate": 4.994486281210429e-05,
+      "loss": 1.7961,
+      "step": 2139
+    },
+    {
+      "epoch": 0.07212177612337145,
+      "grad_norm": 2.1808884143829346,
+      "learning_rate": 4.9939147442363566e-05,
+      "loss": 1.8109,
+      "step": 2170
+    },
+    {
+      "epoch": 0.07315208721084818,
+      "grad_norm": 2.089256525039673,
+      "learning_rate": 4.9933150637733574e-05,
+      "loss": 1.8026,
+      "step": 2201
+    },
+    {
+      "epoch": 0.07418239829832492,
+      "grad_norm": 2.0864951610565186,
+      "learning_rate": 4.992687246588743e-05,
+      "loss": 1.7753,
+      "step": 2232
+    },
+    {
+      "epoch": 0.07521270938580164,
+      "grad_norm": 2.36157488822937,
+      "learning_rate": 4.992031299767347e-05,
+      "loss": 1.7746,
+      "step": 2263
+    },
+    {
+      "epoch": 0.07624302047327838,
+      "grad_norm": 2.5334439277648926,
+      "learning_rate": 4.9913472307114386e-05,
+      "loss": 1.7927,
+      "step": 2294
+    },
+    {
+      "epoch": 0.07727333156075512,
+      "grad_norm": 2.2565715312957764,
+      "learning_rate": 4.9906350471406446e-05,
+      "loss": 1.7668,
+      "step": 2325
+    },
+    {
+      "epoch": 0.07830364264823185,
+      "grad_norm": 2.1043128967285156,
+      "learning_rate": 4.989894757091861e-05,
+      "loss": 1.7771,
+      "step": 2356
+    },
+    {
+      "epoch": 0.07933395373570859,
+      "grad_norm": 1.9659819602966309,
+      "learning_rate": 4.989126368919158e-05,
+      "loss": 1.7666,
+      "step": 2387
+    },
+    {
+      "epoch": 0.08036426482318532,
+      "grad_norm": 2.0778403282165527,
+      "learning_rate": 4.988329891293693e-05,
+      "loss": 1.7405,
+      "step": 2418
+    },
+    {
+      "epoch": 0.08139457591066206,
+      "grad_norm": 2.1767923831939697,
+      "learning_rate": 4.987505333203608e-05,
+      "loss": 1.7495,
+      "step": 2449
+    },
+    {
+      "epoch": 0.0824248869981388,
+      "grad_norm": 2.260143280029297,
+      "learning_rate": 4.9866527039539276e-05,
+      "loss": 1.7504,
+      "step": 2480
+    },
+    {
+      "epoch": 0.08345519808561552,
+      "grad_norm": 2.18271803855896,
+      "learning_rate": 4.9857720131664594e-05,
+      "loss": 1.7456,
+      "step": 2511
+    },
+    {
+      "epoch": 0.08448550917309226,
+      "grad_norm": 2.209594964981079,
+      "learning_rate": 4.9848632707796773e-05,
+      "loss": 1.7528,
+      "step": 2542
+    },
+    {
+      "epoch": 0.085515820260569,
+      "grad_norm": 2.0666229724884033,
+      "learning_rate": 4.9839264870486155e-05,
+      "loss": 1.7517,
+      "step": 2573
+    },
+    {
+      "epoch": 0.08654613134804573,
+      "grad_norm": 2.1070454120635986,
+      "learning_rate": 4.9829616725447526e-05,
+      "loss": 1.7474,
+      "step": 2604
+    },
+    {
+      "epoch": 0.08757644243552247,
+      "grad_norm": 1.9430303573608398,
+      "learning_rate": 4.981968838155888e-05,
+      "loss": 1.7348,
+      "step": 2635
+    },
+    {
+      "epoch": 0.0886067535229992,
+      "grad_norm": 1.9638925790786743,
+      "learning_rate": 4.980947995086024e-05,
+      "loss": 1.7202,
+      "step": 2666
+    },
+    {
+      "epoch": 0.08963706461047594,
+      "grad_norm": 1.8845652341842651,
+      "learning_rate": 4.979899154855234e-05,
+      "loss": 1.7375,
+      "step": 2697
+    },
+    {
+      "epoch": 0.09066737569795268,
+      "grad_norm": 5.712058067321777,
+      "learning_rate": 4.9788223292995386e-05,
+      "loss": 1.7379,
+      "step": 2728
+    },
+    {
+      "epoch": 0.0916976867854294,
+      "grad_norm": 1.9520670175552368,
+      "learning_rate": 4.977717530570768e-05,
+      "loss": 1.7302,
+      "step": 2759
+    },
+    {
+      "epoch": 0.09272799787290614,
+      "grad_norm": 1.8802224397659302,
+      "learning_rate": 4.976584771136425e-05,
+      "loss": 1.74,
+      "step": 2790
+    },
+    {
+      "epoch": 0.09375830896038288,
+      "grad_norm": 2.1098153591156006,
+      "learning_rate": 4.975424063779547e-05,
+      "loss": 1.7024,
+      "step": 2821
+    },
+    {
+      "epoch": 0.09478862004785961,
+      "grad_norm": 2.1568291187286377,
+      "learning_rate": 4.974235421598557e-05,
+      "loss": 1.7131,
+      "step": 2852
+    },
+    {
+      "epoch": 0.09581893113533635,
+      "grad_norm": 1.8769980669021606,
+      "learning_rate": 4.973018858007122e-05,
+      "loss": 1.7008,
+      "step": 2883
+    },
+    {
+      "epoch": 0.09684924222281308,
+      "grad_norm": 1.8325533866882324,
+      "learning_rate": 4.9717743867339963e-05,
+      "loss": 1.7058,
+      "step": 2914
+    },
+    {
+      "epoch": 0.09787955331028982,
+      "grad_norm": 2.086416721343994,
+      "learning_rate": 4.9705020218228695e-05,
+      "loss": 1.711,
+      "step": 2945
+    },
+    {
+      "epoch": 0.09890986439776656,
+      "grad_norm": 1.8294793367385864,
+      "learning_rate": 4.969201777632205e-05,
+      "loss": 1.6998,
+      "step": 2976
+    },
+    {
+      "epoch": 0.09994017548524328,
+      "grad_norm": 2.0608153343200684,
+      "learning_rate": 4.9678736688350846e-05,
+      "loss": 1.6948,
+      "step": 3007
+    },
+    {
+      "epoch": 0.10097048657272002,
+      "grad_norm": 3.2166008949279785,
+      "learning_rate": 4.966517710419033e-05,
+      "loss": 1.6788,
+      "step": 3038
+    },
+    {
+      "epoch": 0.10200079766019676,
+      "grad_norm": 1.9431313276290894,
+      "learning_rate": 4.965133917685858e-05,
+      "loss": 1.7115,
+      "step": 3069
+    },
+    {
+      "epoch": 0.10303110874767349,
+      "grad_norm": 1.967512845993042,
+      "learning_rate": 4.9637223062514714e-05,
+      "loss": 1.7033,
+      "step": 3100
+    },
+    {
+      "epoch": 0.10406141983515023,
+      "grad_norm": 1.9253389835357666,
+      "learning_rate": 4.962282892045718e-05,
+      "loss": 1.6856,
+      "step": 3131
+    },
+    {
+      "epoch": 0.10509173092262696,
+      "grad_norm": 1.986840009689331,
+      "learning_rate": 4.9608156913121904e-05,
+      "loss": 1.723,
+      "step": 3162
+    },
+    {
+      "epoch": 0.1061220420101037,
+      "grad_norm": 1.83523690700531,
+      "learning_rate": 4.959320720608049e-05,
+      "loss": 1.6912,
+      "step": 3193
+    },
+    {
+      "epoch": 0.10715235309758044,
+      "grad_norm": 2.1271955966949463,
+      "learning_rate": 4.9577979968038354e-05,
+      "loss": 1.7032,
+      "step": 3224
+    },
+    {
+      "epoch": 0.10818266418505716,
+      "grad_norm": 1.8383768796920776,
+      "learning_rate": 4.956247537083282e-05,
+      "loss": 1.6726,
+      "step": 3255
+    },
+    {
+      "epoch": 0.1092129752725339,
+      "grad_norm": 1.8806651830673218,
+      "learning_rate": 4.9546693589431145e-05,
+      "loss": 1.6817,
+      "step": 3286
+    },
+    {
+      "epoch": 0.11024328636001064,
+      "grad_norm": 1.7535260915756226,
+      "learning_rate": 4.9530634801928595e-05,
+      "loss": 1.6875,
+      "step": 3317
+    },
+    {
+      "epoch": 0.11127359744748737,
+      "grad_norm": 1.765906810760498,
+      "learning_rate": 4.9514299189546395e-05,
+      "loss": 1.6859,
+      "step": 3348
+    },
+    {
+      "epoch": 0.11230390853496411,
+      "grad_norm": 1.869828462600708,
+      "learning_rate": 4.949768693662973e-05,
+      "loss": 1.6915,
+      "step": 3379
+    },
+    {
+      "epoch": 0.11333421962244083,
+      "grad_norm": 1.8347504138946533,
+      "learning_rate": 4.948079823064559e-05,
+      "loss": 1.6859,
+      "step": 3410
+    },
+    {
+      "epoch": 0.11436453070991758,
+      "grad_norm": 1.7692474126815796,
+      "learning_rate": 4.946363326218074e-05,
+      "loss": 1.6565,
+      "step": 3441
+    },
+    {
+      "epoch": 0.11539484179739432,
+      "grad_norm": 1.8231885433197021,
+      "learning_rate": 4.9446192224939525e-05,
+      "loss": 1.686,
+      "step": 3472
+    },
+    {
+      "epoch": 0.11642515288487104,
+      "grad_norm": 1.7155958414077759,
+      "learning_rate": 4.942847531574167e-05,
+      "loss": 1.6538,
+      "step": 3503
+    },
+    {
+      "epoch": 0.11745546397234778,
+      "grad_norm": 1.787183403968811,
+      "learning_rate": 4.941048273452008e-05,
+      "loss": 1.6776,
+      "step": 3534
+    },
+    {
+      "epoch": 0.11848577505982451,
+      "grad_norm": 1.741213083267212,
+      "learning_rate": 4.9392214684318605e-05,
+      "loss": 1.6784,
+      "step": 3565
+    },
+    {
+      "epoch": 0.11951608614730125,
+      "grad_norm": 1.7836824655532837,
+      "learning_rate": 4.93736713712897e-05,
+      "loss": 1.6557,
+      "step": 3596
+    },
+    {
+      "epoch": 0.12054639723477799,
+      "grad_norm": 1.7103859186172485,
+      "learning_rate": 4.9354853004692124e-05,
+      "loss": 1.6606,
+      "step": 3627
+    },
+    {
+      "epoch": 0.12157670832225471,
+      "grad_norm": 1.7865506410598755,
+      "learning_rate": 4.93357597968886e-05,
+      "loss": 1.6409,
+      "step": 3658
+    },
+    {
+      "epoch": 0.12260701940973145,
+      "grad_norm": 1.7770143747329712,
+      "learning_rate": 4.931639196334338e-05,
+      "loss": 1.6574,
+      "step": 3689
+    },
+    {
+      "epoch": 0.1236373304972082,
+      "grad_norm": 1.857575535774231,
+      "learning_rate": 4.9296749722619826e-05,
+      "loss": 1.6724,
+      "step": 3720
+    },
+    {
+      "epoch": 0.12466764158468492,
+      "grad_norm": 1.8742581605911255,
+      "learning_rate": 4.9276833296377966e-05,
+      "loss": 1.6506,
+      "step": 3751
+    },
+    {
+      "epoch": 0.12569795267216166,
+      "grad_norm": 1.827668309211731,
+      "learning_rate": 4.925664290937196e-05,
+      "loss": 1.6523,
+      "step": 3782
+    },
+    {
+      "epoch": 0.1267282637596384,
+      "grad_norm": 1.7517486810684204,
+      "learning_rate": 4.9236178789447576e-05,
+      "loss": 1.6459,
+      "step": 3813
+    },
+    {
+      "epoch": 0.12775857484711514,
+      "grad_norm": 1.8109570741653442,
+      "learning_rate": 4.921544116753962e-05,
+      "loss": 1.6614,
+      "step": 3844
+    },
+    {
+      "epoch": 0.12878888593459187,
+      "grad_norm": 1.692597508430481,
+      "learning_rate": 4.919443027766935e-05,
+      "loss": 1.6431,
+      "step": 3875
+    },
+    {
+      "epoch": 0.1298191970220686,
+      "grad_norm": 1.8650025129318237,
+      "learning_rate": 4.91731463569418e-05,
+      "loss": 1.6466,
+      "step": 3906
+    },
+    {
+      "epoch": 0.13084950810954532,
+      "grad_norm": 1.6794081926345825,
+      "learning_rate": 4.915158964554312e-05,
+      "loss": 1.6504,
+      "step": 3937
+    },
+    {
+      "epoch": 0.13187981919702207,
+      "grad_norm": 1.7685374021530151,
+      "learning_rate": 4.912976038673786e-05,
+      "loss": 1.6446,
+      "step": 3968
+    },
+    {
+      "epoch": 0.1329101302844988,
+      "grad_norm": 1.7601110935211182,
+      "learning_rate": 4.9107658826866254e-05,
+      "loss": 1.631,
+      "step": 3999
+    },
+    {
+      "epoch": 0.13394044137197553,
+      "grad_norm": 2.0616064071655273,
+      "learning_rate": 4.908528521534139e-05,
+      "loss": 1.6476,
+      "step": 4030
+    },
+    {
+      "epoch": 0.13497075245945228,
+      "grad_norm": 1.8973504304885864,
+      "learning_rate": 4.906263980464644e-05,
+      "loss": 1.6582,
+      "step": 4061
+    },
+    {
+      "epoch": 0.136001063546929,
+      "grad_norm": 1.7768895626068115,
+      "learning_rate": 4.903972285033178e-05,
+      "loss": 1.6159,
+      "step": 4092
+    },
+    {
+      "epoch": 0.13703137463440573,
+      "grad_norm": 1.8264424800872803,
+      "learning_rate": 4.901653461101213e-05,
+      "loss": 1.6289,
+      "step": 4123
+    },
+    {
+      "epoch": 0.1380616857218825,
+      "grad_norm": 1.7140119075775146,
+      "learning_rate": 4.8993075348363626e-05,
+      "loss": 1.6357,
+      "step": 4154
+    },
+    {
+      "epoch": 0.13909199680935921,
+      "grad_norm": 1.6964486837387085,
+      "learning_rate": 4.896934532712084e-05,
+      "loss": 1.6233,
+      "step": 4185
+    },
+    {
+      "epoch": 0.14012230789683594,
+      "grad_norm": 1.8008025884628296,
+      "learning_rate": 4.8945344815073846e-05,
+      "loss": 1.637,
+      "step": 4216
+    },
+    {
+      "epoch": 0.1411526189843127,
+      "grad_norm": 1.562730073928833,
+      "learning_rate": 4.892107408306516e-05,
+      "loss": 1.6379,
+      "step": 4247
+    },
+    {
+      "epoch": 0.14218293007178942,
+      "grad_norm": 1.8273371458053589,
+      "learning_rate": 4.889653340498669e-05,
+      "loss": 1.6246,
+      "step": 4278
+    },
+    {
+      "epoch": 0.14321324115926615,
+      "grad_norm": 56.33716583251953,
+      "learning_rate": 4.8871723057776664e-05,
+      "loss": 1.6457,
+      "step": 4309
+    },
+    {
+      "epoch": 0.1442435522467429,
+      "grad_norm": 1.746523380279541,
+      "learning_rate": 4.8846643321416476e-05,
+      "loss": 1.6343,
+      "step": 4340
+    },
+    {
+      "epoch": 0.14527386333421963,
+      "grad_norm": 1.7737531661987305,
+      "learning_rate": 4.882129447892753e-05,
+      "loss": 1.6447,
+      "step": 4371
+    },
+    {
+      "epoch": 0.14630417442169635,
+      "grad_norm": 1.660485863685608,
+      "learning_rate": 4.8795676816368076e-05,
+      "loss": 1.6192,
+      "step": 4402
+    },
+    {
+      "epoch": 0.14733448550917308,
+      "grad_norm": 1.6823406219482422,
+      "learning_rate": 4.876979062282995e-05,
+      "loss": 1.6253,
+      "step": 4433
+    },
+    {
+      "epoch": 0.14836479659664983,
+      "grad_norm": 7.78139066696167,
+      "learning_rate": 4.8743636190435325e-05,
+      "loss": 1.6234,
+      "step": 4464
+    },
+    {
+      "epoch": 0.14939510768412656,
+      "grad_norm": 1.7426058053970337,
+      "learning_rate": 4.871721381433344e-05,
+      "loss": 1.6337,
+      "step": 4495
+    },
+    {
+      "epoch": 0.1504254187716033,
+      "grad_norm": 1.6294783353805542,
+      "learning_rate": 4.869052379269719e-05,
+      "loss": 1.6217,
+      "step": 4526
+    },
+    {
+      "epoch": 0.15145572985908004,
+      "grad_norm": 1.6523306369781494,
+      "learning_rate": 4.866356642671985e-05,
+      "loss": 1.605,
+      "step": 4557
+    },
+    {
+      "epoch": 0.15248604094655677,
+      "grad_norm": 1.8571300506591797,
+      "learning_rate": 4.8636342020611634e-05,
+      "loss": 1.6218,
+      "step": 4588
+    },
+    {
+      "epoch": 0.1535163520340335,
+      "grad_norm": 1.7754936218261719,
+      "learning_rate": 4.860885088159626e-05,
+      "loss": 1.6171,
+      "step": 4619
+    },
+    {
+      "epoch": 0.15454666312151025,
+      "grad_norm": 1.91987943649292,
+      "learning_rate": 4.858109331990751e-05,
+      "loss": 1.6167,
+      "step": 4650
+    },
+    {
+      "epoch": 0.15557697420898697,
+      "grad_norm": 1.5994452238082886,
+      "learning_rate": 4.855306964878567e-05,
+      "loss": 1.5951,
+      "step": 4681
+    },
+    {
+      "epoch": 0.1566072852964637,
+      "grad_norm": 1.6490916013717651,
+      "learning_rate": 4.8524780184474084e-05,
+      "loss": 1.616,
+      "step": 4712
+    },
+    {
+      "epoch": 0.15763759638394045,
+      "grad_norm": 1.5921640396118164,
+      "learning_rate": 4.8496225246215496e-05,
+      "loss": 1.6346,
+      "step": 4743
+    },
+    {
+      "epoch": 0.15866790747141718,
+      "grad_norm": 1.6729261875152588,
+      "learning_rate": 4.8467405156248505e-05,
+      "loss": 1.6165,
+      "step": 4774
+    },
+    {
+      "epoch": 0.1596982185588939,
+      "grad_norm": 1.628113031387329,
+      "learning_rate": 4.843832023980392e-05,
+      "loss": 1.6119,
+      "step": 4805
+    },
+    {
+      "epoch": 0.16072852964637063,
+      "grad_norm": 1.651647925376892,
+      "learning_rate": 4.840897082510106e-05,
+      "loss": 1.5997,
+      "step": 4836
+    },
+    {
+      "epoch": 0.1617588407338474,
+      "grad_norm": 1.5297720432281494,
+      "learning_rate": 4.8379357243344084e-05,
+      "loss": 1.6242,
+      "step": 4867
+    },
+    {
+      "epoch": 0.1627891518213241,
+      "grad_norm": 1.5779869556427002,
+      "learning_rate": 4.8349479828718236e-05,
+      "loss": 1.6149,
+      "step": 4898
+    },
+    {
+      "epoch": 0.16381946290880084,
+      "grad_norm": 1.5843939781188965,
+      "learning_rate": 4.8319338918386075e-05,
+      "loss": 1.5926,
+      "step": 4929
+    },
+    {
+      "epoch": 0.1648497739962776,
+      "grad_norm": 2.3762106895446777,
+      "learning_rate": 4.828893485248369e-05,
+      "loss": 1.6108,
+      "step": 4960
+    },
+    {
+      "epoch": 0.16588008508375432,
+      "grad_norm": 1.5871953964233398,
+      "learning_rate": 4.825826797411682e-05,
+      "loss": 1.6103,
+      "step": 4991
+    },
+    {
+      "epoch": 0.16691039617123105,
+      "grad_norm": 1.5934125185012817,
+      "learning_rate": 4.822733862935702e-05,
+      "loss": 1.6091,
+      "step": 5022
+    },
+    {
+      "epoch": 0.1679407072587078,
+      "grad_norm": 1.6997628211975098,
+      "learning_rate": 4.819614716723775e-05,
+      "loss": 1.6098,
+      "step": 5053
+    },
+    {
+      "epoch": 0.16897101834618453,
+      "grad_norm": 1.682849645614624,
+      "learning_rate": 4.8164693939750425e-05,
+      "loss": 1.599,
+      "step": 5084
+    },
+    {
+      "epoch": 0.17000132943366125,
+      "grad_norm": 1.709743857383728,
+      "learning_rate": 4.813297930184042e-05,
+      "loss": 1.6194,
+      "step": 5115
+    },
+    {
+      "epoch": 0.171031640521138,
+      "grad_norm": 1.725879430770874,
+      "learning_rate": 4.810100361140314e-05,
+      "loss": 1.6115,
+      "step": 5146
+    },
+    {
+      "epoch": 0.17206195160861473,
+      "grad_norm": 1.6710290908813477,
+      "learning_rate": 4.8068767229279885e-05,
+      "loss": 1.6032,
+      "step": 5177
+    },
+    {
+      "epoch": 0.17309226269609146,
+      "grad_norm": 1.6156634092330933,
+      "learning_rate": 4.8036270519253854e-05,
+      "loss": 1.5973,
+      "step": 5208
+    },
+    {
+      "epoch": 0.1741225737835682,
+      "grad_norm": 1.5654059648513794,
+      "learning_rate": 4.8003513848046e-05,
+      "loss": 1.5817,
+      "step": 5239
+    },
+    {
+      "epoch": 0.17515288487104494,
+      "grad_norm": 1.5789822340011597,
+      "learning_rate": 4.79704975853109e-05,
+      "loss": 1.6138,
+      "step": 5270
+    },
+    {
+      "epoch": 0.17618319595852167,
+      "grad_norm": 1.6022037267684937,
+      "learning_rate": 4.793722210363262e-05,
+      "loss": 1.5998,
+      "step": 5301
+    },
+    {
+      "epoch": 0.1772135070459984,
+      "grad_norm": 1.5142741203308105,
+      "learning_rate": 4.7903687778520414e-05,
+      "loss": 1.6061,
+      "step": 5332
+    },
+    {
+      "epoch": 0.17824381813347515,
+      "grad_norm": 1.6454212665557861,
+      "learning_rate": 4.7869894988404593e-05,
+      "loss": 1.6063,
+      "step": 5363
+    },
+    {
+      "epoch": 0.17927412922095187,
+      "grad_norm": 1.5250823497772217,
+      "learning_rate": 4.783584411463221e-05,
+      "loss": 1.6038,
+      "step": 5394
+    },
+    {
+      "epoch": 0.1803044403084286,
+      "grad_norm": 1.5829335451126099,
+      "learning_rate": 4.780153554146274e-05,
+      "loss": 1.5949,
+      "step": 5425
+    },
+    {
+      "epoch": 0.18133475139590535,
+      "grad_norm": 1.5342432260513306,
+      "learning_rate": 4.7766969656063766e-05,
+      "loss": 1.5913,
+      "step": 5456
+    },
+    {
+      "epoch": 0.18236506248338208,
+      "grad_norm": 1.6397250890731812,
+      "learning_rate": 4.773214684850662e-05,
+      "loss": 1.6102,
+      "step": 5487
+    },
+    {
+      "epoch": 0.1833953735708588,
+      "grad_norm": 1.5228471755981445,
+      "learning_rate": 4.769706751176193e-05,
+      "loss": 1.5885,
+      "step": 5518
+    },
+    {
+      "epoch": 0.18442568465833556,
+      "grad_norm": 1.6186103820800781,
+      "learning_rate": 4.7661732041695264e-05,
+      "loss": 1.6086,
+      "step": 5549
+    },
+    {
+      "epoch": 0.18545599574581229,
+      "grad_norm": 1.6024582386016846,
+      "learning_rate": 4.762614083706258e-05,
+      "loss": 1.6004,
+      "step": 5580
+    },
+    {
+      "epoch": 0.186486306833289,
+      "grad_norm": 1.5443711280822754,
+      "learning_rate": 4.759029429950581e-05,
+      "loss": 1.6048,
+      "step": 5611
+    },
+    {
+      "epoch": 0.18751661792076577,
+      "grad_norm": 1.4831629991531372,
+      "learning_rate": 4.7554192833548235e-05,
+      "loss": 1.5841,
+      "step": 5642
+    },
+    {
+      "epoch": 0.1885469290082425,
+      "grad_norm": 1.6426068544387817,
+      "learning_rate": 4.751783684659e-05,
+      "loss": 1.587,
+      "step": 5673
+    },
+    {
+      "epoch": 0.18957724009571922,
+      "grad_norm": 1.4609078168869019,
+      "learning_rate": 4.748122674890348e-05,
+      "loss": 1.5945,
+      "step": 5704
+    },
+    {
+      "epoch": 0.19060755118319597,
+      "grad_norm": 1.5365614891052246,
+      "learning_rate": 4.7444362953628654e-05,
+      "loss": 1.5737,
+      "step": 5735
+    },
+    {
+      "epoch": 0.1916378622706727,
+      "grad_norm": 1.5755670070648193,
+      "learning_rate": 4.7407245876768424e-05,
+      "loss": 1.5862,
+      "step": 5766
+    },
+    {
+      "epoch": 0.19266817335814942,
+      "grad_norm": 1.6469846963882446,
+      "learning_rate": 4.736987593718397e-05,
+      "loss": 1.5663,
+      "step": 5797
+    },
+    {
+      "epoch": 0.19369848444562615,
+      "grad_norm": 1.5927278995513916,
+      "learning_rate": 4.733225355658999e-05,
+      "loss": 1.5776,
+      "step": 5828
+    },
+    {
+      "epoch": 0.1947287955331029,
+      "grad_norm": 1.5593287944793701,
+      "learning_rate": 4.7294379159549926e-05,
+      "loss": 1.579,
+      "step": 5859
+    },
+    {
+      "epoch": 0.19575910662057963,
+      "grad_norm": 1.534055233001709,
+      "learning_rate": 4.725625317347119e-05,
+      "loss": 1.6017,
+      "step": 5890
+    },
+    {
+      "epoch": 0.19678941770805636,
+      "grad_norm": 1.5846387147903442,
+      "learning_rate": 4.7217876028600374e-05,
+      "loss": 1.5739,
+      "step": 5921
+    },
+    {
+      "epoch": 0.1978197287955331,
+      "grad_norm": 1.5377682447433472,
+      "learning_rate": 4.717924815801832e-05,
+      "loss": 1.57,
+      "step": 5952
+    },
+    {
+      "epoch": 0.19885003988300984,
+      "grad_norm": 1.467956781387329,
+      "learning_rate": 4.714036999763532e-05,
+      "loss": 1.5736,
+      "step": 5983
+    },
+    {
+      "epoch": 0.19988035097048656,
+      "grad_norm": 1.601070523262024,
+      "learning_rate": 4.7101241986186116e-05,
+      "loss": 1.5861,
+      "step": 6014
+    },
+    {
+      "epoch": 0.20091066205796332,
+      "grad_norm": 1.5051921606063843,
+      "learning_rate": 4.7061864565225e-05,
+      "loss": 1.5735,
+      "step": 6045
+    },
+    {
+      "epoch": 0.20194097314544004,
+      "grad_norm": 1.462843418121338,
+      "learning_rate": 4.702223817912081e-05,
+      "loss": 1.582,
+      "step": 6076
+    },
+    {
+      "epoch": 0.20297128423291677,
+      "grad_norm": 1.5698682069778442,
+      "learning_rate": 4.698236327505195e-05,
+      "loss": 1.5647,
+      "step": 6107
+    },
+    {
+      "epoch": 0.20400159532039353,
+      "grad_norm": 1.5633916854858398,
+      "learning_rate": 4.694224030300127e-05,
+      "loss": 1.5741,
+      "step": 6138
+    },
+    {
+      "epoch": 0.20503190640787025,
+      "grad_norm": 1.6174733638763428,
+      "learning_rate": 4.690186971575107e-05,
+      "loss": 1.5634,
+      "step": 6169
+    },
+    {
+      "epoch": 0.20606221749534698,
+      "grad_norm": 1.4957518577575684,
+      "learning_rate": 4.6861251968877916e-05,
+      "loss": 1.575,
+      "step": 6200
+    },
+    {
+      "epoch": 0.2070925285828237,
+      "grad_norm": 1.670933485031128,
+      "learning_rate": 4.68203875207476e-05,
+      "loss": 1.5792,
+      "step": 6231
+    },
+    {
+      "epoch": 0.20812283967030046,
+      "grad_norm": 1.5676430463790894,
+      "learning_rate": 4.677927683250983e-05,
+      "loss": 1.5689,
+      "step": 6262
+    },
+    {
+      "epoch": 0.20915315075777718,
+      "grad_norm": 1.5753976106643677,
+      "learning_rate": 4.6737920368093156e-05,
+      "loss": 1.5594,
+      "step": 6293
+    },
+    {
+      "epoch": 0.2101834618452539,
+      "grad_norm": 1.4973617792129517,
+      "learning_rate": 4.669631859419965e-05,
+      "loss": 1.5593,
+      "step": 6324
+    },
+    {
+      "epoch": 0.21121377293273066,
+      "grad_norm": 1.4691433906555176,
+      "learning_rate": 4.6654471980299676e-05,
+      "loss": 1.5711,
+      "step": 6355
+    },
+    {
+      "epoch": 0.2122440840202074,
+      "grad_norm": 1.407630443572998,
+      "learning_rate": 4.661238099862658e-05,
+      "loss": 1.5787,
+      "step": 6386
+    },
+    {
+      "epoch": 0.21327439510768412,
+      "grad_norm": 1.5011677742004395,
+      "learning_rate": 4.657004612417138e-05,
+      "loss": 1.5751,
+      "step": 6417
+    },
+    {
+      "epoch": 0.21430470619516087,
+      "grad_norm": 1.509750485420227,
+      "learning_rate": 4.6527467834677374e-05,
+      "loss": 1.5583,
+      "step": 6448
+    },
+    {
+      "epoch": 0.2153350172826376,
+      "grad_norm": 1.3919882774353027,
+      "learning_rate": 4.648464661063478e-05,
+      "loss": 1.5712,
+      "step": 6479
+    },
+    {
+      "epoch": 0.21636532837011432,
+      "grad_norm": 1.4854936599731445,
+      "learning_rate": 4.6441582935275264e-05,
+      "loss": 1.5637,
+      "step": 6510
+    },
+    {
+      "epoch": 0.21739563945759108,
+      "grad_norm": 1.4413583278656006,
+      "learning_rate": 4.6398277294566586e-05,
+      "loss": 1.56,
+      "step": 6541
+    },
+    {
+      "epoch": 0.2184259505450678,
+      "grad_norm": 1.5063883066177368,
+      "learning_rate": 4.6354730177207e-05,
+      "loss": 1.5525,
+      "step": 6572
+    },
+    {
+      "epoch": 0.21945626163254453,
+      "grad_norm": 1.4899688959121704,
+      "learning_rate": 4.6310942074619787e-05,
+      "loss": 1.5817,
+      "step": 6603
+    },
+    {
+      "epoch": 0.22048657272002128,
+      "grad_norm": 1.3927967548370361,
+      "learning_rate": 4.626691348094777e-05,
+      "loss": 1.5407,
+      "step": 6634
+    },
+    {
+      "epoch": 0.221516883807498,
+      "grad_norm": 1.5378398895263672,
+      "learning_rate": 4.622264489304762e-05,
+      "loss": 1.5561,
+      "step": 6665
+    },
+    {
+      "epoch": 0.22254719489497474,
+      "grad_norm": 1.554624319076538,
+      "learning_rate": 4.617813681048434e-05,
+      "loss": 1.5859,
+      "step": 6696
+    },
+    {
+      "epoch": 0.22357750598245146,
+      "grad_norm": 1.5356658697128296,
+      "learning_rate": 4.61333897355256e-05,
+      "loss": 1.5531,
+      "step": 6727
+    },
+    {
+      "epoch": 0.22460781706992822,
+      "grad_norm": 1.5534918308258057,
+      "learning_rate": 4.608840417313604e-05,
+      "loss": 1.5774,
+      "step": 6758
+    },
+    {
+      "epoch": 0.22563812815740494,
+      "grad_norm": 1.5660988092422485,
+      "learning_rate": 4.6043180630971646e-05,
+      "loss": 1.5763,
+      "step": 6789
+    },
+    {
+      "epoch": 0.22666843924488167,
+      "grad_norm": 1.4993386268615723,
+      "learning_rate": 4.599771961937391e-05,
+      "loss": 1.5615,
+      "step": 6820
+    },
+    {
+      "epoch": 0.22769875033235842,
+      "grad_norm": 1.4630553722381592,
+      "learning_rate": 4.5952021651364204e-05,
+      "loss": 1.543,
+      "step": 6851
+    },
+    {
+      "epoch": 0.22872906141983515,
+      "grad_norm": 1.470173954963684,
+      "learning_rate": 4.590608724263786e-05,
+      "loss": 1.5674,
+      "step": 6882
+    },
+    {
+      "epoch": 0.22975937250731188,
+      "grad_norm": 1.5867971181869507,
+      "learning_rate": 4.585991691155845e-05,
+      "loss": 1.5702,
+      "step": 6913
+    },
+    {
+      "epoch": 0.23078968359478863,
+      "grad_norm": 1.44207763671875,
+      "learning_rate": 4.581351117915188e-05,
+      "loss": 1.5436,
+      "step": 6944
+    },
+    {
+      "epoch": 0.23181999468226536,
+      "grad_norm": 1.4691039323806763,
+      "learning_rate": 4.5766870569100534e-05,
+      "loss": 1.5465,
+      "step": 6975
+    },
+    {
+      "epoch": 0.23285030576974208,
+      "grad_norm": 1.4807918071746826,
+      "learning_rate": 4.571999560773736e-05,
+      "loss": 1.5564,
+      "step": 7006
+    },
+    {
+      "epoch": 0.23388061685721884,
+      "grad_norm": 1.481487512588501,
+      "learning_rate": 4.5672886824039915e-05,
+      "loss": 1.5466,
+      "step": 7037
+    },
+    {
+      "epoch": 0.23491092794469556,
+      "grad_norm": 1.4518013000488281,
+      "learning_rate": 4.5625544749624435e-05,
+      "loss": 1.5618,
+      "step": 7068
+    },
+    {
+      "epoch": 0.2359412390321723,
+      "grad_norm": 1.4186676740646362,
+      "learning_rate": 4.5577969918739794e-05,
+      "loss": 1.5528,
+      "step": 7099
+    },
+    {
+      "epoch": 0.23697155011964902,
+      "grad_norm": 1.5287110805511475,
+      "learning_rate": 4.5530162868261486e-05,
+      "loss": 1.5457,
+      "step": 7130
+    },
+    {
+      "epoch": 0.23800186120712577,
+      "grad_norm": 1.5516417026519775,
+      "learning_rate": 4.548212413768558e-05,
+      "loss": 1.5467,
+      "step": 7161
+    },
+    {
+      "epoch": 0.2390321722946025,
+      "grad_norm": 1.4710053205490112,
+      "learning_rate": 4.543385426912261e-05,
+      "loss": 1.5431,
+      "step": 7192
+    },
+    {
+      "epoch": 0.24006248338207922,
+      "grad_norm": 1.5005567073822021,
+      "learning_rate": 4.53853538072915e-05,
+      "loss": 1.5592,
+      "step": 7223
+    },
+    {
+      "epoch": 0.24109279446955598,
+      "grad_norm": 1.5864965915679932,
+      "learning_rate": 4.533662329951336e-05,
+      "loss": 1.5694,
+      "step": 7254
+    },
+    {
+      "epoch": 0.2421231055570327,
+      "grad_norm": 1.4661896228790283,
+      "learning_rate": 4.528766329570536e-05,
+      "loss": 1.545,
+      "step": 7285
+    },
+    {
+      "epoch": 0.24315341664450943,
+      "grad_norm": 1.5157560110092163,
+      "learning_rate": 4.523847434837447e-05,
+      "loss": 1.5458,
+      "step": 7316
+    },
+    {
+      "epoch": 0.24418372773198618,
+      "grad_norm": 1.4033585786819458,
+      "learning_rate": 4.518905701261128e-05,
+      "loss": 1.5464,
+      "step": 7347
+    },
+    {
+      "epoch": 0.2452140388194629,
+      "grad_norm": 1.5357593297958374,
+      "learning_rate": 4.5139411846083715e-05,
+      "loss": 1.5497,
+      "step": 7378
+    },
+    {
+      "epoch": 0.24624434990693964,
+      "grad_norm": 1.419507384300232,
+      "learning_rate": 4.508953940903073e-05,
+      "loss": 1.5414,
+      "step": 7409
+    },
+    {
+      "epoch": 0.2472746609944164,
+      "grad_norm": 1.5201773643493652,
+      "learning_rate": 4.5039440264255994e-05,
+      "loss": 1.5503,
+      "step": 7440
+    },
+    {
+      "epoch": 0.24830497208189312,
+      "grad_norm": 1.8000444173812866,
+      "learning_rate": 4.498911497712155e-05,
+      "loss": 1.5448,
+      "step": 7471
+    },
+    {
+      "epoch": 0.24933528316936984,
+      "grad_norm": 1.4876810312271118,
+      "learning_rate": 4.493856411554142e-05,
+      "loss": 1.5524,
+      "step": 7502
+    },
+    {
+      "epoch": 0.25036559425684657,
+      "grad_norm": 1.5130078792572021,
+      "learning_rate": 4.4887788249975206e-05,
+      "loss": 1.5454,
+      "step": 7533
+    },
+    {
+      "epoch": 0.2513959053443233,
+      "grad_norm": 1.4829351902008057,
+      "learning_rate": 4.4836787953421656e-05,
+      "loss": 1.5407,
+      "step": 7564
+    },
+    {
+      "epoch": 0.2524262164318001,
+      "grad_norm": 1.521550178527832,
+      "learning_rate": 4.478556380141218e-05,
+      "loss": 1.5727,
+      "step": 7595
+    },
+    {
+      "epoch": 0.2534565275192768,
+      "grad_norm": 1.4377928972244263,
+      "learning_rate": 4.4734116372004375e-05,
+      "loss": 1.5432,
+      "step": 7626
+    },
+    {
+      "epoch": 0.25448683860675353,
+      "grad_norm": 1.4101744890213013,
+      "learning_rate": 4.4682446245775477e-05,
+      "loss": 1.547,
+      "step": 7657
+    },
+    {
+      "epoch": 0.2555171496942303,
+      "grad_norm": 1.522524356842041,
+      "learning_rate": 4.463055400581586e-05,
+      "loss": 1.5418,
+      "step": 7688
+    },
+    {
+      "epoch": 0.256547460781707,
+      "grad_norm": 1.4160797595977783,
+      "learning_rate": 4.4578440237722374e-05,
+      "loss": 1.5457,
+      "step": 7719
+    },
+    {
+      "epoch": 0.25757777186918374,
+      "grad_norm": 1.4106636047363281,
+      "learning_rate": 4.452610552959183e-05,
+      "loss": 1.5405,
+      "step": 7750
+    },
+    {
+      "epoch": 0.2586080829566605,
+      "grad_norm": 1.422723650932312,
+      "learning_rate": 4.447355047201428e-05,
+      "loss": 1.5423,
+      "step": 7781
+    },
+    {
+      "epoch": 0.2596383940441372,
+      "grad_norm": 1.4362592697143555,
+      "learning_rate": 4.4420775658066414e-05,
+      "loss": 1.5372,
+      "step": 7812
+    },
+    {
+      "epoch": 0.26066870513161394,
+      "grad_norm": 1.4319696426391602,
+      "learning_rate": 4.436778168330484e-05,
+      "loss": 1.5451,
+      "step": 7843
+    },
+    {
+      "epoch": 0.26169901621909064,
+      "grad_norm": 1.4069257974624634,
+      "learning_rate": 4.4314569145759353e-05,
+      "loss": 1.5221,
+      "step": 7874
+    },
+    {
+      "epoch": 0.2627293273065674,
+      "grad_norm": 1.4424949884414673,
+      "learning_rate": 4.42611386459262e-05,
+      "loss": 1.5419,
+      "step": 7905
+    },
+    {
+      "epoch": 0.26375963839404415,
+      "grad_norm": 1.4579105377197266,
+      "learning_rate": 4.420749078676133e-05,
+      "loss": 1.5116,
+      "step": 7936
+    },
+    {
+      "epoch": 0.26478994948152085,
+      "grad_norm": 1.4563167095184326,
+      "learning_rate": 4.4153626173673516e-05,
+      "loss": 1.5296,
+      "step": 7967
+    },
+    {
+      "epoch": 0.2658202605689976,
+      "grad_norm": 1.4440968036651611,
+      "learning_rate": 4.409954541451762e-05,
+      "loss": 1.5548,
+      "step": 7998
+    },
+    {
+      "epoch": 0.26685057165647436,
+      "grad_norm": 1.5711034536361694,
+      "learning_rate": 4.404524911958764e-05,
+      "loss": 1.535,
+      "step": 8029
+    },
+    {
+      "epoch": 0.26788088274395105,
+      "grad_norm": 1.5221564769744873,
+      "learning_rate": 4.399073790160989e-05,
+      "loss": 1.5495,
+      "step": 8060
+    },
+    {
+      "epoch": 0.2689111938314278,
+      "grad_norm": 1.392699956893921,
+      "learning_rate": 4.393601237573607e-05,
+      "loss": 1.546,
+      "step": 8091
+    },
+    {
+      "epoch": 0.26994150491890456,
+      "grad_norm": 1.5343137979507446,
+      "learning_rate": 4.388107315953628e-05,
+      "loss": 1.549,
+      "step": 8122
+    },
+    {
+      "epoch": 0.27097181600638126,
+      "grad_norm": 1.4483468532562256,
+      "learning_rate": 4.382592087299212e-05,
+      "loss": 1.5424,
+      "step": 8153
+    },
+    {
+      "epoch": 0.272002127093858,
+      "grad_norm": 1.4963489770889282,
+      "learning_rate": 4.377055613848964e-05,
+      "loss": 1.508,
+      "step": 8184
+    },
+    {
+      "epoch": 0.27303243818133477,
+      "grad_norm": 1.4839162826538086,
+      "learning_rate": 4.3714979580812355e-05,
+      "loss": 1.5203,
+      "step": 8215
+    },
+    {
+      "epoch": 0.27406274926881147,
+      "grad_norm": 1.4272018671035767,
+      "learning_rate": 4.365919182713416e-05,
+      "loss": 1.5264,
+      "step": 8246
+    },
+    {
+      "epoch": 0.2750930603562882,
+      "grad_norm": 1.3808270692825317,
+      "learning_rate": 4.360319350701226e-05,
+      "loss": 1.5255,
+      "step": 8277
+    },
+    {
+      "epoch": 0.276123371443765,
+      "grad_norm": 1.4179162979125977,
+      "learning_rate": 4.3546985252380115e-05,
+      "loss": 1.535,
+      "step": 8308
+    },
+    {
+      "epoch": 0.2771536825312417,
+      "grad_norm": 1.3617374897003174,
+      "learning_rate": 4.349056769754021e-05,
+      "loss": 1.5295,
+      "step": 8339
+    },
+    {
+      "epoch": 0.27818399361871843,
+      "grad_norm": 1.4745615720748901,
+      "learning_rate": 4.3433941479156994e-05,
+      "loss": 1.5438,
+      "step": 8370
+    },
+    {
+      "epoch": 0.2792143047061952,
+      "grad_norm": 1.3661375045776367,
+      "learning_rate": 4.3377107236249647e-05,
+      "loss": 1.5134,
+      "step": 8401
+    },
+    {
+      "epoch": 0.2802446157936719,
+      "grad_norm": 1.3907949924468994,
+      "learning_rate": 4.332006561018488e-05,
+      "loss": 1.5237,
+      "step": 8432
+    },
+    {
+      "epoch": 0.28127492688114863,
+      "grad_norm": 1.3575704097747803,
+      "learning_rate": 4.3262817244669683e-05,
+      "loss": 1.5226,
+      "step": 8463
+    },
+    {
+      "epoch": 0.2823052379686254,
+      "grad_norm": 1.3836462497711182,
+      "learning_rate": 4.3205362785744083e-05,
+      "loss": 1.5433,
+      "step": 8494
+    },
+    {
+      "epoch": 0.2833355490561021,
+      "grad_norm": 1.6108276844024658,
+      "learning_rate": 4.314770288177384e-05,
+      "loss": 1.5324,
+      "step": 8525
+    },
+    {
+      "epoch": 0.28436586014357884,
+      "grad_norm": 1.4650689363479614,
+      "learning_rate": 4.308983818344313e-05,
+      "loss": 1.535,
+      "step": 8556
+    },
+    {
+      "epoch": 0.2853961712310556,
+      "grad_norm": 1.5836583375930786,
+      "learning_rate": 4.3031769343747206e-05,
+      "loss": 1.5313,
+      "step": 8587
+    },
+    {
+      "epoch": 0.2864264823185323,
+      "grad_norm": 1.5348492860794067,
+      "learning_rate": 4.297349701798505e-05,
+      "loss": 1.5106,
+      "step": 8618
+    },
+    {
+      "epoch": 0.28745679340600905,
+      "grad_norm": 1.4060319662094116,
+      "learning_rate": 4.2915021863751916e-05,
+      "loss": 1.5283,
+      "step": 8649
+    },
+    {
+      "epoch": 0.2884871044934858,
+      "grad_norm": 1.531657099723816,
+      "learning_rate": 4.285634454093198e-05,
+      "loss": 1.5087,
+      "step": 8680
+    },
+    {
+      "epoch": 0.2895174155809625,
+      "grad_norm": 1.4756299257278442,
+      "learning_rate": 4.279746571169086e-05,
+      "loss": 1.5042,
+      "step": 8711
+    },
+    {
+      "epoch": 0.29054772666843925,
+      "grad_norm": 1.3221153020858765,
+      "learning_rate": 4.2738386040468136e-05,
+      "loss": 1.5244,
+      "step": 8742
+    },
+    {
+      "epoch": 0.29157803775591595,
+      "grad_norm": 1.4067268371582031,
+      "learning_rate": 4.2679106193969866e-05,
+      "loss": 1.5012,
+      "step": 8773
+    },
+    {
+      "epoch": 0.2926083488433927,
+      "grad_norm": 1.5192064046859741,
+      "learning_rate": 4.261962684116106e-05,
+      "loss": 1.521,
+      "step": 8804
+    },
+    {
+      "epoch": 0.29363865993086946,
+      "grad_norm": 1.3847788572311401,
+      "learning_rate": 4.2559948653258145e-05,
+      "loss": 1.5128,
+      "step": 8835
+    },
+    {
+      "epoch": 0.29466897101834616,
+      "grad_norm": 1.4612780809402466,
+      "learning_rate": 4.250007230372134e-05,
+      "loss": 1.5371,
+      "step": 8866
+    },
+    {
+      "epoch": 0.2956992821058229,
+      "grad_norm": 1.468971610069275,
+      "learning_rate": 4.2439998468247126e-05,
+      "loss": 1.5199,
+      "step": 8897
+    },
+    {
+      "epoch": 0.29672959319329967,
+      "grad_norm": 1.386236310005188,
+      "learning_rate": 4.2379727824760566e-05,
+      "loss": 1.5273,
+      "step": 8928
+    },
+    {
+      "epoch": 0.29775990428077637,
+      "grad_norm": 1.3843929767608643,
+      "learning_rate": 4.231926105340768e-05,
+      "loss": 1.5011,
+      "step": 8959
+    },
+    {
+      "epoch": 0.2987902153682531,
+      "grad_norm": 1.4554557800292969,
+      "learning_rate": 4.225859883654776e-05,
+      "loss": 1.5311,
+      "step": 8990
+    },
+    {
+      "epoch": 0.2998205264557299,
+      "grad_norm": 1.3674421310424805,
+      "learning_rate": 4.219774185874569e-05,
+      "loss": 1.5302,
+      "step": 9021
+    },
+    {
+      "epoch": 0.3008508375432066,
+      "grad_norm": 1.3804330825805664,
+      "learning_rate": 4.213669080676418e-05,
+      "loss": 1.538,
+      "step": 9052
+    },
+    {
+      "epoch": 0.3018811486306833,
+      "grad_norm": 1.4643255472183228,
+      "learning_rate": 4.2075446369556056e-05,
+      "loss": 1.5172,
+      "step": 9083
+    },
+    {
+      "epoch": 0.3029114597181601,
+      "grad_norm": 1.3375928401947021,
+      "learning_rate": 4.201400923825648e-05,
+      "loss": 1.5123,
+      "step": 9114
+    },
+    {
+      "epoch": 0.3039417708056368,
+      "grad_norm": 1.4321980476379395,
+      "learning_rate": 4.195238010617511e-05,
+      "loss": 1.5196,
+      "step": 9145
+    },
+    {
+      "epoch": 0.30497208189311353,
+      "grad_norm": 1.4312376976013184,
+      "learning_rate": 4.1890559668788344e-05,
+      "loss": 1.5138,
+      "step": 9176
+    },
+    {
+      "epoch": 0.3060023929805903,
+      "grad_norm": 1.3089646100997925,
+      "learning_rate": 4.1828548623731405e-05,
+      "loss": 1.5027,
+      "step": 9207
+    },
+    {
+      "epoch": 0.307032704068067,
+      "grad_norm": 1.4863250255584717,
+      "learning_rate": 4.1766347670790506e-05,
+      "loss": 1.5091,
+      "step": 9238
+    },
+    {
+      "epoch": 0.30806301515554374,
+      "grad_norm": 1.373666763305664,
+      "learning_rate": 4.170395751189495e-05,
+      "loss": 1.5256,
+      "step": 9269
+    },
+    {
+      "epoch": 0.3090933262430205,
+      "grad_norm": 1.4160584211349487,
+      "learning_rate": 4.164137885110921e-05,
+      "loss": 1.4938,
+      "step": 9300
+    },
+    {
+      "epoch": 0.3101236373304972,
+      "grad_norm": 2.112110137939453,
+      "learning_rate": 4.157861239462495e-05,
+      "loss": 1.5106,
+      "step": 9331
+    },
+    {
+      "epoch": 0.31115394841797395,
+      "grad_norm": 1.337058663368225,
+      "learning_rate": 4.1515658850753114e-05,
+      "loss": 1.4999,
+      "step": 9362
+    },
+    {
+      "epoch": 0.3121842595054507,
+      "grad_norm": 1.3625296354293823,
+      "learning_rate": 4.145251892991588e-05,
+      "loss": 1.5136,
+      "step": 9393
+    },
+    {
+      "epoch": 0.3132145705929274,
+      "grad_norm": 1.399491548538208,
+      "learning_rate": 4.138919334463868e-05,
+      "loss": 1.499,
+      "step": 9424
+    },
+    {
+      "epoch": 0.31424488168040415,
+      "grad_norm": 1.4202344417572021,
+      "learning_rate": 4.1325682809542124e-05,
+      "loss": 1.5049,
+      "step": 9455
+    },
+    {
+      "epoch": 0.3152751927678809,
+      "grad_norm": 1.392248272895813,
+      "learning_rate": 4.126198804133398e-05,
+      "loss": 1.5287,
+      "step": 9486
+    },
+    {
+      "epoch": 0.3163055038553576,
+      "grad_norm": 1.3807618618011475,
+      "learning_rate": 4.1198109758801055e-05,
+      "loss": 1.5309,
+      "step": 9517
+    },
+    {
+      "epoch": 0.31733581494283436,
+      "grad_norm": 1.3117905855178833,
+      "learning_rate": 4.113404868280107e-05,
+      "loss": 1.4933,
+      "step": 9548
+    },
+    {
+      "epoch": 0.3183661260303111,
+      "grad_norm": 1.452086091041565,
+      "learning_rate": 4.106980553625457e-05,
+      "loss": 1.5221,
+      "step": 9579
+    },
+    {
+      "epoch": 0.3193964371177878,
+      "grad_norm": 1.477364182472229,
+      "learning_rate": 4.100538104413674e-05,
+      "loss": 1.4904,
+      "step": 9610
+    },
+    {
+      "epoch": 0.32042674820526457,
+      "grad_norm": 1.3584345579147339,
+      "learning_rate": 4.09407759334692e-05,
+      "loss": 1.4953,
+      "step": 9641
+    },
+    {
+      "epoch": 0.32145705929274127,
+      "grad_norm": 1.3619811534881592,
+      "learning_rate": 4.087599093331186e-05,
+      "loss": 1.4956,
+      "step": 9672
+    },
+    {
+      "epoch": 0.322487370380218,
+      "grad_norm": 1.4507052898406982,
+      "learning_rate": 4.081102677475462e-05,
+      "loss": 1.5197,
+      "step": 9703
+    },
+    {
+      "epoch": 0.3235176814676948,
+      "grad_norm": 1.4229698181152344,
+      "learning_rate": 4.0745884190909194e-05,
+      "loss": 1.498,
+      "step": 9734
+    },
+    {
+      "epoch": 0.32454799255517147,
+      "grad_norm": 1.3074679374694824,
+      "learning_rate": 4.0680563916900796e-05,
+      "loss": 1.5146,
+      "step": 9765
+    },
+    {
+      "epoch": 0.3255783036426482,
+      "grad_norm": 1.397815465927124,
+      "learning_rate": 4.0615066689859815e-05,
+      "loss": 1.5291,
+      "step": 9796
+    },
+    {
+      "epoch": 0.326608614730125,
+      "grad_norm": 1.3196336030960083,
+      "learning_rate": 4.0549393248913584e-05,
+      "loss": 1.5077,
+      "step": 9827
+    },
+    {
+      "epoch": 0.3276389258176017,
+      "grad_norm": 1.3129957914352417,
+      "learning_rate": 4.048354433517794e-05,
+      "loss": 1.4965,
+      "step": 9858
+    },
+    {
+      "epoch": 0.32866923690507843,
+      "grad_norm": 1.4380089044570923,
+      "learning_rate": 4.0417520691748916e-05,
+      "loss": 1.5115,
+      "step": 9889
+    },
+    {
+      "epoch": 0.3296995479925552,
+      "grad_norm": 1.3162370920181274,
+      "learning_rate": 4.035132306369438e-05,
+      "loss": 1.5029,
+      "step": 9920
+    },
+    {
+      "epoch": 0.3307298590800319,
+      "grad_norm": 1.3739668130874634,
+      "learning_rate": 4.028495219804555e-05,
+      "loss": 1.5083,
+      "step": 9951
+    },
+    {
+      "epoch": 0.33176017016750864,
+      "grad_norm": 1.3673723936080933,
+      "learning_rate": 4.021840884378864e-05,
+      "loss": 1.5223,
+      "step": 9982
+    },
+    {
+      "epoch": 0.3327904812549854,
+      "grad_norm": 1.3970317840576172,
+      "learning_rate": 4.015169375185633e-05,
+      "loss": 1.5003,
+      "step": 10013
+    },
+    {
+      "epoch": 0.3338207923424621,
+      "grad_norm": 1.2982394695281982,
+      "learning_rate": 4.0084807675119396e-05,
+      "loss": 1.5066,
+      "step": 10044
+    },
+    {
+      "epoch": 0.33485110342993885,
+      "grad_norm": 1.4548689126968384,
+      "learning_rate": 4.0017751368378106e-05,
+      "loss": 1.4993,
+      "step": 10075
+    },
+    {
+      "epoch": 0.3358814145174156,
+      "grad_norm": 1.3693586587905884,
+      "learning_rate": 3.995052558835377e-05,
+      "loss": 1.4987,
+      "step": 10106
+    },
+    {
+      "epoch": 0.3369117256048923,
+      "grad_norm": 1.4046767950057983,
+      "learning_rate": 3.988313109368017e-05,
+      "loss": 1.5098,
+      "step": 10137
+    },
+    {
+      "epoch": 0.33794203669236905,
+      "grad_norm": 1.3772069215774536,
+      "learning_rate": 3.981556864489504e-05,
+      "loss": 1.5165,
+      "step": 10168
+    },
+    {
+      "epoch": 0.3389723477798458,
+      "grad_norm": 1.471211314201355,
+      "learning_rate": 3.974783900443142e-05,
+      "loss": 1.5037,
+      "step": 10199
+    },
+    {
+      "epoch": 0.3400026588673225,
+      "grad_norm": 1.3990979194641113,
+      "learning_rate": 3.9679942936609095e-05,
+      "loss": 1.5096,
+      "step": 10230
+    },
+    {
+      "epoch": 0.34103296995479926,
+      "grad_norm": 1.3779234886169434,
+      "learning_rate": 3.961188120762596e-05,
+      "loss": 1.4914,
+      "step": 10261
+    },
+    {
+      "epoch": 0.342063281042276,
+      "grad_norm": 1.2866768836975098,
+      "learning_rate": 3.954365458554938e-05,
+      "loss": 1.5026,
+      "step": 10292
+    },
+    {
+      "epoch": 0.3430935921297527,
+      "grad_norm": 1.353468894958496,
+      "learning_rate": 3.947526384030751e-05,
+      "loss": 1.5063,
+      "step": 10323
+    },
+    {
+      "epoch": 0.34412390321722947,
+      "grad_norm": 1.3264256715774536,
+      "learning_rate": 3.9406709743680624e-05,
+      "loss": 1.4911,
+      "step": 10354
+    },
+    {
+      "epoch": 0.3451542143047062,
+      "grad_norm": 1.3496876955032349,
+      "learning_rate": 3.9337993069292366e-05,
+      "loss": 1.4921,
+      "step": 10385
+    },
+    {
+      "epoch": 0.3461845253921829,
+      "grad_norm": 1.3812434673309326,
+      "learning_rate": 3.926911459260109e-05,
+      "loss": 1.4826,
+      "step": 10416
+    },
+    {
+      "epoch": 0.34721483647965967,
+      "grad_norm": 1.4926965236663818,
+      "learning_rate": 3.920007509089102e-05,
+      "loss": 1.4994,
+      "step": 10447
+    },
+    {
+      "epoch": 0.3482451475671364,
+      "grad_norm": 1.3446170091629028,
+      "learning_rate": 3.913087534326357e-05,
+      "loss": 1.5114,
+      "step": 10478
+    },
+    {
+      "epoch": 0.3492754586546131,
+      "grad_norm": 1.3100495338439941,
+      "learning_rate": 3.9061516130628475e-05,
+      "loss": 1.5066,
+      "step": 10509
+    },
+    {
+      "epoch": 0.3503057697420899,
+      "grad_norm": 1.395874261856079,
+      "learning_rate": 3.8991998235695025e-05,
+      "loss": 1.4999,
+      "step": 10540
+    },
+    {
+      "epoch": 0.3513360808295666,
+      "grad_norm": 1.3682137727737427,
+      "learning_rate": 3.8922322442963224e-05,
+      "loss": 1.4778,
+      "step": 10571
+    },
+    {
+      "epoch": 0.35236639191704333,
+      "grad_norm": 1.4196573495864868,
+      "learning_rate": 3.885248953871491e-05,
+      "loss": 1.4909,
+      "step": 10602
+    },
+    {
+      "epoch": 0.3533967030045201,
+      "grad_norm": 1.4299864768981934,
+      "learning_rate": 3.8782500311004915e-05,
+      "loss": 1.5025,
+      "step": 10633
+    },
+    {
+      "epoch": 0.3544270140919968,
+      "grad_norm": 1.39677095413208,
+      "learning_rate": 3.871235554965218e-05,
+      "loss": 1.4932,
+      "step": 10664
+    },
+    {
+      "epoch": 0.35545732517947354,
+      "grad_norm": 1.3219736814498901,
+      "learning_rate": 3.864205604623078e-05,
+      "loss": 1.4795,
+      "step": 10695
+    },
+    {
+      "epoch": 0.3564876362669503,
+      "grad_norm": 1.3649324178695679,
+      "learning_rate": 3.857160259406107e-05,
+      "loss": 1.4838,
+      "step": 10726
+    },
+    {
+      "epoch": 0.357517947354427,
+      "grad_norm": 1.4109989404678345,
+      "learning_rate": 3.8500995988200674e-05,
+      "loss": 1.5058,
+      "step": 10757
+    },
+    {
+      "epoch": 0.35854825844190374,
+      "grad_norm": 1.3625038862228394,
+      "learning_rate": 3.843023702543556e-05,
+      "loss": 1.4912,
+      "step": 10788
+    },
+    {
+      "epoch": 0.3595785695293805,
+      "grad_norm": 1.4725775718688965,
+      "learning_rate": 3.8359326504270984e-05,
+      "loss": 1.5012,
+      "step": 10819
+    },
+    {
+      "epoch": 0.3606088806168572,
+      "grad_norm": 1.4126085042953491,
+      "learning_rate": 3.828826522492255e-05,
+      "loss": 1.4977,
+      "step": 10850
+    },
+    {
+      "epoch": 0.36163919170433395,
+      "grad_norm": 1.3949086666107178,
+      "learning_rate": 3.821705398930713e-05,
+      "loss": 1.4903,
+      "step": 10881
+    },
+    {
+      "epoch": 0.3626695027918107,
+      "grad_norm": 1.286792516708374,
+      "learning_rate": 3.814569360103385e-05,
+      "loss": 1.5067,
+      "step": 10912
+    },
+    {
+      "epoch": 0.3636998138792874,
+      "grad_norm": 1.274703025817871,
+      "learning_rate": 3.807418486539499e-05,
+      "loss": 1.4583,
+      "step": 10943
+    },
+    {
+      "epoch": 0.36473012496676416,
+      "grad_norm": 1.401455283164978,
+      "learning_rate": 3.80025285893569e-05,
+      "loss": 1.4834,
+      "step": 10974
+    },
+    {
+      "epoch": 0.3657604360542409,
+      "grad_norm": 1.308361530303955,
+      "learning_rate": 3.793072558155093e-05,
+      "loss": 1.4832,
+      "step": 11005
+    },
+    {
+      "epoch": 0.3667907471417176,
+      "grad_norm": 1.654733419418335,
+      "learning_rate": 3.785877665226426e-05,
+      "loss": 1.4867,
+      "step": 11036
+    },
+    {
+      "epoch": 0.36782105822919436,
+      "grad_norm": 1.3530856370925903,
+      "learning_rate": 3.778668261343079e-05,
+      "loss": 1.4873,
+      "step": 11067
+    },
+    {
+      "epoch": 0.3688513693166711,
+      "grad_norm": 1.3567407131195068,
+      "learning_rate": 3.771444427862192e-05,
+      "loss": 1.4935,
+      "step": 11098
+    },
+    {
+      "epoch": 0.3698816804041478,
+      "grad_norm": 1.3184572458267212,
+      "learning_rate": 3.7642062463037465e-05,
+      "loss": 1.4891,
+      "step": 11129
+    },
+    {
+      "epoch": 0.37091199149162457,
+      "grad_norm": 1.366489291191101,
+      "learning_rate": 3.7569537983496373e-05,
+      "loss": 1.5159,
+      "step": 11160
+    },
+    {
+      "epoch": 0.3719423025791013,
+      "grad_norm": 1.423258662223816,
+      "learning_rate": 3.749687165842753e-05,
+      "loss": 1.4938,
+      "step": 11191
+    },
+    {
+      "epoch": 0.372972613666578,
+      "grad_norm": 1.3226194381713867,
+      "learning_rate": 3.7424064307860536e-05,
+      "loss": 1.499,
+      "step": 11222
+    },
+    {
+      "epoch": 0.3740029247540548,
+      "grad_norm": 1.350500464439392,
+      "learning_rate": 3.735111675341645e-05,
+      "loss": 1.4952,
+      "step": 11253
+    },
+    {
+      "epoch": 0.37503323584153153,
+      "grad_norm": 1.3667839765548706,
+      "learning_rate": 3.7278029818298524e-05,
+      "loss": 1.4763,
+      "step": 11284
+    },
+    {
+      "epoch": 0.37606354692900823,
+      "grad_norm": 1.4876132011413574,
+      "learning_rate": 3.720480432728287e-05,
+      "loss": 1.4913,
+      "step": 11315
+    },
+    {
+      "epoch": 0.377093858016485,
+      "grad_norm": 1.3927743434906006,
+      "learning_rate": 3.71314411067092e-05,
+      "loss": 1.4948,
+      "step": 11346
+    },
+    {
+      "epoch": 0.37812416910396174,
+      "grad_norm": 1.3752413988113403,
+      "learning_rate": 3.70579409844715e-05,
+      "loss": 1.4763,
+      "step": 11377
+    },
+    {
+      "epoch": 0.37915448019143844,
+      "grad_norm": 1.3530951738357544,
+      "learning_rate": 3.698430479000865e-05,
+      "loss": 1.5077,
+      "step": 11408
+    },
+    {
+      "epoch": 0.3801847912789152,
+      "grad_norm": 1.4309345483779907,
+      "learning_rate": 3.691053335429509e-05,
+      "loss": 1.4945,
+      "step": 11439
+    },
+    {
+      "epoch": 0.38121510236639194,
+      "grad_norm": 1.2874380350112915,
+      "learning_rate": 3.683662750983147e-05,
+      "loss": 1.4698,
+      "step": 11470
+    },
+    {
+      "epoch": 0.38224541345386864,
+      "grad_norm": 1.3356250524520874,
+      "learning_rate": 3.676258809063518e-05,
+      "loss": 1.4924,
+      "step": 11501
+    },
+    {
+      "epoch": 0.3832757245413454,
+      "grad_norm": 1.304559588432312,
+      "learning_rate": 3.6688415932231004e-05,
+      "loss": 1.4682,
+      "step": 11532
+    },
+    {
+      "epoch": 0.3843060356288221,
+      "grad_norm": 1.4153447151184082,
+      "learning_rate": 3.661411187164166e-05,
+      "loss": 1.4989,
+      "step": 11563
+    },
+    {
+      "epoch": 0.38533634671629885,
+      "grad_norm": 1.356992244720459,
+      "learning_rate": 3.65396767473784e-05,
+      "loss": 1.4854,
+      "step": 11594
+    },
+    {
+      "epoch": 0.3863666578037756,
+      "grad_norm": 1.322449803352356,
+      "learning_rate": 3.6465111399431465e-05,
+      "loss": 1.4877,
+      "step": 11625
+    },
+    {
+      "epoch": 0.3873969688912523,
+      "grad_norm": 1.3981350660324097,
+      "learning_rate": 3.6390416669260674e-05,
+      "loss": 1.499,
+      "step": 11656
+    },
+    {
+      "epoch": 0.38842727997872906,
+      "grad_norm": 1.324871301651001,
+      "learning_rate": 3.63155933997859e-05,
+      "loss": 1.4814,
+      "step": 11687
+    },
+    {
+      "epoch": 0.3894575910662058,
+      "grad_norm": 1.3940790891647339,
+      "learning_rate": 3.624064243537758e-05,
+      "loss": 1.4754,
+      "step": 11718
+    },
+    {
+      "epoch": 0.3904879021536825,
+      "grad_norm": 1.2880780696868896,
+      "learning_rate": 3.616556462184716e-05,
+      "loss": 1.4832,
+      "step": 11749
+    },
+    {
+      "epoch": 0.39151821324115926,
+      "grad_norm": 1.315329670906067,
+      "learning_rate": 3.609036080643755e-05,
+      "loss": 1.4853,
+      "step": 11780
+    },
+    {
+      "epoch": 0.392548524328636,
+      "grad_norm": 1.4093523025512695,
+      "learning_rate": 3.60150318378136e-05,
+      "loss": 1.4978,
+      "step": 11811
+    },
+    {
+      "epoch": 0.3935788354161127,
+      "grad_norm": 1.271151065826416,
+      "learning_rate": 3.5939578566052465e-05,
+      "loss": 1.4933,
+      "step": 11842
+    },
+    {
+      "epoch": 0.39460914650358947,
+      "grad_norm": 1.2910923957824707,
+      "learning_rate": 3.586400184263408e-05,
+      "loss": 1.4853,
+      "step": 11873
+    },
+    {
+      "epoch": 0.3956394575910662,
+      "grad_norm": 1.2480064630508423,
+      "learning_rate": 3.578830252043148e-05,
+      "loss": 1.4642,
+      "step": 11904
+    },
+    {
+      "epoch": 0.3966697686785429,
+      "grad_norm": 1.263197422027588,
+      "learning_rate": 3.571248145370125e-05,
+      "loss": 1.4812,
+      "step": 11935
+    },
+    {
+      "epoch": 0.3977000797660197,
+      "grad_norm": 1.3231288194656372,
+      "learning_rate": 3.5636539498073794e-05,
+      "loss": 1.4744,
+      "step": 11966
+    },
+    {
+      "epoch": 0.39873039085349643,
+      "grad_norm": 1.3933110237121582,
+      "learning_rate": 3.556047751054378e-05,
+      "loss": 1.4849,
+      "step": 11997
+    },
+    {
+      "epoch": 0.39976070194097313,
+      "grad_norm": 1.3615801334381104,
+      "learning_rate": 3.548429634946039e-05,
+      "loss": 1.4866,
+      "step": 12028
+    },
+    {
+      "epoch": 0.4007910130284499,
+      "grad_norm": 1.298638939857483,
+      "learning_rate": 3.540799687451768e-05,
+      "loss": 1.4664,
+      "step": 12059
+    },
+    {
+      "epoch": 0.40182132411592664,
+      "grad_norm": 1.29216468334198,
+      "learning_rate": 3.533157994674485e-05,
+      "loss": 1.4697,
+      "step": 12090
+    },
+    {
+      "epoch": 0.40285163520340334,
+      "grad_norm": 1.3759845495224,
+      "learning_rate": 3.5255046428496546e-05,
+      "loss": 1.4854,
+      "step": 12121
+    },
+    {
+      "epoch": 0.4038819462908801,
+      "grad_norm": 1.4045615196228027,
+      "learning_rate": 3.517839718344311e-05,
+      "loss": 1.4622,
+      "step": 12152
+    },
+    {
+      "epoch": 0.40491225737835684,
+      "grad_norm": 1.2979034185409546,
+      "learning_rate": 3.510163307656086e-05,
+      "loss": 1.4797,
+      "step": 12183
+    },
+    {
+      "epoch": 0.40594256846583354,
+      "grad_norm": 1.303139567375183,
+      "learning_rate": 3.5024754974122324e-05,
+      "loss": 1.4588,
+      "step": 12214
+    },
+    {
+      "epoch": 0.4069728795533103,
+      "grad_norm": 1.287781834602356,
+      "learning_rate": 3.494776374368643e-05,
+      "loss": 1.4834,
+      "step": 12245
+    },
+    {
+      "epoch": 0.40800319064078705,
+      "grad_norm": 1.3806688785552979,
+      "learning_rate": 3.4870660254088724e-05,
+      "loss": 1.4807,
+      "step": 12276
+    },
+    {
+      "epoch": 0.40903350172826375,
+      "grad_norm": 1.4059745073318481,
+      "learning_rate": 3.479344537543164e-05,
+      "loss": 1.4906,
+      "step": 12307
+    },
+    {
+      "epoch": 0.4100638128157405,
+      "grad_norm": 1.3052942752838135,
+      "learning_rate": 3.4716119979074565e-05,
+      "loss": 1.4801,
+      "step": 12338
+    },
+    {
+      "epoch": 0.41109412390321726,
+      "grad_norm": 1.3306844234466553,
+      "learning_rate": 3.463868493762412e-05,
+      "loss": 1.4911,
+      "step": 12369
+    },
+    {
+      "epoch": 0.41212443499069396,
+      "grad_norm": 1.3276656866073608,
+      "learning_rate": 3.456114112492418e-05,
+      "loss": 1.4678,
+      "step": 12400
+    },
+    {
+      "epoch": 0.4131547460781707,
+      "grad_norm": 1.3164253234863281,
+      "learning_rate": 3.4483489416046164e-05,
+      "loss": 1.4816,
+      "step": 12431
+    },
+    {
+      "epoch": 0.4141850571656474,
+      "grad_norm": 1.3827886581420898,
+      "learning_rate": 3.440573068727905e-05,
+      "loss": 1.481,
+      "step": 12462
+    },
+    {
+      "epoch": 0.41521536825312416,
+      "grad_norm": 1.2899463176727295,
+      "learning_rate": 3.4327865816119495e-05,
+      "loss": 1.4575,
+      "step": 12493
+    },
+    {
+      "epoch": 0.4162456793406009,
+      "grad_norm": 1.3136677742004395,
+      "learning_rate": 3.4249895681262025e-05,
+      "loss": 1.4695,
+      "step": 12524
+    },
+    {
+      "epoch": 0.4172759904280776,
+      "grad_norm": 1.2920372486114502,
+      "learning_rate": 3.417182116258899e-05,
+      "loss": 1.4765,
+      "step": 12555
+    },
+    {
+      "epoch": 0.41830630151555437,
+      "grad_norm": 1.3285510540008545,
+      "learning_rate": 3.409364314116074e-05,
+      "loss": 1.4559,
+      "step": 12586
+    },
+    {
+      "epoch": 0.4193366126030311,
+      "grad_norm": 1.2834984064102173,
+      "learning_rate": 3.401536249920559e-05,
+      "loss": 1.4706,
+      "step": 12617
+    },
+    {
+      "epoch": 0.4203669236905078,
+      "grad_norm": 1.315942645072937,
+      "learning_rate": 3.393698012010998e-05,
+      "loss": 1.4692,
+      "step": 12648
+    },
+    {
+      "epoch": 0.4213972347779846,
+      "grad_norm": 1.3668091297149658,
+      "learning_rate": 3.385849688840839e-05,
+      "loss": 1.4801,
+      "step": 12679
+    },
+    {
+      "epoch": 0.42242754586546133,
+      "grad_norm": 1.312280297279358,
+      "learning_rate": 3.3779913689773414e-05,
+      "loss": 1.4673,
+      "step": 12710
+    },
+    {
+      "epoch": 0.423457856952938,
+      "grad_norm": 1.3579858541488647,
+      "learning_rate": 3.370123141100578e-05,
+      "loss": 1.4578,
+      "step": 12741
+    },
+    {
+      "epoch": 0.4244881680404148,
+      "grad_norm": 1.4001456499099731,
+      "learning_rate": 3.3622450940024305e-05,
+      "loss": 1.4787,
+      "step": 12772
+    },
+    {
+      "epoch": 0.42551847912789154,
+      "grad_norm": 1.352629542350769,
+      "learning_rate": 3.35435731658559e-05,
+      "loss": 1.457,
+      "step": 12803
+    },
+    {
+      "epoch": 0.42654879021536823,
+      "grad_norm": 1.4044222831726074,
+      "learning_rate": 3.346459897862552e-05,
+      "loss": 1.4979,
+      "step": 12834
+    },
+    {
+      "epoch": 0.427579101302845,
+      "grad_norm": 1.2666436433792114,
+      "learning_rate": 3.338552926954613e-05,
+      "loss": 1.4712,
+      "step": 12865
+    },
+    {
+      "epoch": 0.42860941239032174,
+      "grad_norm": 1.2487694025039673,
+      "learning_rate": 3.330636493090868e-05,
+      "loss": 1.4784,
+      "step": 12896
+    },
+    {
+      "epoch": 0.42963972347779844,
+      "grad_norm": 1.2346290349960327,
+      "learning_rate": 3.322710685607193e-05,
+      "loss": 1.4754,
+      "step": 12927
+    },
+    {
+      "epoch": 0.4306700345652752,
+      "grad_norm": 1.2908893823623657,
+      "learning_rate": 3.314775593945251e-05,
+      "loss": 1.4677,
+      "step": 12958
+    },
+    {
+      "epoch": 0.43170034565275195,
+      "grad_norm": 1.3283506631851196,
+      "learning_rate": 3.3068313076514714e-05,
+      "loss": 1.4661,
+      "step": 12989
+    },
+    {
+      "epoch": 0.43273065674022865,
+      "grad_norm": 1.2982537746429443,
+      "learning_rate": 3.298877916376047e-05,
+      "loss": 1.4838,
+      "step": 13020
+    },
+    {
+      "epoch": 0.4337609678277054,
+      "grad_norm": 1.3566454648971558,
+      "learning_rate": 3.290915509871915e-05,
+      "loss": 1.4683,
+      "step": 13051
+    },
+    {
+      "epoch": 0.43479127891518216,
+      "grad_norm": 1.3470877408981323,
+      "learning_rate": 3.282944177993753e-05,
+      "loss": 1.4724,
+      "step": 13082
+    },
+    {
+      "epoch": 0.43582159000265885,
+      "grad_norm": 1.451150894165039,
+      "learning_rate": 3.274964010696957e-05,
+      "loss": 1.4731,
+      "step": 13113
+    },
+    {
+      "epoch": 0.4368519010901356,
+      "grad_norm": 1.3415958881378174,
+      "learning_rate": 3.266975098036629e-05,
+      "loss": 1.4809,
+      "step": 13144
+    },
+    {
+      "epoch": 0.43788221217761236,
+      "grad_norm": 1.2775352001190186,
+      "learning_rate": 3.258977530166562e-05,
+      "loss": 1.4523,
+      "step": 13175
+    },
+    {
+      "epoch": 0.43891252326508906,
+      "grad_norm": 1.365050196647644,
+      "learning_rate": 3.250971397338227e-05,
+      "loss": 1.4611,
+      "step": 13206
+    },
+    {
+      "epoch": 0.4399428343525658,
+      "grad_norm": 1.3481686115264893,
+      "learning_rate": 3.2429567898997404e-05,
+      "loss": 1.4708,
+      "step": 13237
+    },
+    {
+      "epoch": 0.44097314544004257,
+      "grad_norm": 1.3418121337890625,
+      "learning_rate": 3.234933798294859e-05,
+      "loss": 1.485,
+      "step": 13268
+    },
+    {
+      "epoch": 0.44200345652751927,
+      "grad_norm": 1.3098441362380981,
+      "learning_rate": 3.2269025130619535e-05,
+      "loss": 1.472,
+      "step": 13299
+    },
+    {
+      "epoch": 0.443033767614996,
+      "grad_norm": 1.2792437076568604,
+      "learning_rate": 3.218863024832985e-05,
+      "loss": 1.4592,
+      "step": 13330
+    },
+    {
+      "epoch": 0.4440640787024727,
+      "grad_norm": 1.3804035186767578,
+      "learning_rate": 3.2108154243324864e-05,
+      "loss": 1.4546,
+      "step": 13361
+    },
+    {
+      "epoch": 0.4450943897899495,
+      "grad_norm": 1.287787675857544,
+      "learning_rate": 3.2027598023765345e-05,
+      "loss": 1.4477,
+      "step": 13392
+    },
+    {
+      "epoch": 0.44612470087742623,
+      "grad_norm": 1.5964646339416504,
+      "learning_rate": 3.194696249871729e-05,
+      "loss": 1.4468,
+      "step": 13423
+    },
+    {
+      "epoch": 0.4471550119649029,
+      "grad_norm": 1.3253474235534668,
+      "learning_rate": 3.186624857814164e-05,
+      "loss": 1.4588,
+      "step": 13454
+    },
+    {
+      "epoch": 0.4481853230523797,
+      "grad_norm": 1.288176417350769,
+      "learning_rate": 3.178545717288401e-05,
+      "loss": 1.4644,
+      "step": 13485
+    },
+    {
+      "epoch": 0.44921563413985643,
+      "grad_norm": 1.3357142210006714,
+      "learning_rate": 3.170458919466444e-05,
+      "loss": 1.4871,
+      "step": 13516
+    },
+    {
+      "epoch": 0.45024594522733313,
+      "grad_norm": 1.2954436540603638,
+      "learning_rate": 3.1623645556067063e-05,
+      "loss": 1.4571,
+      "step": 13547
+    },
+    {
+      "epoch": 0.4512762563148099,
+      "grad_norm": 1.344789981842041,
+      "learning_rate": 3.154262717052985e-05,
+      "loss": 1.459,
+      "step": 13578
+    },
+    {
+      "epoch": 0.45230656740228664,
+      "grad_norm": 1.2648475170135498,
+      "learning_rate": 3.146153495233426e-05,
+      "loss": 1.4496,
+      "step": 13609
+    },
+    {
+      "epoch": 0.45333687848976334,
+      "grad_norm": 1.312733769416809,
+      "learning_rate": 3.1380369816594944e-05,
+      "loss": 1.4309,
+      "step": 13640
+    },
+    {
+      "epoch": 0.4543671895772401,
+      "grad_norm": 1.3719325065612793,
+      "learning_rate": 3.129913267924946e-05,
+      "loss": 1.4723,
+      "step": 13671
+    },
+    {
+      "epoch": 0.45539750066471685,
+      "grad_norm": 1.2850617170333862,
+      "learning_rate": 3.121782445704782e-05,
+      "loss": 1.4599,
+      "step": 13702
+    },
+    {
+      "epoch": 0.45642781175219355,
+      "grad_norm": 1.3335177898406982,
+      "learning_rate": 3.11364460675423e-05,
+      "loss": 1.4821,
+      "step": 13733
+    },
+    {
+      "epoch": 0.4574581228396703,
+      "grad_norm": 1.1675069332122803,
+      "learning_rate": 3.1054998429076934e-05,
+      "loss": 1.453,
+      "step": 13764
+    },
+    {
+      "epoch": 0.45848843392714705,
+      "grad_norm": 1.283544898033142,
+      "learning_rate": 3.097348246077728e-05,
+      "loss": 1.4545,
+      "step": 13795
+    },
+    {
+      "epoch": 0.45951874501462375,
+      "grad_norm": 1.4358693361282349,
+      "learning_rate": 3.0891899082539924e-05,
+      "loss": 1.4673,
+      "step": 13826
+    },
+    {
+      "epoch": 0.4605490561021005,
+      "grad_norm": 1.2551497220993042,
+      "learning_rate": 3.0810249215022233e-05,
+      "loss": 1.4532,
+      "step": 13857
+    },
+    {
+      "epoch": 0.46157936718957726,
+      "grad_norm": 1.2574602365493774,
+      "learning_rate": 3.0728533779631865e-05,
+      "loss": 1.4762,
+      "step": 13888
+    },
+    {
+      "epoch": 0.46260967827705396,
+      "grad_norm": 1.2202764749526978,
+      "learning_rate": 3.064675369851637e-05,
+      "loss": 1.4461,
+      "step": 13919
+    },
+    {
+      "epoch": 0.4636399893645307,
+      "grad_norm": 1.2787501811981201,
+      "learning_rate": 3.056490989455289e-05,
+      "loss": 1.4607,
+      "step": 13950
+    },
+    {
+      "epoch": 0.46467030045200747,
+      "grad_norm": 1.2511006593704224,
+      "learning_rate": 3.0483003291337596e-05,
+      "loss": 1.4548,
+      "step": 13981
+    },
+    {
+      "epoch": 0.46570061153948417,
+      "grad_norm": 1.2749834060668945,
+      "learning_rate": 3.040103481317539e-05,
+      "loss": 1.4394,
+      "step": 14012
+    },
+    {
+      "epoch": 0.4667309226269609,
+      "grad_norm": 1.223057746887207,
+      "learning_rate": 3.03190053850694e-05,
+      "loss": 1.4684,
+      "step": 14043
+    },
+    {
+      "epoch": 0.4677612337144377,
+      "grad_norm": 1.39846932888031,
+      "learning_rate": 3.0236915932710573e-05,
+      "loss": 1.4657,
+      "step": 14074
+    },
+    {
+      "epoch": 0.4687915448019144,
+      "grad_norm": 1.5305665731430054,
+      "learning_rate": 3.0154767382467232e-05,
+      "loss": 1.4795,
+      "step": 14105
+    },
+    {
+      "epoch": 0.4698218558893911,
+      "grad_norm": 1.2569035291671753,
+      "learning_rate": 3.0072560661374582e-05,
+      "loss": 1.4756,
+      "step": 14136
+    },
+    {
+      "epoch": 0.4708521669768679,
+      "grad_norm": 1.3472824096679688,
+      "learning_rate": 2.999029669712431e-05,
+      "loss": 1.4682,
+      "step": 14167
+    },
+    {
+      "epoch": 0.4718824780643446,
+      "grad_norm": 1.271714210510254,
+      "learning_rate": 2.990797641805408e-05,
+      "loss": 1.4509,
+      "step": 14198
+    },
+    {
+      "epoch": 0.47291278915182133,
+      "grad_norm": 1.3342047929763794,
+      "learning_rate": 2.982560075313704e-05,
+      "loss": 1.4528,
+      "step": 14229
+    },
+    {
+      "epoch": 0.47394310023929803,
+      "grad_norm": 1.5821506977081299,
+      "learning_rate": 2.9743170631971368e-05,
+      "loss": 1.4609,
+      "step": 14260
+    },
+    {
+      "epoch": 0.4749734113267748,
+      "grad_norm": 1.2598062753677368,
+      "learning_rate": 2.9660686984769792e-05,
+      "loss": 1.471,
+      "step": 14291
+    },
+    {
+      "epoch": 0.47600372241425154,
+      "grad_norm": 1.2648885250091553,
+      "learning_rate": 2.9578150742349047e-05,
+      "loss": 1.4708,
+      "step": 14322
+    },
+    {
+      "epoch": 0.47703403350172824,
+      "grad_norm": 1.559665560722351,
+      "learning_rate": 2.949556283611942e-05,
+      "loss": 1.4516,
+      "step": 14353
+    },
+    {
+      "epoch": 0.478064344589205,
+      "grad_norm": 1.2621581554412842,
+      "learning_rate": 2.9412924198074206e-05,
+      "loss": 1.446,
+      "step": 14384
+    },
+    {
+      "epoch": 0.47909465567668175,
+      "grad_norm": 1.2775017023086548,
+      "learning_rate": 2.9330235760779208e-05,
+      "loss": 1.4496,
+      "step": 14415
+    },
+    {
+      "epoch": 0.48012496676415845,
+      "grad_norm": 1.2010388374328613,
+      "learning_rate": 2.9247498457362188e-05,
+      "loss": 1.4606,
+      "step": 14446
+    },
+    {
+      "epoch": 0.4811552778516352,
+      "grad_norm": 1.3053895235061646,
+      "learning_rate": 2.9164713221502373e-05,
+      "loss": 1.4536,
+      "step": 14477
+    },
+    {
+      "epoch": 0.48218558893911195,
+      "grad_norm": 1.311596155166626,
+      "learning_rate": 2.9081880987419912e-05,
+      "loss": 1.4409,
+      "step": 14508
+    },
+    {
+      "epoch": 0.48321590002658865,
+      "grad_norm": 1.3888933658599854,
+      "learning_rate": 2.8999002689865296e-05,
+      "loss": 1.4314,
+      "step": 14539
+    },
+    {
+      "epoch": 0.4842462111140654,
+      "grad_norm": 1.288619875907898,
+      "learning_rate": 2.8916079264108852e-05,
+      "loss": 1.4539,
+      "step": 14570
+    },
+    {
+      "epoch": 0.48527652220154216,
+      "grad_norm": 1.2974294424057007,
+      "learning_rate": 2.883311164593017e-05,
+      "loss": 1.4627,
+      "step": 14601
+    },
+    {
+      "epoch": 0.48630683328901886,
+      "grad_norm": 1.2057379484176636,
+      "learning_rate": 2.875010077160754e-05,
+      "loss": 1.4578,
+      "step": 14632
+    },
+    {
+      "epoch": 0.4873371443764956,
+      "grad_norm": 1.363971471786499,
+      "learning_rate": 2.866704757790741e-05,
+      "loss": 1.4671,
+      "step": 14663
+    },
+    {
+      "epoch": 0.48836745546397237,
+      "grad_norm": 1.2696925401687622,
+      "learning_rate": 2.858395300207376e-05,
+      "loss": 1.4333,
+      "step": 14694
+    },
+    {
+      "epoch": 0.48939776655144906,
+      "grad_norm": 1.2653478384017944,
+      "learning_rate": 2.8500817981817607e-05,
+      "loss": 1.4662,
+      "step": 14725
+    },
+    {
+      "epoch": 0.4904280776389258,
+      "grad_norm": 1.3011239767074585,
+      "learning_rate": 2.8417643455306336e-05,
+      "loss": 1.4589,
+      "step": 14756
+    },
+    {
+      "epoch": 0.4914583887264026,
+      "grad_norm": 1.3312432765960693,
+      "learning_rate": 2.8334430361153185e-05,
+      "loss": 1.4368,
+      "step": 14787
+    },
+    {
+      "epoch": 0.49248869981387927,
+      "grad_norm": 1.3015661239624023,
+      "learning_rate": 2.8251179638406612e-05,
+      "loss": 1.466,
+      "step": 14818
+    },
+    {
+      "epoch": 0.493519010901356,
+      "grad_norm": 1.3215759992599487,
+      "learning_rate": 2.8167892226539704e-05,
+      "loss": 1.4486,
+      "step": 14849
+    },
+    {
+      "epoch": 0.4945493219888328,
+      "grad_norm": 1.2909883260726929,
+      "learning_rate": 2.8084569065439588e-05,
+      "loss": 1.4433,
+      "step": 14880
+    },
+    {
+      "epoch": 0.4955796330763095,
+      "grad_norm": 1.364015817642212,
+      "learning_rate": 2.8001211095396807e-05,
+      "loss": 1.4449,
+      "step": 14911
+    },
+    {
+      "epoch": 0.49660994416378623,
+      "grad_norm": 1.2468819618225098,
+      "learning_rate": 2.791781925709473e-05,
+      "loss": 1.4572,
+      "step": 14942
+    },
+    {
+      "epoch": 0.497640255251263,
+      "grad_norm": 1.2739325761795044,
+      "learning_rate": 2.7834394491598908e-05,
+      "loss": 1.4478,
+      "step": 14973
+    },
+    {
+      "epoch": 0.4986705663387397,
+      "grad_norm": 1.3384937047958374,
+      "learning_rate": 2.7750937740346485e-05,
+      "loss": 1.4429,
+      "step": 15004
+    },
+    {
+      "epoch": 0.49970087742621644,
+      "grad_norm": 1.231088399887085,
+      "learning_rate": 2.7667449945135564e-05,
+      "loss": 1.4631,
+      "step": 15035
+    },
+    {
+      "epoch": 0.5007311885136931,
+      "grad_norm": 1.2262307405471802,
+      "learning_rate": 2.7583932048114557e-05,
+      "loss": 1.4508,
+      "step": 15066
+    },
+    {
+      "epoch": 0.5017614996011699,
+      "grad_norm": 1.3427774906158447,
+      "learning_rate": 2.7500384991771587e-05,
+      "loss": 1.4441,
+      "step": 15097
+    },
+    {
+      "epoch": 0.5027918106886466,
+      "grad_norm": 1.2950241565704346,
+      "learning_rate": 2.7416809718923825e-05,
+      "loss": 1.4427,
+      "step": 15128
+    },
+    {
+      "epoch": 0.5038221217761234,
+      "grad_norm": 1.4129016399383545,
+      "learning_rate": 2.7333207172706864e-05,
+      "loss": 1.4562,
+      "step": 15159
+    },
+    {
+      "epoch": 0.5048524328636002,
+      "grad_norm": 1.2751520872116089,
+      "learning_rate": 2.7249578296564088e-05,
+      "loss": 1.4517,
+      "step": 15190
+    },
+    {
+      "epoch": 0.5058827439510768,
+      "grad_norm": 1.302485466003418,
+      "learning_rate": 2.7165924034235973e-05,
+      "loss": 1.4327,
+      "step": 15221
+    },
+    {
+      "epoch": 0.5069130550385536,
+      "grad_norm": 1.295390009880066,
+      "learning_rate": 2.708224532974953e-05,
+      "loss": 1.4455,
+      "step": 15252
+    }
+  ],
+  "logging_steps": 31,
+  "max_steps": 30517,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 2,
+  "save_steps": 3052,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 1.1319725270111355e+19,
+  "train_batch_size": 8,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-15260/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a5a23be0ff07d6d3142f7c0980f91dddba845519c24fcb411cbb4b9ddb1513ff
+size 5304

checkpoint-18312/config.json

@@ -0,0 +1,39 @@
+{
+  "_name_or_path": "/mnt/parscratch/users/acp23ay/private/models/Llama-3.1-8B-Instruct-ta-madlad-mean/",
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 128000,
+  "eos_token_id": [
+    128001,
+    128008,
+    128009
+  ],
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 131072,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 8,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": {
+    "factor": 8.0,
+    "high_freq_factor": 4.0,
+    "low_freq_factor": 1.0,
+    "original_max_position_embeddings": 8192,
+    "rope_type": "llama3"
+  },
+  "rope_theta": 500000.0,
+  "tie_word_embeddings": false,
+  "torch_dtype": "float32",
+  "transformers_version": "4.44.0.dev0",
+  "use_cache": true,
+  "vocab_size": 138256
+}

checkpoint-18312/generation_config.json

@@ -0,0 +1,12 @@
+{
+  "bos_token_id": 128000,
+  "do_sample": true,
+  "eos_token_id": [
+    128001,
+    128008,
+    128009
+  ],
+  "temperature": 0.6,
+  "top_p": 0.9,
+  "transformers_version": "4.44.0.dev0"
+}

checkpoint-18312/model-00001-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:176c8b34bed0fe83d7de45fa0883e11f12556007adaec5461125663407ad56e9
+size 4983197184

checkpoint-18312/model-00002-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:168629c67732309d49f60a5ec48a6d160212bc987365e82e183dfbf74ba0c1f3
+size 4899116432

checkpoint-18312/model-00003-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1a223e87240e50dfb05634ea909cf1ec715d37a12b681b14a3a6e97f28688f2b
+size 4999813112

checkpoint-18312/model-00004-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3047d199c611c6d12b1cca780f033ad89df9c076f742674f7ac4a892ca9573b3
+size 4999813128

checkpoint-18312/model-00005-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0a2f91d749d664ecd22c71f3d54488966fc6899a03924b8fea3613c8f83398a8
+size 4832007496

checkpoint-18312/model-00006-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fca2d0a06f00b713aa3dad380d1fec95db9cd12c94b4ab55eaf000762605a13f
+size 4999813120

checkpoint-18312/model-00007-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9be579eb0b874544937eb7b61367b399fdf570f8845569d7e909a3d9d90b5922
+size 2734998184

checkpoint-18312/model.safetensors.index.json

@@ -0,0 +1,298 @@
+{
+  "metadata": {
+    "total_size": 32448724992
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00007-of-00007.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00007-of-00007.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00007-of-00007.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00007-of-00007.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00007-of-00007.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "model.norm.weight": "model-00007-of-00007.safetensors"
+  }
+}

checkpoint-18312/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7b9f31e5915c8018416928425cb14962b8311870cc229d12289dfdf38b52cb27
+size 16040396334

checkpoint-18312/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9d9cd6a0487226e5bd30d1846894c82af483733ab4381b75bae9c0745e05d405
+size 14244

checkpoint-18312/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a0e65c3d6f29e706fd941a38280ce5628189a6998eac6d29abbeab00ad838d00
+size 1064

checkpoint-18312/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a5a23be0ff07d6d3142f7c0980f91dddba845519c24fcb411cbb4b9ddb1513ff
+size 5304

checkpoint-21364/config.json

@@ -0,0 +1,39 @@
+{
+  "_name_or_path": "/mnt/parscratch/users/acp23ay/private/models/Llama-3.1-8B-Instruct-ta-madlad-mean/",
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 128000,
+  "eos_token_id": [
+    128001,
+    128008,
+    128009
+  ],
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 131072,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 8,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": {
+    "factor": 8.0,
+    "high_freq_factor": 4.0,
+    "low_freq_factor": 1.0,
+    "original_max_position_embeddings": 8192,
+    "rope_type": "llama3"
+  },
+  "rope_theta": 500000.0,
+  "tie_word_embeddings": false,
+  "torch_dtype": "float32",
+  "transformers_version": "4.44.0.dev0",
+  "use_cache": true,
+  "vocab_size": 138256
+}

checkpoint-21364/generation_config.json

@@ -0,0 +1,12 @@
+{
+  "bos_token_id": 128000,
+  "do_sample": true,
+  "eos_token_id": [
+    128001,
+    128008,
+    128009
+  ],
+  "temperature": 0.6,
+  "top_p": 0.9,
+  "transformers_version": "4.44.0.dev0"
+}

checkpoint-21364/model-00001-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6c8ea81152baa18da53fb7a56880bb8364fd5199a4d6623e2f74408a5fc25d11
+size 4983197184

checkpoint-21364/model-00002-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:168629c67732309d49f60a5ec48a6d160212bc987365e82e183dfbf74ba0c1f3
+size 4899116432

checkpoint-21364/model-00003-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1a223e87240e50dfb05634ea909cf1ec715d37a12b681b14a3a6e97f28688f2b
+size 4999813112