Model save

README.md

@@ -0,0 +1,59 @@
+---
+license: apache-2.0
+base_model: orpo-explorers/kaist-mistral-orpo-capybara-beta-0.05
+tags:
+- trl
+- orpo
+- generated_from_trainer
+model-index:
+- name: kaist-mistral-orpo-capybara-ohp-3epoch-beta-0.2
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# kaist-mistral-orpo-capybara-ohp-3epoch-beta-0.2
+
+This model is a fine-tuned version of [orpo-explorers/kaist-mistral-orpo-capybara-beta-0.05](https://huggingface.co/orpo-explorers/kaist-mistral-orpo-capybara-beta-0.05) on an unknown dataset.
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 5e-06
+- train_batch_size: 8
+- eval_batch_size: 8
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 4
+- gradient_accumulation_steps: 2
+- total_train_batch_size: 64
+- total_eval_batch_size: 32
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: cosine
+- num_epochs: 3
+
+### Training results
+
+
+
+### Framework versions
+
+- Transformers 4.39.3
+- Pytorch 2.1.2.post303
+- Datasets 2.18.0
+- Tokenizers 0.15.2

all_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 2.99,
+    "train_loss": 0.45327196680595605,
+    "train_runtime": 9631.2387,
+    "train_samples": 14928,
+    "train_samples_per_second": 4.65,
+    "train_steps_per_second": 0.073
+}

generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "transformers_version": "4.39.3"
+}

model-00001-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2311b8d3e2b1f87992026c15425a2341ef67a7bac3dbb21ea51cd7012397289c
+size 4943162336

model-00002-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e3e95d39ab494c2e5c1a384e4e19e522ee0075aed538e907111e121ee20b0325
+size 4999819336

model-00003-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e53a10a0d5f91c7537273be1f2dea9b33014b3ecdd4054ce589ad02c9576a32d
+size 4540516344

model.safetensors.index.json

@@ -0,0 +1,298 @@
+{
+  "metadata": {
+    "total_size": 14483464192
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00003-of-00003.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.norm.weight": "model-00003-of-00003.safetensors"
+  }
+}

train_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 2.99,
+    "train_loss": 0.45327196680595605,
+    "train_runtime": 9631.2387,
+    "train_samples": 14928,
+    "train_samples_per_second": 4.65,
+    "train_steps_per_second": 0.073
+}

trainer_state.json

@@ -0,0 +1,2532 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 2.993576017130621,
+  "eval_steps": 500,
+  "global_step": 699,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.02,
+      "grad_norm": 5.09375,
+      "learning_rate": 4.999368785249369e-06,
+      "log_odds_chosen": 0.06298448145389557,
+      "log_odds_ratio": -0.7070032954216003,
+      "logits/chosen": -2.986868143081665,
+      "logits/rejected": -2.972076892852783,
+      "logps/chosen": -0.7561292052268982,
+      "logps/rejected": -0.8044994473457336,
+      "loss": 0.6324,
+      "nll_loss": 0.44739198684692383,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -0.15122584998607635,
+      "rewards/margins": 0.00967404805123806,
+      "rewards/rejected": -0.16089990735054016,
+      "step": 5
+    },
+    {
+      "epoch": 0.04,
+      "grad_norm": 4.8125,
+      "learning_rate": 4.997475459743124e-06,
+      "log_odds_chosen": 0.0681157261133194,
+      "log_odds_ratio": -0.7049038410186768,
+      "logits/chosen": -2.9765102863311768,
+      "logits/rejected": -2.961024045944214,
+      "logps/chosen": -0.7070090174674988,
+      "logps/rejected": -0.7467236518859863,
+      "loss": 0.6065,
+      "nll_loss": 0.4899875521659851,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -0.14140181243419647,
+      "rewards/margins": 0.007942913100123405,
+      "rewards/rejected": -0.14934472739696503,
+      "step": 10
+    },
+    {
+      "epoch": 0.06,
+      "grad_norm": 3.703125,
+      "learning_rate": 4.994320979557256e-06,
+      "log_odds_chosen": -0.007408215198665857,
+      "log_odds_ratio": -0.7562711834907532,
+      "logits/chosen": -2.9262454509735107,
+      "logits/rejected": -2.9222850799560547,
+      "logps/chosen": -0.7819508910179138,
+      "logps/rejected": -0.7833510637283325,
+      "loss": 0.5555,
+      "nll_loss": 0.3846359848976135,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": -0.15639019012451172,
+      "rewards/margins": 0.0002800210495479405,
+      "rewards/rejected": -0.1566702127456665,
+      "step": 15
+    },
+    {
+      "epoch": 0.09,
+      "grad_norm": 4.5,
+      "learning_rate": 4.989906937615302e-06,
+      "log_odds_chosen": 0.09043633192777634,
+      "log_odds_ratio": -0.703377902507782,
+      "logits/chosen": -2.915590763092041,
+      "logits/rejected": -2.920872688293457,
+      "logps/chosen": -0.7168866395950317,
+      "logps/rejected": -0.7641353011131287,
+      "loss": 0.5684,
+      "nll_loss": 0.4744255542755127,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.14337733387947083,
+      "rewards/margins": 0.009449719451367855,
+      "rewards/rejected": -0.15282705426216125,
+      "step": 20
+    },
+    {
+      "epoch": 0.11,
+      "grad_norm": 4.53125,
+      "learning_rate": 4.984235562883971e-06,
+      "log_odds_chosen": 0.045434046536684036,
+      "log_odds_ratio": -0.72969651222229,
+      "logits/chosen": -2.8840372562408447,
+      "logits/rejected": -2.892764091491699,
+      "logps/chosen": -0.7538484930992126,
+      "logps/rejected": -0.7586053013801575,
+      "loss": 0.5889,
+      "nll_loss": 0.4635290503501892,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": -0.15076971054077148,
+      "rewards/margins": 0.0009513672557659447,
+      "rewards/rejected": -0.1517210602760315,
+      "step": 25
+    },
+    {
+      "epoch": 0.13,
+      "grad_norm": 4.09375,
+      "learning_rate": 4.977309719247571e-06,
+      "log_odds_chosen": 0.18096226453781128,
+      "log_odds_ratio": -0.6608779430389404,
+      "logits/chosen": -3.06424880027771,
+      "logits/rejected": -3.0758631229400635,
+      "logps/chosen": -0.6707120537757874,
+      "logps/rejected": -0.7431062459945679,
+      "loss": 0.5515,
+      "nll_loss": 0.4012710154056549,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.1341424286365509,
+      "rewards/margins": 0.014478841796517372,
+      "rewards/rejected": -0.14862126111984253,
+      "step": 30
+    },
+    {
+      "epoch": 0.15,
+      "grad_norm": 4.0625,
+      "learning_rate": 4.969132904061834e-06,
+      "log_odds_chosen": -0.1473471224308014,
+      "log_odds_ratio": -0.812741756439209,
+      "logits/chosen": -3.1558823585510254,
+      "logits/rejected": -3.139392375946045,
+      "logps/chosen": -0.684949517250061,
+      "logps/rejected": -0.6302833557128906,
+      "loss": 0.5873,
+      "nll_loss": 0.3987911641597748,
+      "rewards/accuracies": 0.4124999940395355,
+      "rewards/chosen": -0.13698987662792206,
+      "rewards/margins": -0.010933215729892254,
+      "rewards/rejected": -0.12605665624141693,
+      "step": 35
+    },
+    {
+      "epoch": 0.17,
+      "grad_norm": 4.21875,
+      "learning_rate": 4.959709246387847e-06,
+      "log_odds_chosen": 0.059796012938022614,
+      "log_odds_ratio": -0.7307142019271851,
+      "logits/chosen": -3.1404786109924316,
+      "logits/rejected": -3.1371572017669678,
+      "logps/chosen": -0.741878867149353,
+      "logps/rejected": -0.7699283361434937,
+      "loss": 0.6016,
+      "nll_loss": 0.43490996956825256,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -0.14837577939033508,
+      "rewards/margins": 0.005609904881566763,
+      "rewards/rejected": -0.15398569405078888,
+      "step": 40
+    },
+    {
+      "epoch": 0.19,
+      "grad_norm": 4.1875,
+      "learning_rate": 4.9490435049069925e-06,
+      "log_odds_chosen": 0.177604541182518,
+      "log_odds_ratio": -0.649376630783081,
+      "logits/chosen": -3.102951765060425,
+      "logits/rejected": -3.1247200965881348,
+      "logps/chosen": -0.6893263459205627,
+      "logps/rejected": -0.7772015333175659,
+      "loss": 0.5496,
+      "nll_loss": 0.36714619398117065,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.13786527514457703,
+      "rewards/margins": 0.017575044184923172,
+      "rewards/rejected": -0.1554403007030487,
+      "step": 45
+    },
+    {
+      "epoch": 0.21,
+      "grad_norm": 4.0625,
+      "learning_rate": 4.9371410655179495e-06,
+      "log_odds_chosen": 0.12347964197397232,
+      "log_odds_ratio": -0.6704575419425964,
+      "logits/chosen": -3.136650562286377,
+      "logits/rejected": -3.140460252761841,
+      "logps/chosen": -0.6930066347122192,
+      "logps/rejected": -0.7664076089859009,
+      "loss": 0.5561,
+      "nll_loss": 0.3619331419467926,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.13860133290290833,
+      "rewards/margins": 0.014680176973342896,
+      "rewards/rejected": -0.15328152477741241,
+      "step": 50
+    },
+    {
+      "epoch": 0.24,
+      "grad_norm": 3.96875,
+      "learning_rate": 4.924007938616967e-06,
+      "log_odds_chosen": 0.08203879743814468,
+      "log_odds_ratio": -0.6842275261878967,
+      "logits/chosen": -3.1045684814453125,
+      "logits/rejected": -3.0924317836761475,
+      "logps/chosen": -0.6679835915565491,
+      "logps/rejected": -0.7231112718582153,
+      "loss": 0.5663,
+      "nll_loss": 0.3903297185897827,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.133596733212471,
+      "rewards/margins": 0.011025508865714073,
+      "rewards/rejected": -0.14462223649024963,
+      "step": 55
+    },
+    {
+      "epoch": 0.26,
+      "grad_norm": 5.0,
+      "learning_rate": 4.909650756062782e-06,
+      "log_odds_chosen": 0.21534709632396698,
+      "log_odds_ratio": -0.6632246971130371,
+      "logits/chosen": -3.071376323699951,
+      "logits/rejected": -3.0663795471191406,
+      "logps/chosen": -0.7033484578132629,
+      "logps/rejected": -0.8316154479980469,
+      "loss": 0.5824,
+      "nll_loss": 0.4880998134613037,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.14066967368125916,
+      "rewards/margins": 0.025653395801782608,
+      "rewards/rejected": -0.16632308065891266,
+      "step": 60
+    },
+    {
+      "epoch": 0.28,
+      "grad_norm": 4.875,
+      "learning_rate": 4.894076767827721e-06,
+      "log_odds_chosen": 0.07506684958934784,
+      "log_odds_ratio": -0.7063411474227905,
+      "logits/chosen": -3.1346957683563232,
+      "logits/rejected": -3.1342291831970215,
+      "logps/chosen": -0.6925743222236633,
+      "logps/rejected": -0.740242600440979,
+      "loss": 0.6192,
+      "nll_loss": 0.46551480889320374,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.13851484656333923,
+      "rewards/margins": 0.009533658623695374,
+      "rewards/rejected": -0.1480485200881958,
+      "step": 65
+    },
+    {
+      "epoch": 0.3,
+      "grad_norm": 4.59375,
+      "learning_rate": 4.8772938383366615e-06,
+      "log_odds_chosen": 0.020953459665179253,
+      "log_odds_ratio": -0.7297095060348511,
+      "logits/chosen": -3.1903274059295654,
+      "logits/rejected": -3.2007622718811035,
+      "logps/chosen": -0.6915528774261475,
+      "logps/rejected": -0.717432975769043,
+      "loss": 0.5988,
+      "nll_loss": 0.43604594469070435,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.13831056654453278,
+      "rewards/margins": 0.005176021717488766,
+      "rewards/rejected": -0.14348658919334412,
+      "step": 70
+    },
+    {
+      "epoch": 0.32,
+      "grad_norm": 3.859375,
+      "learning_rate": 4.8593104424957275e-06,
+      "log_odds_chosen": 0.060929737985134125,
+      "log_odds_ratio": -0.714811384677887,
+      "logits/chosen": -3.196502923965454,
+      "logits/rejected": -3.1885693073272705,
+      "logps/chosen": -0.6992107629776001,
+      "logps/rejected": -0.7475954294204712,
+      "loss": 0.5942,
+      "nll_loss": 0.4152112603187561,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.1398421823978424,
+      "rewards/margins": 0.009676935151219368,
+      "rewards/rejected": -0.14951911568641663,
+      "step": 75
+    },
+    {
+      "epoch": 0.34,
+      "grad_norm": 4.53125,
+      "learning_rate": 4.840135661412696e-06,
+      "log_odds_chosen": 0.24067020416259766,
+      "log_odds_ratio": -0.6120481491088867,
+      "logits/chosen": -3.169623851776123,
+      "logits/rejected": -3.183824062347412,
+      "logps/chosen": -0.6291205883026123,
+      "logps/rejected": -0.7557007670402527,
+      "loss": 0.5789,
+      "nll_loss": 0.3959899842739105,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.12582412362098694,
+      "rewards/margins": 0.025316035374999046,
+      "rewards/rejected": -0.15114016830921173,
+      "step": 80
+    },
+    {
+      "epoch": 0.36,
+      "grad_norm": 4.40625,
+      "learning_rate": 4.819779177811294e-06,
+      "log_odds_chosen": 0.1330593079328537,
+      "log_odds_ratio": -0.6760435700416565,
+      "logits/chosen": -3.2041194438934326,
+      "logits/rejected": -3.1831653118133545,
+      "logps/chosen": -0.6773617267608643,
+      "logps/rejected": -0.7431901693344116,
+      "loss": 0.5506,
+      "nll_loss": 0.39580783247947693,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.1354723572731018,
+      "rewards/margins": 0.013165691867470741,
+      "rewards/rejected": -0.1486380398273468,
+      "step": 85
+    },
+    {
+      "epoch": 0.39,
+      "grad_norm": 3.890625,
+      "learning_rate": 4.7982512711416995e-06,
+      "log_odds_chosen": 0.16101466119289398,
+      "log_odds_ratio": -0.676543116569519,
+      "logits/chosen": -3.1818435192108154,
+      "logits/rejected": -3.1896145343780518,
+      "logps/chosen": -0.7060041427612305,
+      "logps/rejected": -0.8224822878837585,
+      "loss": 0.5841,
+      "nll_loss": 0.4883156418800354,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.14120084047317505,
+      "rewards/margins": 0.023295629769563675,
+      "rewards/rejected": -0.16449645161628723,
+      "step": 90
+    },
+    {
+      "epoch": 0.41,
+      "grad_norm": 4.25,
+      "learning_rate": 4.775562812389704e-06,
+      "log_odds_chosen": 0.1345851719379425,
+      "log_odds_ratio": -0.6816418766975403,
+      "logits/chosen": -3.2042739391326904,
+      "logits/rejected": -3.2034194469451904,
+      "logps/chosen": -0.6822711229324341,
+      "logps/rejected": -0.7532314658164978,
+      "loss": 0.5903,
+      "nll_loss": 0.4755779802799225,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -0.13645422458648682,
+      "rewards/margins": 0.014192057773470879,
+      "rewards/rejected": -0.15064629912376404,
+      "step": 95
+    },
+    {
+      "epoch": 0.43,
+      "grad_norm": 3.71875,
+      "learning_rate": 4.751725258587172e-06,
+      "log_odds_chosen": 0.14706887304782867,
+      "log_odds_ratio": -0.6651312112808228,
+      "logits/chosen": -3.1903748512268066,
+      "logits/rejected": -3.1974258422851562,
+      "logps/chosen": -0.6400434374809265,
+      "logps/rejected": -0.7148961424827576,
+      "loss": 0.5851,
+      "nll_loss": 0.4396067261695862,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.12800869345664978,
+      "rewards/margins": 0.014970543794333935,
+      "rewards/rejected": -0.142979234457016,
+      "step": 100
+    },
+    {
+      "epoch": 0.45,
+      "grad_norm": 3.890625,
+      "learning_rate": 4.726750647026569e-06,
+      "log_odds_chosen": 0.03515060991048813,
+      "log_odds_ratio": -0.7285342216491699,
+      "logits/chosen": -3.110316514968872,
+      "logits/rejected": -3.1469106674194336,
+      "logps/chosen": -0.7531843781471252,
+      "logps/rejected": -0.7670533061027527,
+      "loss": 0.585,
+      "nll_loss": 0.46621793508529663,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -0.15063688158988953,
+      "rewards/margins": 0.0027738078497350216,
+      "rewards/rejected": -0.1534106731414795,
+      "step": 105
+    },
+    {
+      "epoch": 0.47,
+      "grad_norm": 4.0,
+      "learning_rate": 4.700651589182461e-06,
+      "log_odds_chosen": 0.11814375966787338,
+      "log_odds_ratio": -0.6774541735649109,
+      "logits/chosen": -3.1227428913116455,
+      "logits/rejected": -3.158555269241333,
+      "logps/chosen": -0.7057517170906067,
+      "logps/rejected": -0.7718692421913147,
+      "loss": 0.5403,
+      "nll_loss": 0.37699857354164124,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.1411503553390503,
+      "rewards/margins": 0.013223511166870594,
+      "rewards/rejected": -0.15437385439872742,
+      "step": 110
+    },
+    {
+      "epoch": 0.49,
+      "grad_norm": 4.0625,
+      "learning_rate": 4.6734412643430795e-06,
+      "log_odds_chosen": 0.15334436297416687,
+      "log_odds_ratio": -0.6591517329216003,
+      "logits/chosen": -3.152284622192383,
+      "logits/rejected": -3.137619972229004,
+      "logps/chosen": -0.6621846556663513,
+      "logps/rejected": -0.7469123005867004,
+      "loss": 0.5819,
+      "nll_loss": 0.4604349136352539,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.13243696093559265,
+      "rewards/margins": 0.01694551296532154,
+      "rewards/rejected": -0.14938244223594666,
+      "step": 115
+    },
+    {
+      "epoch": 0.51,
+      "grad_norm": 3.6875,
+      "learning_rate": 4.64513341295515e-06,
+      "log_odds_chosen": 0.25181251764297485,
+      "log_odds_ratio": -0.6330325603485107,
+      "logits/chosen": -3.1038594245910645,
+      "logits/rejected": -3.127596139907837,
+      "logps/chosen": -0.6145869493484497,
+      "logps/rejected": -0.7382071614265442,
+      "loss": 0.5614,
+      "nll_loss": 0.4344588816165924,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.12291739135980606,
+      "rewards/margins": 0.024724040180444717,
+      "rewards/rejected": -0.14764143526554108,
+      "step": 120
+    },
+    {
+      "epoch": 0.54,
+      "grad_norm": 4.5625,
+      "learning_rate": 4.615742329685358e-06,
+      "log_odds_chosen": 0.06189741939306259,
+      "log_odds_ratio": -0.7182206511497498,
+      "logits/chosen": -3.124662399291992,
+      "logits/rejected": -3.129429340362549,
+      "logps/chosen": -0.7151820063591003,
+      "logps/rejected": -0.7391675710678101,
+      "loss": 0.5781,
+      "nll_loss": 0.42448896169662476,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.1430363953113556,
+      "rewards/margins": 0.004797111731022596,
+      "rewards/rejected": -0.14783351123332977,
+      "step": 125
+    },
+    {
+      "epoch": 0.56,
+      "grad_norm": 4.1875,
+      "learning_rate": 4.58528285620194e-06,
+      "log_odds_chosen": 0.15051795542240143,
+      "log_odds_ratio": -0.6667842864990234,
+      "logits/chosen": -3.1157212257385254,
+      "logits/rejected": -3.131880521774292,
+      "logps/chosen": -0.6726551651954651,
+      "logps/rejected": -0.7339776754379272,
+      "loss": 0.5517,
+      "nll_loss": 0.3924528658390045,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.13453105092048645,
+      "rewards/margins": 0.012264521792531013,
+      "rewards/rejected": -0.14679554104804993,
+      "step": 130
+    },
+    {
+      "epoch": 0.58,
+      "grad_norm": 3.8125,
+      "learning_rate": 4.553770373680062e-06,
+      "log_odds_chosen": 0.03864496201276779,
+      "log_odds_ratio": -0.738520085811615,
+      "logits/chosen": -3.108069896697998,
+      "logits/rejected": -3.111837148666382,
+      "logps/chosen": -0.6787298321723938,
+      "logps/rejected": -0.7142878770828247,
+      "loss": 0.5869,
+      "nll_loss": 0.36721861362457275,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.13574595749378204,
+      "rewards/margins": 0.007111614104360342,
+      "rewards/rejected": -0.14285758137702942,
+      "step": 135
+    },
+    {
+      "epoch": 0.6,
+      "grad_norm": 4.21875,
+      "learning_rate": 4.521220795034763e-06,
+      "log_odds_chosen": 0.15647068619728088,
+      "log_odds_ratio": -0.673542857170105,
+      "logits/chosen": -3.060241222381592,
+      "logits/rejected": -3.0633511543273926,
+      "logps/chosen": -0.6453939080238342,
+      "logps/rejected": -0.7274571657180786,
+      "loss": 0.5505,
+      "nll_loss": 0.41087573766708374,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.12907877564430237,
+      "rewards/margins": 0.01641263999044895,
+      "rewards/rejected": -0.14549142122268677,
+      "step": 140
+    },
+    {
+      "epoch": 0.62,
+      "grad_norm": 4.28125,
+      "learning_rate": 4.487650556885378e-06,
+      "log_odds_chosen": 0.13851861655712128,
+      "log_odds_ratio": -0.6815603971481323,
+      "logits/chosen": -3.110565185546875,
+      "logits/rejected": -3.108269691467285,
+      "logps/chosen": -0.7905935049057007,
+      "logps/rejected": -0.863776683807373,
+      "loss": 0.5402,
+      "nll_loss": 0.4115132689476013,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.15811872482299805,
+      "rewards/margins": 0.014636626467108727,
+      "rewards/rejected": -0.17275534570217133,
+      "step": 145
+    },
+    {
+      "epoch": 0.64,
+      "grad_norm": 4.03125,
+      "learning_rate": 4.453076611255507e-06,
+      "log_odds_chosen": 0.1597544103860855,
+      "log_odds_ratio": -0.6715651750564575,
+      "logits/chosen": -3.1071646213531494,
+      "logits/rejected": -3.0957367420196533,
+      "logps/chosen": -0.6362262964248657,
+      "logps/rejected": -0.7219537496566772,
+      "loss": 0.5297,
+      "nll_loss": 0.38165420293807983,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.12724526226520538,
+      "rewards/margins": 0.01714547723531723,
+      "rewards/rejected": -0.14439073204994202,
+      "step": 150
+    },
+    {
+      "epoch": 0.66,
+      "grad_norm": 4.21875,
+      "learning_rate": 4.417516417012725e-06,
+      "log_odds_chosen": -0.022897567600011826,
+      "log_odds_ratio": -0.7432743310928345,
+      "logits/chosen": -3.0967166423797607,
+      "logits/rejected": -3.0849483013153076,
+      "logps/chosen": -0.703900933265686,
+      "logps/rejected": -0.6957621574401855,
+      "loss": 0.5717,
+      "nll_loss": 0.43085795640945435,
+      "rewards/accuracies": 0.4625000059604645,
+      "rewards/chosen": -0.14078018069267273,
+      "rewards/margins": -0.0016277560498565435,
+      "rewards/rejected": -0.1391524374485016,
+      "step": 155
+    },
+    {
+      "epoch": 0.69,
+      "grad_norm": 3.90625,
+      "learning_rate": 4.380987931052347e-06,
+      "log_odds_chosen": 0.12808199226856232,
+      "log_odds_ratio": -0.683525562286377,
+      "logits/chosen": -3.105473041534424,
+      "logits/rejected": -3.1155142784118652,
+      "logps/chosen": -0.7174890637397766,
+      "logps/rejected": -0.7977532148361206,
+      "loss": 0.5653,
+      "nll_loss": 0.37563520669937134,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.14349780976772308,
+      "rewards/margins": 0.016052838414907455,
+      "rewards/rejected": -0.15955063700675964,
+      "step": 160
+    },
+    {
+      "epoch": 0.71,
+      "grad_norm": 4.28125,
+      "learning_rate": 4.343509599229697e-06,
+      "log_odds_chosen": 0.07662512362003326,
+      "log_odds_ratio": -0.7142797708511353,
+      "logits/chosen": -3.1126039028167725,
+      "logits/rejected": -3.1231586933135986,
+      "logps/chosen": -0.6783756017684937,
+      "logps/rejected": -0.7291387319564819,
+      "loss": 0.5674,
+      "nll_loss": 0.4361972212791443,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.1356751173734665,
+      "rewards/margins": 0.010152625851333141,
+      "rewards/rejected": -0.1458277702331543,
+      "step": 165
+    },
+    {
+      "epoch": 0.73,
+      "grad_norm": 3.828125,
+      "learning_rate": 4.305100347045476e-06,
+      "log_odds_chosen": 0.1829986721277237,
+      "log_odds_ratio": -0.6602972745895386,
+      "logits/chosen": -3.135678768157959,
+      "logits/rejected": -3.1565449237823486,
+      "logps/chosen": -0.6898233890533447,
+      "logps/rejected": -0.7847301363945007,
+      "loss": 0.548,
+      "nll_loss": 0.3952903151512146,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.13796468079090118,
+      "rewards/margins": 0.018981358036398888,
+      "rewards/rejected": -0.156946063041687,
+      "step": 170
+    },
+    {
+      "epoch": 0.75,
+      "grad_norm": 4.46875,
+      "learning_rate": 4.265779570088914e-06,
+      "log_odds_chosen": 0.23545940220355988,
+      "log_odds_ratio": -0.6459792852401733,
+      "logits/chosen": -3.142263174057007,
+      "logits/rejected": -3.1482534408569336,
+      "logps/chosen": -0.6683803796768188,
+      "logps/rejected": -0.8047019839286804,
+      "loss": 0.5477,
+      "nll_loss": 0.4291974604129791,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.13367608189582825,
+      "rewards/margins": 0.02726433239877224,
+      "rewards/rejected": -0.16094039380550385,
+      "step": 175
+    },
+    {
+      "epoch": 0.77,
+      "grad_norm": 4.59375,
+      "learning_rate": 4.22556712424355e-06,
+      "log_odds_chosen": 0.13412100076675415,
+      "log_odds_ratio": -0.678640604019165,
+      "logits/chosen": -3.165506601333618,
+      "logits/rejected": -3.1562082767486572,
+      "logps/chosen": -0.696603000164032,
+      "logps/rejected": -0.7526296377182007,
+      "loss": 0.5847,
+      "nll_loss": 0.383400559425354,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.13932058215141296,
+      "rewards/margins": 0.011205337010324001,
+      "rewards/rejected": -0.15052591264247894,
+      "step": 180
+    },
+    {
+      "epoch": 0.79,
+      "grad_norm": 3.71875,
+      "learning_rate": 4.184483315660565e-06,
+      "log_odds_chosen": 0.20085343718528748,
+      "log_odds_ratio": -0.6461249589920044,
+      "logits/chosen": -3.1610589027404785,
+      "logits/rejected": -3.1671133041381836,
+      "logps/chosen": -0.6673001050949097,
+      "logps/rejected": -0.7822000980377197,
+      "loss": 0.5358,
+      "nll_loss": 0.4822109341621399,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.13346004486083984,
+      "rewards/margins": 0.022979993373155594,
+      "rewards/rejected": -0.15644003450870514,
+      "step": 185
+    },
+    {
+      "epoch": 0.81,
+      "grad_norm": 3.625,
+      "learning_rate": 4.1425488905047485e-06,
+      "log_odds_chosen": 0.06839115917682648,
+      "log_odds_ratio": -0.7076748609542847,
+      "logits/chosen": -3.1488382816314697,
+      "logits/rejected": -3.1493427753448486,
+      "logps/chosen": -0.6919032335281372,
+      "logps/rejected": -0.7248092293739319,
+      "loss": 0.5477,
+      "nll_loss": 0.4083670973777771,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.13838066160678864,
+      "rewards/margins": 0.006581192370504141,
+      "rewards/rejected": -0.14496184885501862,
+      "step": 190
+    },
+    {
+      "epoch": 0.84,
+      "grad_norm": 4.15625,
+      "learning_rate": 4.099785024478276e-06,
+      "log_odds_chosen": 0.2139725238084793,
+      "log_odds_ratio": -0.628895103931427,
+      "logits/chosen": -3.124779224395752,
+      "logits/rejected": -3.1284875869750977,
+      "logps/chosen": -0.5759474635124207,
+      "logps/rejected": -0.6854532361030579,
+      "loss": 0.5441,
+      "nll_loss": 0.37735462188720703,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.11518951505422592,
+      "rewards/margins": 0.021901141852140427,
+      "rewards/rejected": -0.13709065318107605,
+      "step": 195
+    },
+    {
+      "epoch": 0.86,
+      "grad_norm": 3.78125,
+      "learning_rate": 4.056213312127573e-06,
+      "log_odds_chosen": 0.18503764271736145,
+      "log_odds_ratio": -0.6680355668067932,
+      "logits/chosen": -3.078535556793213,
+      "logits/rejected": -3.0459985733032227,
+      "logps/chosen": -0.747636079788208,
+      "logps/rejected": -0.8532911539077759,
+      "loss": 0.5496,
+      "nll_loss": 0.39417606592178345,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.14952722191810608,
+      "rewards/margins": 0.021131008863449097,
+      "rewards/rejected": -0.1706582009792328,
+      "step": 200
+    },
+    {
+      "epoch": 0.88,
+      "grad_norm": 3.625,
+      "learning_rate": 4.011855755938674e-06,
+      "log_odds_chosen": 0.01333926897495985,
+      "log_odds_ratio": -0.7361116409301758,
+      "logits/chosen": -3.0914862155914307,
+      "logits/rejected": -3.105278253555298,
+      "logps/chosen": -0.6985930800437927,
+      "logps/rejected": -0.6959635615348816,
+      "loss": 0.5328,
+      "nll_loss": 0.3898618221282959,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -0.13971860706806183,
+      "rewards/margins": -0.0005258865421637893,
+      "rewards/rejected": -0.13919273018836975,
+      "step": 205
+    },
+    {
+      "epoch": 0.9,
+      "grad_norm": 3.9375,
+      "learning_rate": 3.9667347552265945e-06,
+      "log_odds_chosen": 0.10643716901540756,
+      "log_odds_ratio": -0.6941254734992981,
+      "logits/chosen": -3.106863021850586,
+      "logits/rejected": -3.1205527782440186,
+      "logps/chosen": -0.6641048192977905,
+      "logps/rejected": -0.705162525177002,
+      "loss": 0.5271,
+      "nll_loss": 0.3007195293903351,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.1328209638595581,
+      "rewards/margins": 0.008211550302803516,
+      "rewards/rejected": -0.14103251695632935,
+      "step": 210
+    },
+    {
+      "epoch": 0.92,
+      "grad_norm": 3.703125,
+      "learning_rate": 3.920873094824305e-06,
+      "log_odds_chosen": 0.1166054978966713,
+      "log_odds_ratio": -0.6895023584365845,
+      "logits/chosen": -3.114652633666992,
+      "logits/rejected": -3.115201473236084,
+      "logps/chosen": -0.7006415724754333,
+      "logps/rejected": -0.7791968584060669,
+      "loss": 0.5836,
+      "nll_loss": 0.42902565002441406,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.14012829959392548,
+      "rewards/margins": 0.015711039304733276,
+      "rewards/rejected": -0.15583935379981995,
+      "step": 215
+    },
+    {
+      "epoch": 0.94,
+      "grad_norm": 3.84375,
+      "learning_rate": 3.874293933577034e-06,
+      "log_odds_chosen": 0.10644589364528656,
+      "log_odds_ratio": -0.6865199208259583,
+      "logits/chosen": -3.119800090789795,
+      "logits/rejected": -3.135976791381836,
+      "logps/chosen": -0.693631649017334,
+      "logps/rejected": -0.7763968706130981,
+      "loss": 0.557,
+      "nll_loss": 0.3763471245765686,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.13872632384300232,
+      "rewards/margins": 0.016553033143281937,
+      "rewards/rejected": -0.15527936816215515,
+      "step": 220
+    },
+    {
+      "epoch": 0.96,
+      "grad_norm": 3.859375,
+      "learning_rate": 3.8270207926477e-06,
+      "log_odds_chosen": 0.06535528600215912,
+      "log_odds_ratio": -0.7187795639038086,
+      "logits/chosen": -3.1649396419525146,
+      "logits/rejected": -3.150834560394287,
+      "logps/chosen": -0.6850636601448059,
+      "logps/rejected": -0.7298869490623474,
+      "loss": 0.5689,
+      "nll_loss": 0.39077669382095337,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -0.13701273500919342,
+      "rewards/margins": 0.00896465964615345,
+      "rewards/rejected": -0.14597737789154053,
+      "step": 225
+    },
+    {
+      "epoch": 0.99,
+      "grad_norm": 4.21875,
+      "learning_rate": 3.7790775436393954e-06,
+      "log_odds_chosen": 0.11255147308111191,
+      "log_odds_ratio": -0.6830418705940247,
+      "logits/chosen": -3.0758774280548096,
+      "logits/rejected": -3.0805001258850098,
+      "logps/chosen": -0.6779443621635437,
+      "logps/rejected": -0.7496141791343689,
+      "loss": 0.5525,
+      "nll_loss": 0.413738876581192,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.1355888843536377,
+      "rewards/margins": 0.014333980157971382,
+      "rewards/rejected": -0.14992286264896393,
+      "step": 230
+    },
+    {
+      "epoch": 1.01,
+      "grad_norm": 4.09375,
+      "learning_rate": 3.7304883965408944e-06,
+      "log_odds_chosen": 0.25222933292388916,
+      "log_odds_ratio": -0.6448281407356262,
+      "logits/chosen": -3.1005845069885254,
+      "logits/rejected": -3.1040127277374268,
+      "logps/chosen": -0.6791092157363892,
+      "logps/rejected": -0.7944923639297485,
+      "loss": 0.5541,
+      "nll_loss": 0.43306249380111694,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.1358218491077423,
+      "rewards/margins": 0.023076623678207397,
+      "rewards/rejected": -0.1588984727859497,
+      "step": 235
+    },
+    {
+      "epoch": 1.03,
+      "grad_norm": 4.5,
+      "learning_rate": 3.68127788750129e-06,
+      "log_odds_chosen": 0.862420380115509,
+      "log_odds_ratio": -0.4000455439090729,
+      "logits/chosen": -3.131739854812622,
+      "logits/rejected": -3.1033473014831543,
+      "logps/chosen": -0.48904451727867126,
+      "logps/rejected": -0.9010938405990601,
+      "loss": 0.4516,
+      "nll_loss": 0.37263303995132446,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": -0.09780890494585037,
+      "rewards/margins": 0.08240985870361328,
+      "rewards/rejected": -0.18021877110004425,
+      "step": 240
+    },
+    {
+      "epoch": 1.05,
+      "grad_norm": 3.875,
+      "learning_rate": 3.6314708664399378e-06,
+      "log_odds_chosen": 0.8805850744247437,
+      "log_odds_ratio": -0.39087027311325073,
+      "logits/chosen": -3.1137661933898926,
+      "logits/rejected": -3.133709669113159,
+      "logps/chosen": -0.468756765127182,
+      "logps/rejected": -0.9099340438842773,
+      "loss": 0.4385,
+      "nll_loss": 0.33198338747024536,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.09375137090682983,
+      "rewards/margins": 0.08823545277118683,
+      "rewards/rejected": -0.18198682367801666,
+      "step": 245
+    },
+    {
+      "epoch": 1.07,
+      "grad_norm": 4.09375,
+      "learning_rate": 3.581092484497941e-06,
+      "log_odds_chosen": 0.9555109739303589,
+      "log_odds_ratio": -0.3677138686180115,
+      "logits/chosen": -3.079671859741211,
+      "logits/rejected": -3.095301866531372,
+      "logps/chosen": -0.46135392785072327,
+      "logps/rejected": -0.900970458984375,
+      "loss": 0.4354,
+      "nll_loss": 0.3639451861381531,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -0.09227079153060913,
+      "rewards/margins": 0.08792328834533691,
+      "rewards/rejected": -0.18019406497478485,
+      "step": 250
+    },
+    {
+      "epoch": 1.09,
+      "grad_norm": 4.6875,
+      "learning_rate": 3.5301681813375343e-06,
+      "log_odds_chosen": 0.8514328002929688,
+      "log_odds_ratio": -0.40046748518943787,
+      "logits/chosen": -3.065979480743408,
+      "logits/rejected": -3.0509727001190186,
+      "logps/chosen": -0.5572352409362793,
+      "logps/rejected": -0.9676912426948547,
+      "loss": 0.4566,
+      "nll_loss": 0.3993757665157318,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -0.1114470362663269,
+      "rewards/margins": 0.08209122717380524,
+      "rewards/rejected": -0.19353827834129333,
+      "step": 255
+    },
+    {
+      "epoch": 1.11,
+      "grad_norm": 4.59375,
+      "learning_rate": 3.4787236722957747e-06,
+      "log_odds_chosen": 0.9488438367843628,
+      "log_odds_ratio": -0.39394739270210266,
+      "logits/chosen": -3.103703260421753,
+      "logits/rejected": -3.120194435119629,
+      "logps/chosen": -0.5076486468315125,
+      "logps/rejected": -0.9956499934196472,
+      "loss": 0.4487,
+      "nll_loss": 0.3443233370780945,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -0.10152973979711533,
+      "rewards/margins": 0.09760028123855591,
+      "rewards/rejected": -0.19913002848625183,
+      "step": 260
+    },
+    {
+      "epoch": 1.13,
+      "grad_norm": 4.53125,
+      "learning_rate": 3.4267849353990178e-06,
+      "log_odds_chosen": 1.0445611476898193,
+      "log_odds_ratio": -0.3562309145927429,
+      "logits/chosen": -3.0731451511383057,
+      "logits/rejected": -3.0952889919281006,
+      "logps/chosen": -0.5211637020111084,
+      "logps/rejected": -1.046607494354248,
+      "loss": 0.4208,
+      "nll_loss": 0.3219150900840759,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -0.10423272848129272,
+      "rewards/margins": 0.10508877038955688,
+      "rewards/rejected": -0.2093214988708496,
+      "step": 265
+    },
+    {
+      "epoch": 1.16,
+      "grad_norm": 4.46875,
+      "learning_rate": 3.3743781982447533e-06,
+      "log_odds_chosen": 1.1367815732955933,
+      "log_odds_ratio": -0.34295934438705444,
+      "logits/chosen": -3.10606050491333,
+      "logits/rejected": -3.115898609161377,
+      "logps/chosen": -0.4709681570529938,
+      "logps/rejected": -1.036333441734314,
+      "loss": 0.4494,
+      "nll_loss": 0.37718456983566284,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -0.09419362246990204,
+      "rewards/margins": 0.11307307332754135,
+      "rewards/rejected": -0.2072666883468628,
+      "step": 270
+    },
+    {
+      "epoch": 1.18,
+      "grad_norm": 4.71875,
+      "learning_rate": 3.3215299247573974e-06,
+      "log_odds_chosen": 0.904280960559845,
+      "log_odds_ratio": -0.3935766816139221,
+      "logits/chosen": -3.0998826026916504,
+      "logits/rejected": -3.12553071975708,
+      "logps/chosen": -0.5041223168373108,
+      "logps/rejected": -0.9397950172424316,
+      "loss": 0.4368,
+      "nll_loss": 0.38810276985168457,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.10082447528839111,
+      "rewards/margins": 0.08713451027870178,
+      "rewards/rejected": -0.1879589855670929,
+      "step": 275
+    },
+    {
+      "epoch": 1.2,
+      "grad_norm": 5.34375,
+      "learning_rate": 3.268266801824768e-06,
+      "log_odds_chosen": 1.1236122846603394,
+      "log_odds_ratio": -0.3298667073249817,
+      "logits/chosen": -3.0786471366882324,
+      "logits/rejected": -3.088914632797241,
+      "logps/chosen": -0.49954742193222046,
+      "logps/rejected": -1.0479991436004639,
+      "loss": 0.4035,
+      "nll_loss": 0.35881948471069336,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -0.09990949928760529,
+      "rewards/margins": 0.1096903458237648,
+      "rewards/rejected": -0.20959985256195068,
+      "step": 280
+    },
+    {
+      "epoch": 1.22,
+      "grad_norm": 4.5625,
+      "learning_rate": 3.2146157258219534e-06,
+      "log_odds_chosen": 0.9877004623413086,
+      "log_odds_ratio": -0.36029869318008423,
+      "logits/chosen": -3.104013204574585,
+      "logits/rejected": -3.1102428436279297,
+      "logps/chosen": -0.4857029318809509,
+      "logps/rejected": -0.9460722804069519,
+      "loss": 0.4266,
+      "nll_loss": 0.31352436542510986,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -0.0971405953168869,
+      "rewards/margins": 0.09207387268543243,
+      "rewards/rejected": -0.18921445310115814,
+      "step": 285
+    },
+    {
+      "epoch": 1.24,
+      "grad_norm": 4.6875,
+      "learning_rate": 3.1606037890294013e-06,
+      "log_odds_chosen": 0.9617363214492798,
+      "log_odds_ratio": -0.36377251148223877,
+      "logits/chosen": -3.1021533012390137,
+      "logits/rejected": -3.1139049530029297,
+      "logps/chosen": -0.5411451458930969,
+      "logps/rejected": -1.0121533870697021,
+      "loss": 0.4047,
+      "nll_loss": 0.3547790050506592,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -0.10822904109954834,
+      "rewards/margins": 0.09420166909694672,
+      "rewards/rejected": -0.20243068039417267,
+      "step": 290
+    },
+    {
+      "epoch": 1.26,
+      "grad_norm": 4.5625,
+      "learning_rate": 3.106258265952082e-06,
+      "log_odds_chosen": 0.9374507665634155,
+      "log_odds_ratio": -0.3822116255760193,
+      "logits/chosen": -3.1063599586486816,
+      "logits/rejected": -3.102320432662964,
+      "logps/chosen": -0.5152262449264526,
+      "logps/rejected": -0.9716517329216003,
+      "loss": 0.4226,
+      "nll_loss": 0.3514345586299896,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": -0.10304524749517441,
+      "rewards/margins": 0.0912851020693779,
+      "rewards/rejected": -0.1943303644657135,
+      "step": 295
+    },
+    {
+      "epoch": 1.28,
+      "grad_norm": 4.6875,
+      "learning_rate": 3.0516065995466336e-06,
+      "log_odds_chosen": 1.088078260421753,
+      "log_odds_ratio": -0.3645312786102295,
+      "logits/chosen": -3.108431100845337,
+      "logits/rejected": -3.1180756092071533,
+      "logps/chosen": -0.4901120066642761,
+      "logps/rejected": -1.0360512733459473,
+      "loss": 0.4289,
+      "nll_loss": 0.3283267021179199,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -0.09802242368459702,
+      "rewards/margins": 0.10918784141540527,
+      "rewards/rejected": -0.2072102576494217,
+      "step": 300
+    },
+    {
+      "epoch": 1.31,
+      "grad_norm": 4.40625,
+      "learning_rate": 2.9966763873634424e-06,
+      "log_odds_chosen": 1.2089542150497437,
+      "log_odds_ratio": -0.3031514286994934,
+      "logits/chosen": -3.1029577255249023,
+      "logits/rejected": -3.1095030307769775,
+      "logps/chosen": -0.48879045248031616,
+      "logps/rejected": -1.1196680068969727,
+      "loss": 0.4317,
+      "nll_loss": 0.3550404906272888,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -0.09775808453559875,
+      "rewards/margins": 0.12617555260658264,
+      "rewards/rejected": -0.2239336222410202,
+      "step": 305
+    },
+    {
+      "epoch": 1.33,
+      "grad_norm": 4.5,
+      "learning_rate": 2.941495367610656e-06,
+      "log_odds_chosen": 1.095005989074707,
+      "log_odds_ratio": -0.3307039439678192,
+      "logits/chosen": -3.049807071685791,
+      "logits/rejected": -3.064573049545288,
+      "logps/chosen": -0.45625075697898865,
+      "logps/rejected": -0.9708633422851562,
+      "loss": 0.4099,
+      "nll_loss": 0.32795679569244385,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.09125015884637833,
+      "rewards/margins": 0.10292251408100128,
+      "rewards/rejected": -0.194172665476799,
+      "step": 310
+    },
+    {
+      "epoch": 1.35,
+      "grad_norm": 6.03125,
+      "learning_rate": 2.8860914051471722e-06,
+      "log_odds_chosen": 0.9841387867927551,
+      "log_odds_ratio": -0.3754274845123291,
+      "logits/chosen": -3.06512713432312,
+      "logits/rejected": -3.047639846801758,
+      "logps/chosen": -0.5017204284667969,
+      "logps/rejected": -0.9973098635673523,
+      "loss": 0.4311,
+      "nll_loss": 0.36222928762435913,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": -0.10034408420324326,
+      "rewards/margins": 0.09911789745092392,
+      "rewards/rejected": -0.19946198165416718,
+      "step": 315
+    },
+    {
+      "epoch": 1.37,
+      "grad_norm": 4.375,
+      "learning_rate": 2.830492477411671e-06,
+      "log_odds_chosen": 1.045000433921814,
+      "log_odds_ratio": -0.37661224603652954,
+      "logits/chosen": -3.064300537109375,
+      "logits/rejected": -3.076216697692871,
+      "logps/chosen": -0.5427008867263794,
+      "logps/rejected": -1.0882201194763184,
+      "loss": 0.4399,
+      "nll_loss": 0.3907185196876526,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -0.10854017734527588,
+      "rewards/margins": 0.10910383611917496,
+      "rewards/rejected": -0.21764402091503143,
+      "step": 320
+    },
+    {
+      "epoch": 1.39,
+      "grad_norm": 4.96875,
+      "learning_rate": 2.774726660294799e-06,
+      "log_odds_chosen": 1.176218032836914,
+      "log_odds_ratio": -0.32928329706192017,
+      "logits/chosen": -3.094344139099121,
+      "logits/rejected": -3.099308967590332,
+      "logps/chosen": -0.5172286033630371,
+      "logps/rejected": -1.1497700214385986,
+      "loss": 0.4094,
+      "nll_loss": 0.30435997247695923,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -0.10344573110342026,
+      "rewards/margins": 0.12650831043720245,
+      "rewards/rejected": -0.22995400428771973,
+      "step": 325
+    },
+    {
+      "epoch": 1.41,
+      "grad_norm": 11.0,
+      "learning_rate": 2.7188221139616303e-06,
+      "log_odds_chosen": 1.014714002609253,
+      "log_odds_ratio": -0.3674837648868561,
+      "logits/chosen": -3.068471670150757,
+      "logits/rejected": -3.0866987705230713,
+      "logps/chosen": -0.48677653074264526,
+      "logps/rejected": -0.9665210843086243,
+      "loss": 0.4141,
+      "nll_loss": 0.36226484179496765,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": -0.09735531359910965,
+      "rewards/margins": 0.09594890475273132,
+      "rewards/rejected": -0.19330421090126038,
+      "step": 330
+    },
+    {
+      "epoch": 1.43,
+      "grad_norm": 4.875,
+      "learning_rate": 2.662807068631585e-06,
+      "log_odds_chosen": 1.0920135974884033,
+      "log_odds_ratio": -0.36240848898887634,
+      "logits/chosen": -3.066415309906006,
+      "logits/rejected": -3.0779404640197754,
+      "logps/chosen": -0.5206084251403809,
+      "logps/rejected": -1.0640580654144287,
+      "loss": 0.4196,
+      "nll_loss": 0.34643101692199707,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -0.10412168502807617,
+      "rewards/margins": 0.10868990421295166,
+      "rewards/rejected": -0.21281161904335022,
+      "step": 335
+    },
+    {
+      "epoch": 1.46,
+      "grad_norm": 4.5,
+      "learning_rate": 2.606709810322957e-06,
+      "log_odds_chosen": 1.0554051399230957,
+      "log_odds_ratio": -0.3538056015968323,
+      "logits/chosen": -3.086264133453369,
+      "logits/rejected": -3.0979015827178955,
+      "logps/chosen": -0.4939126968383789,
+      "logps/rejected": -1.0222851037979126,
+      "loss": 0.4479,
+      "nll_loss": 0.34655773639678955,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -0.09878254681825638,
+      "rewards/margins": 0.10567446053028107,
+      "rewards/rejected": -0.20445701479911804,
+      "step": 340
+    },
+    {
+      "epoch": 1.48,
+      "grad_norm": 4.5,
+      "learning_rate": 2.550558666569279e-06,
+      "log_odds_chosen": 1.0255191326141357,
+      "log_odds_ratio": -0.3848777115345001,
+      "logits/chosen": -3.0852413177490234,
+      "logits/rejected": -3.0870513916015625,
+      "logps/chosen": -0.49635806679725647,
+      "logps/rejected": -0.99615478515625,
+      "loss": 0.4092,
+      "nll_loss": 0.3158537745475769,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": -0.09927161037921906,
+      "rewards/margins": 0.0999593436717987,
+      "rewards/rejected": -0.19923093914985657,
+      "step": 345
+    },
+    {
+      "epoch": 1.5,
+      "grad_norm": 5.3125,
+      "learning_rate": 2.4943819921147125e-06,
+      "log_odds_chosen": 1.0540813207626343,
+      "log_odds_ratio": -0.3640359342098236,
+      "logits/chosen": -3.0244834423065186,
+      "logits/rejected": -3.0654783248901367,
+      "logps/chosen": -0.5098503828048706,
+      "logps/rejected": -1.0328686237335205,
+      "loss": 0.4198,
+      "nll_loss": 0.38225817680358887,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -0.10197006165981293,
+      "rewards/margins": 0.10460364818572998,
+      "rewards/rejected": -0.2065737247467041,
+      "step": 350
+    },
+    {
+      "epoch": 1.52,
+      "grad_norm": 4.53125,
+      "learning_rate": 2.438208154595703e-06,
+      "log_odds_chosen": 1.0023833513259888,
+      "log_odds_ratio": -0.3944670557975769,
+      "logits/chosen": -3.053039312362671,
+      "logits/rejected": -3.0673396587371826,
+      "logps/chosen": -0.5665103197097778,
+      "logps/rejected": -1.0646257400512695,
+      "loss": 0.4357,
+      "nll_loss": 0.39022982120513916,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -0.1133020743727684,
+      "rewards/margins": 0.09962306171655655,
+      "rewards/rejected": -0.21292512118816376,
+      "step": 355
+    },
+    {
+      "epoch": 1.54,
+      "grad_norm": 4.40625,
+      "learning_rate": 2.3820655202161237e-06,
+      "log_odds_chosen": 0.9806038737297058,
+      "log_odds_ratio": -0.3737329840660095,
+      "logits/chosen": -3.0929837226867676,
+      "logits/rejected": -3.113180637359619,
+      "logps/chosen": -0.5550697445869446,
+      "logps/rejected": -1.0932964086532593,
+      "loss": 0.3961,
+      "nll_loss": 0.35079020261764526,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": -0.11101396381855011,
+      "rewards/margins": 0.10764533281326294,
+      "rewards/rejected": -0.21865928173065186,
+      "step": 360
+    },
+    {
+      "epoch": 1.56,
+      "grad_norm": 4.625,
+      "learning_rate": 2.3259824394231427e-06,
+      "log_odds_chosen": 1.031070351600647,
+      "log_odds_ratio": -0.36041268706321716,
+      "logits/chosen": -3.0083768367767334,
+      "logits/rejected": -3.0171380043029785,
+      "logps/chosen": -0.46207037568092346,
+      "logps/rejected": -0.9668736457824707,
+      "loss": 0.4175,
+      "nll_loss": 0.3523089289665222,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -0.09241408109664917,
+      "rewards/margins": 0.10096064954996109,
+      "rewards/rejected": -0.19337472319602966,
+      "step": 365
+    },
+    {
+      "epoch": 1.58,
+      "grad_norm": 4.75,
+      "learning_rate": 2.2699872325910458e-06,
+      "log_odds_chosen": 1.117046594619751,
+      "log_odds_ratio": -0.37522634863853455,
+      "logits/chosen": -3.033770799636841,
+      "logits/rejected": -3.018136501312256,
+      "logps/chosen": -0.5326264500617981,
+      "logps/rejected": -1.1620831489562988,
+      "loss": 0.3983,
+      "nll_loss": 0.3253507912158966,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -0.10652528703212738,
+      "rewards/margins": 0.12589131295681,
+      "rewards/rejected": -0.23241659998893738,
+      "step": 370
+    },
+    {
+      "epoch": 1.61,
+      "grad_norm": 5.125,
+      "learning_rate": 2.214108175720246e-06,
+      "log_odds_chosen": 0.949042022228241,
+      "log_odds_ratio": -0.37258926033973694,
+      "logits/chosen": -3.006155490875244,
+      "logits/rejected": -3.0606319904327393,
+      "logps/chosen": -0.4789462685585022,
+      "logps/rejected": -0.9343479871749878,
+      "loss": 0.4191,
+      "nll_loss": 0.3158559799194336,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": -0.09578924626111984,
+      "rewards/margins": 0.09108035266399384,
+      "rewards/rejected": -0.18686962127685547,
+      "step": 375
+    },
+    {
+      "epoch": 1.63,
+      "grad_norm": 5.03125,
+      "learning_rate": 2.158373486158703e-06,
+      "log_odds_chosen": 0.9779850244522095,
+      "log_odds_ratio": -0.3736100196838379,
+      "logits/chosen": -3.0543103218078613,
+      "logits/rejected": -3.0758256912231445,
+      "logps/chosen": -0.46442002058029175,
+      "logps/rejected": -0.9138116836547852,
+      "loss": 0.4153,
+      "nll_loss": 0.33239707350730896,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": -0.09288401901721954,
+      "rewards/margins": 0.08987830579280853,
+      "rewards/rejected": -0.18276233971118927,
+      "step": 380
+    },
+    {
+      "epoch": 1.65,
+      "grad_norm": 4.90625,
+      "learning_rate": 2.1028113083529543e-06,
+      "log_odds_chosen": 1.1226699352264404,
+      "log_odds_ratio": -0.34678876399993896,
+      "logits/chosen": -3.077775478363037,
+      "logits/rejected": -3.0885062217712402,
+      "logps/chosen": -0.4337676465511322,
+      "logps/rejected": -0.9120414853096008,
+      "loss": 0.4173,
+      "nll_loss": 0.3161032497882843,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": -0.08675353974103928,
+      "rewards/margins": 0.09565478563308716,
+      "rewards/rejected": -0.18240833282470703,
+      "step": 385
+    },
+    {
+      "epoch": 1.67,
+      "grad_norm": 4.78125,
+      "learning_rate": 2.0474496996359676e-06,
+      "log_odds_chosen": 1.1200788021087646,
+      "log_odds_ratio": -0.3556548058986664,
+      "logits/chosen": -3.057023525238037,
+      "logits/rejected": -3.0625433921813965,
+      "logps/chosen": -0.47518497705459595,
+      "logps/rejected": -1.0216314792633057,
+      "loss": 0.4278,
+      "nll_loss": 0.35161006450653076,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -0.09503699094057083,
+      "rewards/margins": 0.10928928852081299,
+      "rewards/rejected": -0.20432627201080322,
+      "step": 390
+    },
+    {
+      "epoch": 1.69,
+      "grad_norm": 4.8125,
+      "learning_rate": 1.992316616058974e-06,
+      "log_odds_chosen": 0.982783317565918,
+      "log_odds_ratio": -0.3863145709037781,
+      "logits/chosen": -3.0365030765533447,
+      "logits/rejected": -3.0551135540008545,
+      "logps/chosen": -0.5143071413040161,
+      "logps/rejected": -0.9792839288711548,
+      "loss": 0.4238,
+      "nll_loss": 0.32200440764427185,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": -0.1028614267706871,
+      "rewards/margins": 0.09299537539482117,
+      "rewards/rejected": -0.19585679471492767,
+      "step": 395
+    },
+    {
+      "epoch": 1.71,
+      "grad_norm": 4.65625,
+      "learning_rate": 1.937439898274455e-06,
+      "log_odds_chosen": 1.0216920375823975,
+      "log_odds_ratio": -0.36191970109939575,
+      "logits/chosen": -3.094931125640869,
+      "logits/rejected": -3.1032814979553223,
+      "logps/chosen": -0.4898260235786438,
+      "logps/rejected": -0.9827303886413574,
+      "loss": 0.4067,
+      "nll_loss": 0.33159953355789185,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -0.09796520322561264,
+      "rewards/margins": 0.09858085960149765,
+      "rewards/rejected": -0.1965460628271103,
+      "step": 400
+    },
+    {
+      "epoch": 1.73,
+      "grad_norm": 5.4375,
+      "learning_rate": 1.882847257477398e-06,
+      "log_odds_chosen": 0.9538644552230835,
+      "log_odds_ratio": -0.3876429796218872,
+      "logits/chosen": -3.086061954498291,
+      "logits/rejected": -3.0957791805267334,
+      "logps/chosen": -0.5137572288513184,
+      "logps/rejected": -0.9913498759269714,
+      "loss": 0.4317,
+      "nll_loss": 0.34138768911361694,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.10275144875049591,
+      "rewards/margins": 0.09551852941513062,
+      "rewards/rejected": -0.19826997816562653,
+      "step": 405
+    },
+    {
+      "epoch": 1.76,
+      "grad_norm": 4.25,
+      "learning_rate": 1.8285662614119185e-06,
+      "log_odds_chosen": 1.1262160539627075,
+      "log_odds_ratio": -0.36318790912628174,
+      "logits/chosen": -3.102365016937256,
+      "logits/rejected": -3.0945730209350586,
+      "logps/chosen": -0.5087975859642029,
+      "logps/rejected": -1.0988045930862427,
+      "loss": 0.3526,
+      "nll_loss": 0.2773934006690979,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": -0.10175951570272446,
+      "rewards/margins": 0.11800140142440796,
+      "rewards/rejected": -0.2197609394788742,
+      "step": 410
+    },
+    {
+      "epoch": 1.78,
+      "grad_norm": 4.6875,
+      "learning_rate": 1.7746243204503342e-06,
+      "log_odds_chosen": 1.0629944801330566,
+      "log_odds_ratio": -0.35559511184692383,
+      "logits/chosen": -3.082392692565918,
+      "logits/rejected": -3.081556558609009,
+      "logps/chosen": -0.5505409240722656,
+      "logps/rejected": -1.1507011651992798,
+      "loss": 0.4123,
+      "nll_loss": 0.35343068838119507,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -0.11010818183422089,
+      "rewards/margins": 0.12003204971551895,
+      "rewards/rejected": -0.23014025390148163,
+      "step": 415
+    },
+    {
+      "epoch": 1.8,
+      "grad_norm": 4.96875,
+      "learning_rate": 1.7210486737516947e-06,
+      "log_odds_chosen": 1.1333677768707275,
+      "log_odds_ratio": -0.3568989932537079,
+      "logits/chosen": -3.0343739986419678,
+      "logits/rejected": -3.0451571941375732,
+      "logps/chosen": -0.5825051069259644,
+      "logps/rejected": -1.2207900285720825,
+      "loss": 0.4581,
+      "nll_loss": 0.43837055563926697,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -0.11650101095438004,
+      "rewards/margins": 0.12765701115131378,
+      "rewards/rejected": -0.24415802955627441,
+      "step": 420
+    },
+    {
+      "epoch": 1.82,
+      "grad_norm": 4.75,
+      "learning_rate": 1.6678663755067765e-06,
+      "log_odds_chosen": 1.1494513750076294,
+      "log_odds_ratio": -0.34156522154808044,
+      "logits/chosen": -3.067613124847412,
+      "logits/rejected": -3.0813510417938232,
+      "logps/chosen": -0.5272513628005981,
+      "logps/rejected": -1.1254432201385498,
+      "loss": 0.4457,
+      "nll_loss": 0.3762899935245514,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": -0.10545027256011963,
+      "rewards/margins": 0.1196383684873581,
+      "rewards/rejected": -0.22508864104747772,
+      "step": 425
+    },
+    {
+      "epoch": 1.84,
+      "grad_norm": 5.3125,
+      "learning_rate": 1.6151042812764798e-06,
+      "log_odds_chosen": 1.0926806926727295,
+      "log_odds_ratio": -0.35124513506889343,
+      "logits/chosen": -3.1003775596618652,
+      "logits/rejected": -3.0869221687316895,
+      "logps/chosen": -0.5167126059532166,
+      "logps/rejected": -1.0946365594863892,
+      "loss": 0.4301,
+      "nll_loss": 0.3912351429462433,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -0.10334251821041107,
+      "rewards/margins": 0.11558476835489273,
+      "rewards/rejected": -0.2189272940158844,
+      "step": 430
+    },
+    {
+      "epoch": 1.86,
+      "grad_norm": 5.40625,
+      "learning_rate": 1.5627890344305256e-06,
+      "log_odds_chosen": 1.0168402194976807,
+      "log_odds_ratio": -0.37472400069236755,
+      "logits/chosen": -3.073152542114258,
+      "logits/rejected": -3.062530279159546,
+      "logps/chosen": -0.4963172376155853,
+      "logps/rejected": -0.9966226816177368,
+      "loss": 0.4033,
+      "nll_loss": 0.3146648108959198,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -0.09926344454288483,
+      "rewards/margins": 0.10006110370159149,
+      "rewards/rejected": -0.1993245631456375,
+      "step": 435
+    },
+    {
+      "epoch": 1.88,
+      "grad_norm": 4.375,
+      "learning_rate": 1.5109470526933083e-06,
+      "log_odds_chosen": 1.3515523672103882,
+      "log_odds_ratio": -0.2944795489311218,
+      "logits/chosen": -3.0687355995178223,
+      "logits/rejected": -3.062213897705078,
+      "logps/chosen": -0.45598679780960083,
+      "logps/rejected": -1.140779733657837,
+      "loss": 0.4087,
+      "nll_loss": 0.32488110661506653,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -0.09119735658168793,
+      "rewards/margins": 0.13695858418941498,
+      "rewards/rejected": -0.2281559407711029,
+      "step": 440
+    },
+    {
+      "epoch": 1.91,
+      "grad_norm": 4.75,
+      "learning_rate": 1.4596045148036878e-06,
+      "log_odds_chosen": 1.054136037826538,
+      "log_odds_ratio": -0.3551008701324463,
+      "logits/chosen": -3.0608222484588623,
+      "logits/rejected": -3.068776845932007,
+      "logps/chosen": -0.4831467270851135,
+      "logps/rejected": -1.002421259880066,
+      "loss": 0.4281,
+      "nll_loss": 0.39750441908836365,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -0.09662934392690659,
+      "rewards/margins": 0.10385493189096451,
+      "rewards/rejected": -0.2004842460155487,
+      "step": 445
+    },
+    {
+      "epoch": 1.93,
+      "grad_norm": 4.9375,
+      "learning_rate": 1.4087873472954638e-06,
+      "log_odds_chosen": 1.0801405906677246,
+      "log_odds_ratio": -0.3470378816127777,
+      "logits/chosen": -3.1068406105041504,
+      "logits/rejected": -3.1202595233917236,
+      "logps/chosen": -0.4985682964324951,
+      "logps/rejected": -1.0175180435180664,
+      "loss": 0.3913,
+      "nll_loss": 0.3334752023220062,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -0.09971366077661514,
+      "rewards/margins": 0.10378994792699814,
+      "rewards/rejected": -0.20350360870361328,
+      "step": 450
+    },
+    {
+      "epoch": 1.95,
+      "grad_norm": 4.53125,
+      "learning_rate": 1.3585212114052104e-06,
+      "log_odds_chosen": 0.9953893423080444,
+      "log_odds_ratio": -0.37009868025779724,
+      "logits/chosen": -3.096047878265381,
+      "logits/rejected": -3.1102185249328613,
+      "logps/chosen": -0.5521574020385742,
+      "logps/rejected": -1.0702391862869263,
+      "loss": 0.4248,
+      "nll_loss": 0.3823598027229309,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -0.1104314923286438,
+      "rewards/margins": 0.10361631959676743,
+      "rewards/rejected": -0.21404781937599182,
+      "step": 455
+    },
+    {
+      "epoch": 1.97,
+      "grad_norm": 4.90625,
+      "learning_rate": 1.308831490114072e-06,
+      "log_odds_chosen": 1.0446946620941162,
+      "log_odds_ratio": -0.3528267443180084,
+      "logits/chosen": -3.1083786487579346,
+      "logits/rejected": -3.115349531173706,
+      "logps/chosen": -0.4900852143764496,
+      "logps/rejected": -1.0257424116134644,
+      "loss": 0.4454,
+      "nll_loss": 0.30312374234199524,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": -0.09801704436540604,
+      "rewards/margins": 0.10713144391775131,
+      "rewards/rejected": -0.20514848828315735,
+      "step": 460
+    },
+    {
+      "epoch": 1.99,
+      "grad_norm": 4.71875,
+      "learning_rate": 1.2597432753300753e-06,
+      "log_odds_chosen": 1.0419663190841675,
+      "log_odds_ratio": -0.3548406660556793,
+      "logits/chosen": -3.0856945514678955,
+      "logits/rejected": -3.095820188522339,
+      "logps/chosen": -0.5246998071670532,
+      "logps/rejected": -1.0223771333694458,
+      "loss": 0.4401,
+      "nll_loss": 0.44383564591407776,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": -0.10493995994329453,
+      "rewards/margins": 0.09953545033931732,
+      "rewards/rejected": -0.20447543263435364,
+      "step": 465
+    },
+    {
+      "epoch": 2.01,
+      "grad_norm": 5.1875,
+      "learning_rate": 1.211281355217424e-06,
+      "log_odds_chosen": 1.294676423072815,
+      "log_odds_ratio": -0.3114815652370453,
+      "logits/chosen": -3.0750784873962402,
+      "logits/rejected": -3.051118850708008,
+      "logps/chosen": -0.41240859031677246,
+      "logps/rejected": -1.030005693435669,
+      "loss": 0.3968,
+      "nll_loss": 0.33392661809921265,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -0.08248171955347061,
+      "rewards/margins": 0.1235194206237793,
+      "rewards/rejected": -0.2060011327266693,
+      "step": 470
+    },
+    {
+      "epoch": 2.03,
+      "grad_norm": 4.1875,
+      "learning_rate": 1.1634702016791722e-06,
+      "log_odds_chosen": 1.320809245109558,
+      "log_odds_ratio": -0.3077535033226013,
+      "logits/chosen": -3.092358350753784,
+      "logits/rejected": -3.0973782539367676,
+      "logps/chosen": -0.47875672578811646,
+      "logps/rejected": -1.1592457294464111,
+      "loss": 0.3882,
+      "nll_loss": 0.3666633367538452,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -0.09575134515762329,
+      "rewards/margins": 0.13609783351421356,
+      "rewards/rejected": -0.23184919357299805,
+      "step": 475
+    },
+    {
+      "epoch": 2.06,
+      "grad_norm": 4.75,
+      "learning_rate": 1.116333957999608e-06,
+      "log_odds_chosen": 1.5810959339141846,
+      "log_odds_ratio": -0.253048837184906,
+      "logits/chosen": -3.034074068069458,
+      "logits/rejected": -3.0341789722442627,
+      "logps/chosen": -0.40957292914390564,
+      "logps/rejected": -1.1751987934112549,
+      "loss": 0.3951,
+      "nll_loss": 0.3588045537471771,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.08191458880901337,
+      "rewards/margins": 0.15312522649765015,
+      "rewards/rejected": -0.23503978550434113,
+      "step": 480
+    },
+    {
+      "epoch": 2.08,
+      "grad_norm": 4.5,
+      "learning_rate": 1.0698964266525706e-06,
+      "log_odds_chosen": 1.760838508605957,
+      "log_odds_ratio": -0.23076216876506805,
+      "logits/chosen": -3.011136531829834,
+      "logits/rejected": -3.034350872039795,
+      "logps/chosen": -0.3941037952899933,
+      "logps/rejected": -1.2701072692871094,
+      "loss": 0.3596,
+      "nll_loss": 0.3343369960784912,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.07882075756788254,
+      "rewards/margins": 0.17520073056221008,
+      "rewards/rejected": -0.25402146577835083,
+      "step": 485
+    },
+    {
+      "epoch": 2.1,
+      "grad_norm": 4.5625,
+      "learning_rate": 1.024181057281875e-06,
+      "log_odds_chosen": 1.696458101272583,
+      "log_odds_ratio": -0.22928357124328613,
+      "logits/chosen": -3.036642551422119,
+      "logits/rejected": -3.054276943206787,
+      "logps/chosen": -0.4274802803993225,
+      "logps/rejected": -1.2700543403625488,
+      "loss": 0.3901,
+      "nll_loss": 0.3817359507083893,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -0.08549605309963226,
+      "rewards/margins": 0.16851481795310974,
+      "rewards/rejected": -0.2540109157562256,
+      "step": 490
+    },
+    {
+      "epoch": 2.12,
+      "grad_norm": 4.09375,
+      "learning_rate": 9.792109348599036e-07,
+      "log_odds_chosen": 1.7724546194076538,
+      "log_odds_ratio": -0.22515392303466797,
+      "logits/chosen": -3.0674076080322266,
+      "logits/rejected": -3.0711779594421387,
+      "logps/chosen": -0.3904511034488678,
+      "logps/rejected": -1.2670060396194458,
+      "loss": 0.3537,
+      "nll_loss": 0.31840625405311584,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.07809022068977356,
+      "rewards/margins": 0.17531099915504456,
+      "rewards/rejected": -0.2534012198448181,
+      "step": 495
+    },
+    {
+      "epoch": 2.14,
+      "grad_norm": 4.0625,
+      "learning_rate": 9.350087680303441e-07,
+      "log_odds_chosen": 1.543501615524292,
+      "log_odds_ratio": -0.27717000246047974,
+      "logits/chosen": -3.0165324211120605,
+      "logits/rejected": -3.0413506031036377,
+      "logps/chosen": -0.4150218367576599,
+      "logps/rejected": -1.1760425567626953,
+      "loss": 0.347,
+      "nll_loss": 0.30745163559913635,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -0.08300437033176422,
+      "rewards/margins": 0.15220412611961365,
+      "rewards/rejected": -0.23520851135253906,
+      "step": 500
+    },
+    {
+      "epoch": 2.16,
+      "grad_norm": 4.28125,
+      "learning_rate": 8.915968776409673e-07,
+      "log_odds_chosen": 1.6620781421661377,
+      "log_odds_ratio": -0.2480989396572113,
+      "logits/chosen": -2.983677387237549,
+      "logits/rejected": -3.0116188526153564,
+      "logps/chosen": -0.41676464676856995,
+      "logps/rejected": -1.2172657251358032,
+      "loss": 0.383,
+      "nll_loss": 0.37900644540786743,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -0.08335293084383011,
+      "rewards/margins": 0.16010019183158875,
+      "rewards/rejected": -0.24345311522483826,
+      "step": 505
+    },
+    {
+      "epoch": 2.18,
+      "grad_norm": 4.09375,
+      "learning_rate": 8.48997185472226e-07,
+      "log_odds_chosen": 1.6592066287994385,
+      "log_odds_ratio": -0.24125425517559052,
+      "logits/chosen": -3.0089008808135986,
+      "logits/rejected": -3.0470426082611084,
+      "logps/chosen": -0.3994481563568115,
+      "logps/rejected": -1.2132692337036133,
+      "loss": 0.3553,
+      "nll_loss": 0.310027539730072,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -0.07988963276147842,
+      "rewards/margins": 0.16276419162750244,
+      "rewards/rejected": -0.24265384674072266,
+      "step": 510
+    },
+    {
+      "epoch": 2.21,
+      "grad_norm": 4.3125,
+      "learning_rate": 8.072312031673757e-07,
+      "log_odds_chosen": 1.6442210674285889,
+      "log_odds_ratio": -0.25921303033828735,
+      "logits/chosen": -3.0381431579589844,
+      "logits/rejected": -3.039233446121216,
+      "logps/chosen": -0.4131619334220886,
+      "logps/rejected": -1.2261427640914917,
+      "loss": 0.3536,
+      "nll_loss": 0.26558876037597656,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -0.0826324000954628,
+      "rewards/margins": 0.16259616613388062,
+      "rewards/rejected": -0.245228573679924,
+      "step": 515
+    },
+    {
+      "epoch": 2.23,
+      "grad_norm": 4.15625,
+      "learning_rate": 7.663200213697012e-07,
+      "log_odds_chosen": 1.571980595588684,
+      "log_odds_ratio": -0.25617414712905884,
+      "logits/chosen": -3.0464415550231934,
+      "logits/rejected": -3.056257724761963,
+      "logps/chosen": -0.45917248725891113,
+      "logps/rejected": -1.2653144598007202,
+      "loss": 0.3709,
+      "nll_loss": 0.2762446701526642,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -0.09183450043201447,
+      "rewards/margins": 0.16122838854789734,
+      "rewards/rejected": -0.253062903881073,
+      "step": 520
+    },
+    {
+      "epoch": 2.25,
+      "grad_norm": 3.734375,
+      "learning_rate": 7.26284299072334e-07,
+      "log_odds_chosen": 1.5702177286148071,
+      "log_odds_ratio": -0.25459572672843933,
+      "logits/chosen": -3.0374367237091064,
+      "logits/rejected": -3.0311105251312256,
+      "logps/chosen": -0.3985027074813843,
+      "logps/rejected": -1.1368948221206665,
+      "loss": 0.3357,
+      "nll_loss": 0.3179529309272766,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -0.0797005444765091,
+      "rewards/margins": 0.147678405046463,
+      "rewards/rejected": -0.22737888991832733,
+      "step": 525
+    },
+    {
+      "epoch": 2.27,
+      "grad_norm": 4.125,
+      "learning_rate": 6.871442531860495e-07,
+      "log_odds_chosen": 1.7363159656524658,
+      "log_odds_ratio": -0.21836872398853302,
+      "logits/chosen": -3.0608057975769043,
+      "logits/rejected": -3.075495958328247,
+      "logps/chosen": -0.4046151638031006,
+      "logps/rejected": -1.2606099843978882,
+      "loss": 0.3318,
+      "nll_loss": 0.2710905075073242,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.08092303574085236,
+      "rewards/margins": 0.1711989939212799,
+      "rewards/rejected": -0.2521219849586487,
+      "step": 530
+    },
+    {
+      "epoch": 2.29,
+      "grad_norm": 4.3125,
+      "learning_rate": 6.489196483302895e-07,
+      "log_odds_chosen": 1.8925565481185913,
+      "log_odds_ratio": -0.23906934261322021,
+      "logits/chosen": -3.055502414703369,
+      "logits/rejected": -3.0740723609924316,
+      "logps/chosen": -0.3803566098213196,
+      "logps/rejected": -1.2806613445281982,
+      "loss": 0.3988,
+      "nll_loss": 0.34115779399871826,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -0.07607132941484451,
+      "rewards/margins": 0.18006090819835663,
+      "rewards/rejected": -0.25613224506378174,
+      "step": 535
+    },
+    {
+      "epoch": 2.31,
+      "grad_norm": 5.03125,
+      "learning_rate": 6.11629786852592e-07,
+      "log_odds_chosen": 1.6516625881195068,
+      "log_odds_ratio": -0.22780461609363556,
+      "logits/chosen": -3.0414628982543945,
+      "logits/rejected": -3.0669968128204346,
+      "logps/chosen": -0.45682182908058167,
+      "logps/rejected": -1.346400260925293,
+      "loss": 0.3627,
+      "nll_loss": 0.31683245301246643,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -0.09136436879634857,
+      "rewards/margins": 0.17791566252708435,
+      "rewards/rejected": -0.2692800462245941,
+      "step": 540
+    },
+    {
+      "epoch": 2.33,
+      "grad_norm": 4.15625,
+      "learning_rate": 5.752934990814477e-07,
+      "log_odds_chosen": 1.6729217767715454,
+      "log_odds_ratio": -0.2464570254087448,
+      "logits/chosen": -3.0506749153137207,
+      "logits/rejected": -3.053323268890381,
+      "logps/chosen": -0.42784634232521057,
+      "logps/rejected": -1.2647467851638794,
+      "loss": 0.3509,
+      "nll_loss": 0.28274843096733093,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -0.08556927740573883,
+      "rewards/margins": 0.16738007962703705,
+      "rewards/rejected": -0.2529493570327759,
+      "step": 545
+    },
+    {
+      "epoch": 2.36,
+      "grad_norm": 4.40625,
+      "learning_rate": 5.399291338175158e-07,
+      "log_odds_chosen": 1.715243935585022,
+      "log_odds_ratio": -0.2278958261013031,
+      "logits/chosen": -3.029287338256836,
+      "logits/rejected": -3.0767955780029297,
+      "logps/chosen": -0.3604409694671631,
+      "logps/rejected": -1.1182682514190674,
+      "loss": 0.3488,
+      "nll_loss": 0.2922760248184204,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -0.07208819687366486,
+      "rewards/margins": 0.15156546235084534,
+      "rewards/rejected": -0.223653644323349,
+      "step": 550
+    },
+    {
+      "epoch": 2.38,
+      "grad_norm": 4.46875,
+      "learning_rate": 5.055545490679981e-07,
+      "log_odds_chosen": 1.8542802333831787,
+      "log_odds_ratio": -0.23659536242485046,
+      "logits/chosen": -3.0523695945739746,
+      "logits/rejected": -3.0597970485687256,
+      "logps/chosen": -0.45028233528137207,
+      "logps/rejected": -1.422621488571167,
+      "loss": 0.3544,
+      "nll_loss": 0.29960012435913086,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -0.09005647152662277,
+      "rewards/margins": 0.19446782767772675,
+      "rewards/rejected": -0.2845243215560913,
+      "step": 555
+    },
+    {
+      "epoch": 2.4,
+      "grad_norm": 4.5,
+      "learning_rate": 4.721871030288458e-07,
+      "log_odds_chosen": 1.7230913639068604,
+      "log_odds_ratio": -0.22683128714561462,
+      "logits/chosen": -3.0413856506347656,
+      "logits/rejected": -3.048562526702881,
+      "logps/chosen": -0.388246089220047,
+      "logps/rejected": -1.230250358581543,
+      "loss": 0.3764,
+      "nll_loss": 0.2896038293838501,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.07764922082424164,
+      "rewards/margins": 0.1684008538722992,
+      "rewards/rejected": -0.24605008959770203,
+      "step": 560
+    },
+    {
+      "epoch": 2.42,
+      "grad_norm": 4.125,
+      "learning_rate": 4.3984364531936497e-07,
+      "log_odds_chosen": 1.764896035194397,
+      "log_odds_ratio": -0.2130444496870041,
+      "logits/chosen": -2.9862923622131348,
+      "logits/rejected": -3.0289909839630127,
+      "logps/chosen": -0.431601345539093,
+      "logps/rejected": -1.3558611869812012,
+      "loss": 0.3606,
+      "nll_loss": 0.31346598267555237,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.08632026612758636,
+      "rewards/margins": 0.1848519742488861,
+      "rewards/rejected": -0.2711722254753113,
+      "step": 565
+    },
+    {
+      "epoch": 2.44,
+      "grad_norm": 4.25,
+      "learning_rate": 4.0854050847362966e-07,
+      "log_odds_chosen": 1.7146213054656982,
+      "log_odds_ratio": -0.22591781616210938,
+      "logits/chosen": -3.002671480178833,
+      "logits/rejected": -3.0256471633911133,
+      "logps/chosen": -0.42149290442466736,
+      "logps/rejected": -1.249757170677185,
+      "loss": 0.349,
+      "nll_loss": 0.28486207127571106,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.08429859578609467,
+      "rewards/margins": 0.16565285623073578,
+      "rewards/rejected": -0.24995143711566925,
+      "step": 570
+    },
+    {
+      "epoch": 2.46,
+      "grad_norm": 3.953125,
+      "learning_rate": 3.782934996930135e-07,
+      "log_odds_chosen": 1.5356372594833374,
+      "log_odds_ratio": -0.24702449142932892,
+      "logits/chosen": -3.0242128372192383,
+      "logits/rejected": -3.0081803798675537,
+      "logps/chosen": -0.4465559124946594,
+      "logps/rejected": -1.2361305952072144,
+      "loss": 0.3485,
+      "nll_loss": 0.28072112798690796,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -0.08931118249893188,
+      "rewards/margins": 0.15791496634483337,
+      "rewards/rejected": -0.24722616374492645,
+      "step": 575
+    },
+    {
+      "epoch": 2.48,
+      "grad_norm": 5.09375,
+      "learning_rate": 3.49117892864001e-07,
+      "log_odds_chosen": 1.6950727701187134,
+      "log_odds_ratio": -0.2591921091079712,
+      "logits/chosen": -3.076004981994629,
+      "logits/rejected": -3.0650579929351807,
+      "logps/chosen": -0.4506012499332428,
+      "logps/rejected": -1.3095630407333374,
+      "loss": 0.3732,
+      "nll_loss": 0.33324551582336426,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": -0.09012024849653244,
+      "rewards/margins": 0.17179235816001892,
+      "rewards/rejected": -0.26191261410713196,
+      "step": 580
+    },
+    {
+      "epoch": 2.51,
+      "grad_norm": 4.59375,
+      "learning_rate": 3.2102842084530293e-07,
+      "log_odds_chosen": 1.7578738927841187,
+      "log_odds_ratio": -0.2337111532688141,
+      "logits/chosen": -3.021003246307373,
+      "logits/rejected": -3.0427439212799072,
+      "logps/chosen": -0.4114065170288086,
+      "logps/rejected": -1.3049488067626953,
+      "loss": 0.3608,
+      "nll_loss": 0.3577117323875427,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -0.08228130638599396,
+      "rewards/margins": 0.17870847880840302,
+      "rewards/rejected": -0.26098981499671936,
+      "step": 585
+    },
+    {
+      "epoch": 2.53,
+      "grad_norm": 4.4375,
+      "learning_rate": 2.940392680281798e-07,
+      "log_odds_chosen": 1.8219060897827148,
+      "log_odds_ratio": -0.22410018742084503,
+      "logits/chosen": -3.0648932456970215,
+      "logits/rejected": -3.0744993686676025,
+      "logps/chosen": -0.43264156579971313,
+      "logps/rejected": -1.3604795932769775,
+      "loss": 0.3818,
+      "nll_loss": 0.3207879960536957,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -0.08652831614017487,
+      "rewards/margins": 0.18556760251522064,
+      "rewards/rejected": -0.2720959484577179,
+      "step": 590
+    },
+    {
+      "epoch": 2.55,
+      "grad_norm": 4.3125,
+      "learning_rate": 2.681640631737245e-07,
+      "log_odds_chosen": 1.6481034755706787,
+      "log_odds_ratio": -0.25634631514549255,
+      "logits/chosen": -2.9935100078582764,
+      "logits/rejected": -2.9783313274383545,
+      "logps/chosen": -0.4389920234680176,
+      "logps/rejected": -1.241737961769104,
+      "loss": 0.3654,
+      "nll_loss": 0.3387255072593689,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -0.08779840171337128,
+      "rewards/margins": 0.16054919362068176,
+      "rewards/rejected": -0.24834756553173065,
+      "step": 595
+    },
+    {
+      "epoch": 2.57,
+      "grad_norm": 4.90625,
+      "learning_rate": 2.4341587253072035e-07,
+      "log_odds_chosen": 1.658064842224121,
+      "log_odds_ratio": -0.23493357002735138,
+      "logits/chosen": -3.0567028522491455,
+      "logits/rejected": -3.068887233734131,
+      "logps/chosen": -0.41281405091285706,
+      "logps/rejected": -1.230850338935852,
+      "loss": 0.3731,
+      "nll_loss": 0.31240588426589966,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.08256281167268753,
+      "rewards/margins": 0.16360723972320557,
+      "rewards/rejected": -0.2461700737476349,
+      "step": 600
+    },
+    {
+      "epoch": 2.59,
+      "grad_norm": 3.8125,
+      "learning_rate": 2.1980719323755533e-07,
+      "log_odds_chosen": 1.6675655841827393,
+      "log_odds_ratio": -0.24681706726551056,
+      "logits/chosen": -3.039940595626831,
+      "logits/rejected": -3.0746922492980957,
+      "logps/chosen": -0.44124722480773926,
+      "logps/rejected": -1.2557570934295654,
+      "loss": 0.375,
+      "nll_loss": 0.31266745924949646,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -0.08824944496154785,
+      "rewards/margins": 0.16290199756622314,
+      "rewards/rejected": -0.2511514127254486,
+      "step": 605
+    },
+    {
+      "epoch": 2.61,
+      "grad_norm": 4.90625,
+      "learning_rate": 1.9734994701152004e-07,
+      "log_odds_chosen": 1.6424992084503174,
+      "log_odds_ratio": -0.25031501054763794,
+      "logits/chosen": -3.055297374725342,
+      "logits/rejected": -3.062410831451416,
+      "logps/chosen": -0.42411407828330994,
+      "logps/rejected": -1.2362747192382812,
+      "loss": 0.3487,
+      "nll_loss": 0.30518966913223267,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.08482281118631363,
+      "rewards/margins": 0.16243210434913635,
+      "rewards/rejected": -0.24725492298603058,
+      "step": 610
+    },
+    {
+      "epoch": 2.63,
+      "grad_norm": 5.0,
+      "learning_rate": 1.7605547412867574e-07,
+      "log_odds_chosen": 2.0515856742858887,
+      "log_odds_ratio": -0.209257572889328,
+      "logits/chosen": -3.0167908668518066,
+      "logits/rejected": -3.015455484390259,
+      "logps/chosen": -0.36906909942626953,
+      "logps/rejected": -1.372367262840271,
+      "loss": 0.3962,
+      "nll_loss": 0.3273320496082306,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -0.07381383329629898,
+      "rewards/margins": 0.20065966248512268,
+      "rewards/rejected": -0.27447348833084106,
+      "step": 615
+    },
+    {
+      "epoch": 2.66,
+      "grad_norm": 4.5,
+      "learning_rate": 1.5593452769733425e-07,
+      "log_odds_chosen": 1.8267831802368164,
+      "log_odds_ratio": -0.21358974277973175,
+      "logits/chosen": -3.0070767402648926,
+      "logits/rejected": -3.028520107269287,
+      "logps/chosen": -0.42965587973594666,
+      "logps/rejected": -1.3311954736709595,
+      "loss": 0.3591,
+      "nll_loss": 0.3173789978027344,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.08593117445707321,
+      "rewards/margins": 0.18030789494514465,
+      "rewards/rejected": -0.26623910665512085,
+      "step": 620
+    },
+    {
+      "epoch": 2.68,
+      "grad_norm": 4.625,
+      "learning_rate": 1.369972682280435e-07,
+      "log_odds_chosen": 1.6776520013809204,
+      "log_odds_ratio": -0.24110755324363708,
+      "logits/chosen": -3.0178725719451904,
+      "logits/rejected": -3.041224956512451,
+      "logps/chosen": -0.4312921166419983,
+      "logps/rejected": -1.2802475690841675,
+      "loss": 0.358,
+      "nll_loss": 0.2894738018512726,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.0862584188580513,
+      "rewards/margins": 0.169791117310524,
+      "rewards/rejected": -0.2560495436191559,
+      "step": 625
+    },
+    {
+      "epoch": 2.7,
+      "grad_norm": 4.15625,
+      "learning_rate": 1.1925325850281416e-07,
+      "log_odds_chosen": 1.7899662256240845,
+      "log_odds_ratio": -0.22585618495941162,
+      "logits/chosen": -3.020197868347168,
+      "logits/rejected": -3.054832935333252,
+      "logps/chosen": -0.38960325717926025,
+      "logps/rejected": -1.2320774793624878,
+      "loss": 0.3746,
+      "nll_loss": 0.31026405096054077,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -0.07792064547538757,
+      "rewards/margins": 0.16849488019943237,
+      "rewards/rejected": -0.24641552567481995,
+      "step": 630
+    },
+    {
+      "epoch": 2.72,
+      "grad_norm": 4.625,
+      "learning_rate": 1.0271145874618526e-07,
+      "log_odds_chosen": 1.7390544414520264,
+      "log_odds_ratio": -0.20484574139118195,
+      "logits/chosen": -3.0023467540740967,
+      "logits/rejected": -3.039856433868408,
+      "logps/chosen": -0.45310506224632263,
+      "logps/rejected": -1.3705298900604248,
+      "loss": 0.3542,
+      "nll_loss": 0.32287827134132385,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.09062100946903229,
+      "rewards/margins": 0.1834850013256073,
+      "rewards/rejected": -0.274105966091156,
+      "step": 635
+    },
+    {
+      "epoch": 2.74,
+      "grad_norm": 3.921875,
+      "learning_rate": 8.738022210056318e-08,
+      "log_odds_chosen": 1.708214521408081,
+      "log_odds_ratio": -0.2522994875907898,
+      "logits/chosen": -3.050048828125,
+      "logits/rejected": -3.041829824447632,
+      "logps/chosen": -0.38343745470046997,
+      "logps/rejected": -1.1687476634979248,
+      "loss": 0.3592,
+      "nll_loss": 0.3253324627876282,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.07668749988079071,
+      "rewards/margins": 0.15706202387809753,
+      "rewards/rejected": -0.23374950885772705,
+      "step": 640
+    },
+    {
+      "epoch": 2.76,
+      "grad_norm": 4.34375,
+      "learning_rate": 7.326729040812136e-08,
+      "log_odds_chosen": 1.5645431280136108,
+      "log_odds_ratio": -0.2705634534358978,
+      "logits/chosen": -3.0365240573883057,
+      "logits/rejected": -3.0504961013793945,
+      "logps/chosen": -0.4279886782169342,
+      "logps/rejected": -1.212249517440796,
+      "loss": 0.3985,
+      "nll_loss": 0.3430742025375366,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -0.08559773117303848,
+      "rewards/margins": 0.15685215592384338,
+      "rewards/rejected": -0.24244992434978485,
+      "step": 645
+    },
+    {
+      "epoch": 2.78,
+      "grad_norm": 4.71875,
+      "learning_rate": 6.037979030138697e-08,
+      "log_odds_chosen": 1.7532436847686768,
+      "log_odds_ratio": -0.23098623752593994,
+      "logits/chosen": -3.0512053966522217,
+      "logits/rejected": -3.0449166297912598,
+      "logps/chosen": -0.41148900985717773,
+      "logps/rejected": -1.261879324913025,
+      "loss": 0.3527,
+      "nll_loss": 0.32637524604797363,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.08229780942201614,
+      "rewards/margins": 0.1700780689716339,
+      "rewards/rejected": -0.25237587094306946,
+      "step": 650
+    },
+    {
+      "epoch": 2.81,
+      "grad_norm": 4.71875,
+      "learning_rate": 4.872422960449264e-08,
+      "log_odds_chosen": 1.7282205820083618,
+      "log_odds_ratio": -0.24718597531318665,
+      "logits/chosen": -3.029437303543091,
+      "logits/rejected": -3.033756732940674,
+      "logps/chosen": -0.39554673433303833,
+      "logps/rejected": -1.2203553915023804,
+      "loss": 0.3649,
+      "nll_loss": 0.2759321331977844,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -0.07910935580730438,
+      "rewards/margins": 0.1649617701768875,
+      "rewards/rejected": -0.2440710961818695,
+      "step": 655
+    },
+    {
+      "epoch": 2.83,
+      "grad_norm": 4.59375,
+      "learning_rate": 3.830649404690939e-08,
+      "log_odds_chosen": 1.5842225551605225,
+      "log_odds_ratio": -0.2688426673412323,
+      "logits/chosen": -3.002917766571045,
+      "logits/rejected": -3.0281500816345215,
+      "logps/chosen": -0.4680345058441162,
+      "logps/rejected": -1.28192937374115,
+      "loss": 0.3717,
+      "nll_loss": 0.33302080631256104,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -0.09360691159963608,
+      "rewards/margins": 0.16277894377708435,
+      "rewards/rejected": -0.25638583302497864,
+      "step": 660
+    },
+    {
+      "epoch": 2.85,
+      "grad_norm": 4.59375,
+      "learning_rate": 2.91318442913191e-08,
+      "log_odds_chosen": 1.861280083656311,
+      "log_odds_ratio": -0.22339418530464172,
+      "logits/chosen": -3.0236425399780273,
+      "logits/rejected": -3.0673043727874756,
+      "logps/chosen": -0.4108535349369049,
+      "logps/rejected": -1.3879892826080322,
+      "loss": 0.3649,
+      "nll_loss": 0.25299254059791565,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -0.08217070251703262,
+      "rewards/margins": 0.19542714953422546,
+      "rewards/rejected": -0.2775978446006775,
+      "step": 665
+    },
+    {
+      "epoch": 2.87,
+      "grad_norm": 4.5625,
+      "learning_rate": 2.120491327712776e-08,
+      "log_odds_chosen": 1.5737533569335938,
+      "log_odds_ratio": -0.28394436836242676,
+      "logits/chosen": -3.027447462081909,
+      "logits/rejected": -3.0334548950195312,
+      "logps/chosen": -0.44802603125572205,
+      "logps/rejected": -1.257824182510376,
+      "loss": 0.3932,
+      "nll_loss": 0.3312658965587616,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": -0.08960519731044769,
+      "rewards/margins": 0.16195961833000183,
+      "rewards/rejected": -0.25156480073928833,
+      "step": 670
+    },
+    {
+      "epoch": 2.89,
+      "grad_norm": 4.5625,
+      "learning_rate": 1.452970388096192e-08,
+      "log_odds_chosen": 1.8042373657226562,
+      "log_odds_ratio": -0.21902310848236084,
+      "logits/chosen": -3.0271263122558594,
+      "logits/rejected": -3.0365521907806396,
+      "logps/chosen": -0.39096662402153015,
+      "logps/rejected": -1.293257713317871,
+      "loss": 0.3928,
+      "nll_loss": 0.344696581363678,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -0.07819332182407379,
+      "rewards/margins": 0.18045823276042938,
+      "rewards/rejected": -0.2586515545845032,
+      "step": 675
+    },
+    {
+      "epoch": 2.91,
+      "grad_norm": 4.46875,
+      "learning_rate": 9.109586895329203e-09,
+      "log_odds_chosen": 1.4978595972061157,
+      "log_odds_ratio": -0.24738021194934845,
+      "logits/chosen": -3.0044121742248535,
+      "logits/rejected": -3.0400876998901367,
+      "logps/chosen": -0.3905513882637024,
+      "logps/rejected": -1.0503321886062622,
+      "loss": 0.3701,
+      "nll_loss": 0.30373144149780273,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.07811026275157928,
+      "rewards/margins": 0.13195618987083435,
+      "rewards/rejected": -0.21006643772125244,
+      "step": 680
+    },
+    {
+      "epoch": 2.93,
+      "grad_norm": 4.5,
+      "learning_rate": 4.947299326462929e-09,
+      "log_odds_chosen": 1.703033208847046,
+      "log_odds_ratio": -0.23556920886039734,
+      "logits/chosen": -3.018275737762451,
+      "logits/rejected": -3.023285388946533,
+      "logps/chosen": -0.41973185539245605,
+      "logps/rejected": -1.2890293598175049,
+      "loss": 0.3621,
+      "nll_loss": 0.34329932928085327,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -0.08394636958837509,
+      "rewards/margins": 0.17385947704315186,
+      "rewards/rejected": -0.25780585408210754,
+      "step": 685
+    },
+    {
+      "epoch": 2.96,
+      "grad_norm": 4.78125,
+      "learning_rate": 2.044943012210754e-09,
+      "log_odds_chosen": 1.9632880687713623,
+      "log_odds_ratio": -0.1982704997062683,
+      "logits/chosen": -3.0093159675598145,
+      "logits/rejected": -3.0189404487609863,
+      "logps/chosen": -0.3720591068267822,
+      "logps/rejected": -1.3600140810012817,
+      "loss": 0.3384,
+      "nll_loss": 0.26117128133773804,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.07441182434558868,
+      "rewards/margins": 0.19759100675582886,
+      "rewards/rejected": -0.27200281620025635,
+      "step": 690
+    },
+    {
+      "epoch": 2.98,
+      "grad_norm": 4.5,
+      "learning_rate": 4.0398356066673195e-10,
+      "log_odds_chosen": 1.7920351028442383,
+      "log_odds_ratio": -0.21135714650154114,
+      "logits/chosen": -3.0156197547912598,
+      "logits/rejected": -3.0277273654937744,
+      "logps/chosen": -0.41774335503578186,
+      "logps/rejected": -1.3216874599456787,
+      "loss": 0.3905,
+      "nll_loss": 0.38513460755348206,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -0.08354867249727249,
+      "rewards/margins": 0.1807888001203537,
+      "rewards/rejected": -0.2643374800682068,
+      "step": 695
+    },
+    {
+      "epoch": 2.99,
+      "step": 699,
+      "total_flos": 0.0,
+      "train_loss": 0.45327196680595605,
+      "train_runtime": 9631.2387,
+      "train_samples_per_second": 4.65,
+      "train_steps_per_second": 0.073
+    }
+  ],
+  "logging_steps": 5,
+  "max_steps": 699,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 500,
+  "total_flos": 0.0,
+  "train_batch_size": 8,
+  "trial_name": null,
+  "trial_params": null
+}