Model save

README.md

@@ -0,0 +1,79 @@
+---
+tags:
+- trl
+- dpo
+- generated_from_trainer
+model-index:
+- name: arceeai-cpt-sft-dpo-full
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# arceeai-cpt-sft-dpo-full
+
+This model was trained from scratch on the None dataset.
+It achieves the following results on the evaluation set:
+- Loss: 0.4290
+- Rewards/chosen: -2.1400
+- Rewards/rejected: -3.3730
+- Rewards/accuracies: 0.7680
+- Rewards/margins: 1.2330
+- Logps/rejected: -667.6970
+- Logps/chosen: -599.8088
+- Logits/rejected: -3.8772
+- Logits/chosen: -3.7995
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 5e-07
+- train_batch_size: 4
+- eval_batch_size: 2
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 8
+- gradient_accumulation_steps: 2
+- total_train_batch_size: 64
+- total_eval_batch_size: 16
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: cosine
+- lr_scheduler_warmup_ratio: 0.1
+- num_epochs: 1
+
+### Training results
+
+| Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
+|:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
+| 0.6051        | 0.1   | 100  | 0.5848          | -0.2214        | -0.5948          | 0.6920             | 0.3733          | -389.8707      | -407.9477    | -3.9800         | -3.8777       |
+| 0.5134        | 0.21  | 200  | 0.5025          | -1.5935        | -2.5024          | 0.7160             | 0.9089          | -580.6380      | -545.1561    | -3.9154         | -3.8176       |
+| 0.4489        | 0.31  | 300  | 0.4614          | -1.5620        | -2.6097          | 0.7760             | 1.0477          | -591.3610      | -542.0072    | -3.7594         | -3.6703       |
+| 0.4359        | 0.42  | 400  | 0.4467          | -2.0879        | -3.2160          | 0.7680             | 1.1281          | -651.9947      | -594.5918    | -3.7022         | -3.6221       |
+| 0.4271        | 0.52  | 500  | 0.4441          | -2.0549        | -3.2181          | 0.7840             | 1.1631          | -652.2027      | -591.3006    | -3.8189         | -3.7408       |
+| 0.4181        | 0.63  | 600  | 0.4366          | -1.9876        | -3.1678          | 0.7760             | 1.1802          | -647.1777      | -584.5698    | -3.7950         | -3.7170       |
+| 0.4           | 0.73  | 700  | 0.4317          | -2.1647        | -3.3521          | 0.7640             | 1.1874          | -665.6046      | -602.2762    | -3.8739         | -3.7970       |
+| 0.4123        | 0.84  | 800  | 0.4291          | -2.2039        | -3.4491          | 0.7680             | 1.2453          | -675.3075      | -606.1934    | -3.8606         | -3.7827       |
+| 0.4394        | 0.94  | 900  | 0.4292          | -2.1325        | -3.3633          | 0.7680             | 1.2308          | -666.7250      | -599.0574    | -3.8777         | -3.8001       |
+
+
+### Framework versions
+
+- Transformers 4.36.2
+- Pytorch 2.1.2+cu121
+- Datasets 2.16.1
+- Tokenizers 0.15.0

all_results.json

@@ -0,0 +1,21 @@
+{
+    "epoch": 1.0,
+    "eval_logits/chosen": -3.799543857574463,
+    "eval_logits/rejected": -3.8772289752960205,
+    "eval_logps/chosen": -599.8087768554688,
+    "eval_logps/rejected": -667.6969604492188,
+    "eval_loss": 0.42904943227767944,
+    "eval_rewards/accuracies": 0.7680000066757202,
+    "eval_rewards/chosen": -2.1400206089019775,
+    "eval_rewards/margins": 1.2330021858215332,
+    "eval_rewards/rejected": -3.37302303314209,
+    "eval_runtime": 203.6811,
+    "eval_samples": 2000,
+    "eval_samples_per_second": 9.819,
+    "eval_steps_per_second": 0.614,
+    "train_loss": 0.46914771214829687,
+    "train_runtime": 18559.523,
+    "train_samples": 61135,
+    "train_samples_per_second": 3.294,
+    "train_steps_per_second": 0.051
+}

eval_results.json

@@ -0,0 +1,16 @@
+{
+    "epoch": 1.0,
+    "eval_logits/chosen": -3.799543857574463,
+    "eval_logits/rejected": -3.8772289752960205,
+    "eval_logps/chosen": -599.8087768554688,
+    "eval_logps/rejected": -667.6969604492188,
+    "eval_loss": 0.42904943227767944,
+    "eval_rewards/accuracies": 0.7680000066757202,
+    "eval_rewards/chosen": -2.1400206089019775,
+    "eval_rewards/margins": 1.2330021858215332,
+    "eval_rewards/rejected": -3.37302303314209,
+    "eval_runtime": 203.6811,
+    "eval_samples": 2000,
+    "eval_samples_per_second": 9.819,
+    "eval_steps_per_second": 0.614
+}

generation_config.json

@@ -0,0 +1,8 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "pad_token_id": 2,
+  "transformers_version": "4.36.2",
+  "use_cache": false
+}

model-00001-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1f9e5b3dfc14747ca262e757a8b77662b763a5e7f8a581fdb47282625a2c16e4
+size 4943162336

model-00002-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fa44ecb4cf31bd9efec7fbb071c6091f5b4d4e86b7e73eeaed73d5e2ebd6c22c
+size 4999819336

model-00003-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:73e089ca44da9d856a20c4122fb53bda54d6ec1fb4333a48abc030a432a7089c
+size 4915916184

model-00004-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8c8843adb9c05d74fddc9056248d3d9f33164409c9442799792c3bb1a69a3a45
+size 1369500352

model.safetensors.index.json

@@ -0,0 +1,334 @@
+{
+  "metadata": {
+    "total_size": 16228360192
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00004-of-00004.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.32.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.32.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.32.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.32.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.32.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.32.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.32.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.32.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.32.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.33.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.33.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.33.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.33.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.33.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.34.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.norm.weight": "model-00004-of-00004.safetensors"
+  }
+}

runs/Jan19_21-42-08_ip-172-31-92-198/events.out.tfevents.1705700562.ip-172-31-92-198.84660.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:cdc67af3adc4af292d12f8dac124fccbaaf698241bf647f5965d4336e35b7e98
-size 68760
+oid sha256:1e0af0207cc0c4038f780f955c231d21c5025e1d4823d12f680d95b1b127a8b7
+size 72284

runs/Jan19_21-42-08_ip-172-31-92-198/events.out.tfevents.1705719325.ip-172-31-92-198.84660.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:59a2df9e88966461df4ff06a87bcdc38220327fae3bb8d168267dd689ff2f3dd
+size 828

train_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 1.0,
+    "train_loss": 0.46914771214829687,
+    "train_runtime": 18559.523,
+    "train_samples": 61135,
+    "train_samples_per_second": 3.294,
+    "train_steps_per_second": 0.051
+}

trainer_state.json

@@ -0,0 +1,1518 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.9994767137624281,
+  "eval_steps": 100,
+  "global_step": 955,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0,
+      "learning_rate": 5.208333333333333e-09,
+      "logits/chosen": -4.156338691711426,
+      "logits/rejected": -4.146947383880615,
+      "logps/chosen": -276.527099609375,
+      "logps/rejected": -253.10324096679688,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 1
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 5.208333333333333e-08,
+      "logits/chosen": -4.072511672973633,
+      "logits/rejected": -4.162118911743164,
+      "logps/chosen": -398.7220764160156,
+      "logps/rejected": -310.1617736816406,
+      "loss": 0.6929,
+      "rewards/accuracies": 0.4722222089767456,
+      "rewards/chosen": 0.0006214036839082837,
+      "rewards/margins": 0.0005763211520388722,
+      "rewards/rejected": 4.508249185164459e-05,
+      "step": 10
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 1.0416666666666667e-07,
+      "logits/chosen": -4.09468412399292,
+      "logits/rejected": -4.120311737060547,
+      "logps/chosen": -304.5065612792969,
+      "logps/rejected": -286.7430725097656,
+      "loss": 0.693,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": 0.0015097814612090588,
+      "rewards/margins": -0.0002035536599578336,
+      "rewards/rejected": 0.001713335164822638,
+      "step": 20
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 1.5624999999999999e-07,
+      "logits/chosen": -4.106563568115234,
+      "logits/rejected": -4.1498003005981445,
+      "logps/chosen": -371.30047607421875,
+      "logps/rejected": -341.87310791015625,
+      "loss": 0.6908,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.012897265143692493,
+      "rewards/margins": 0.008245532400906086,
+      "rewards/rejected": 0.0046517327427864075,
+      "step": 30
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 2.0833333333333333e-07,
+      "logits/chosen": -4.051548480987549,
+      "logits/rejected": -4.108038425445557,
+      "logps/chosen": -362.06304931640625,
+      "logps/rejected": -345.527099609375,
+      "loss": 0.6869,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.03629542142152786,
+      "rewards/margins": 0.018107738345861435,
+      "rewards/rejected": 0.018187683075666428,
+      "step": 40
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 2.604166666666667e-07,
+      "logits/chosen": -4.083974361419678,
+      "logits/rejected": -4.089003562927246,
+      "logps/chosen": -328.7674865722656,
+      "logps/rejected": -337.3554382324219,
+      "loss": 0.6775,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.07210709154605865,
+      "rewards/margins": 0.03594246506690979,
+      "rewards/rejected": 0.03616461902856827,
+      "step": 50
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 3.1249999999999997e-07,
+      "logits/chosen": -4.070071220397949,
+      "logits/rejected": -4.125251293182373,
+      "logps/chosen": -364.65863037109375,
+      "logps/rejected": -336.72119140625,
+      "loss": 0.6673,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.14206922054290771,
+      "rewards/margins": 0.05489688366651535,
+      "rewards/rejected": 0.08717232942581177,
+      "step": 60
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 3.645833333333333e-07,
+      "logits/chosen": -4.12211799621582,
+      "logits/rejected": -4.217160701751709,
+      "logps/chosen": -381.5180358886719,
+      "logps/rejected": -344.3231506347656,
+      "loss": 0.6475,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.19605371356010437,
+      "rewards/margins": 0.1255684494972229,
+      "rewards/rejected": 0.07048525661230087,
+      "step": 70
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 4.1666666666666667e-07,
+      "logits/chosen": -3.8739821910858154,
+      "logits/rejected": -3.94097900390625,
+      "logps/chosen": -425.6156311035156,
+      "logps/rejected": -385.5126037597656,
+      "loss": 0.6165,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": 0.13462719321250916,
+      "rewards/margins": 0.24346613883972168,
+      "rewards/rejected": -0.10883896052837372,
+      "step": 80
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 4.6874999999999996e-07,
+      "logits/chosen": -3.8873085975646973,
+      "logits/rejected": -3.9699835777282715,
+      "logps/chosen": -428.96026611328125,
+      "logps/rejected": -417.627197265625,
+      "loss": 0.617,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": 0.02063266560435295,
+      "rewards/margins": 0.2929636836051941,
+      "rewards/rejected": -0.27233099937438965,
+      "step": 90
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 4.999732492681437e-07,
+      "logits/chosen": -3.9544475078582764,
+      "logits/rejected": -3.9958584308624268,
+      "logps/chosen": -388.5050964355469,
+      "logps/rejected": -426.59765625,
+      "loss": 0.6051,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.17338475584983826,
+      "rewards/margins": 0.28527265787124634,
+      "rewards/rejected": -0.458657443523407,
+      "step": 100
+    },
+    {
+      "epoch": 0.1,
+      "eval_logits/chosen": -3.8776931762695312,
+      "eval_logits/rejected": -3.979984760284424,
+      "eval_logps/chosen": -407.94769287109375,
+      "eval_logps/rejected": -389.8706970214844,
+      "eval_loss": 0.5848276615142822,
+      "eval_rewards/accuracies": 0.6919999718666077,
+      "eval_rewards/chosen": -0.22141031920909882,
+      "eval_rewards/margins": 0.3733499050140381,
+      "eval_rewards/rejected": -0.5947602391242981,
+      "eval_runtime": 202.6123,
+      "eval_samples_per_second": 9.871,
+      "eval_steps_per_second": 0.617,
+      "step": 100
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 4.996723692767926e-07,
+      "logits/chosen": -4.082424640655518,
+      "logits/rejected": -4.156807899475098,
+      "logps/chosen": -365.0426025390625,
+      "logps/rejected": -362.7803649902344,
+      "loss": 0.5643,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.3058136999607086,
+      "rewards/margins": 0.49873948097229004,
+      "rewards/rejected": -0.8045531511306763,
+      "step": 110
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 4.990375746213598e-07,
+      "logits/chosen": -4.234003067016602,
+      "logits/rejected": -4.319502830505371,
+      "logps/chosen": -424.35980224609375,
+      "logps/rejected": -453.1839904785156,
+      "loss": 0.5679,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.47503072023391724,
+      "rewards/margins": 0.55732262134552,
+      "rewards/rejected": -1.0323532819747925,
+      "step": 120
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 4.980697142834314e-07,
+      "logits/chosen": -4.125961780548096,
+      "logits/rejected": -4.213648319244385,
+      "logps/chosen": -462.6040954589844,
+      "logps/rejected": -459.0274963378906,
+      "loss": 0.5456,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.616773247718811,
+      "rewards/margins": 0.5351831912994385,
+      "rewards/rejected": -1.15195631980896,
+      "step": 130
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 4.967700826904229e-07,
+      "logits/chosen": -4.07401704788208,
+      "logits/rejected": -4.130145072937012,
+      "logps/chosen": -381.79718017578125,
+      "logps/rejected": -404.578369140625,
+      "loss": 0.5092,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.7604867219924927,
+      "rewards/margins": 0.5355066657066345,
+      "rewards/rejected": -1.2959933280944824,
+      "step": 140
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 4.951404179843962e-07,
+      "logits/chosen": -3.923344850540161,
+      "logits/rejected": -3.991914749145508,
+      "logps/chosen": -435.2518005371094,
+      "logps/rejected": -472.1197204589844,
+      "loss": 0.5597,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.4754977226257324,
+      "rewards/margins": 0.5571349859237671,
+      "rewards/rejected": -1.032632827758789,
+      "step": 150
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 4.931828996974498e-07,
+      "logits/chosen": -3.7589492797851562,
+      "logits/rejected": -3.853282928466797,
+      "logps/chosen": -413.491455078125,
+      "logps/rejected": -455.9912109375,
+      "loss": 0.5265,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.14779019355773926,
+      "rewards/margins": 0.467332661151886,
+      "rewards/rejected": -0.6151228547096252,
+      "step": 160
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 4.909001458367866e-07,
+      "logits/chosen": -3.895158290863037,
+      "logits/rejected": -4.054537773132324,
+      "logps/chosen": -404.14532470703125,
+      "logps/rejected": -403.3351135253906,
+      "loss": 0.5177,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.5085878968238831,
+      "rewards/margins": 0.7046168446540833,
+      "rewards/rejected": -1.2132047414779663,
+      "step": 170
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 4.882952093833627e-07,
+      "logits/chosen": -3.9119632244110107,
+      "logits/rejected": -3.9340500831604004,
+      "logps/chosen": -415.5750427246094,
+      "logps/rejected": -543.4241333007812,
+      "loss": 0.4752,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -1.0291075706481934,
+      "rewards/margins": 0.9823731184005737,
+      "rewards/rejected": -2.0114803314208984,
+      "step": 180
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 4.853715742087946e-07,
+      "logits/chosen": -3.9852688312530518,
+      "logits/rejected": -4.081984519958496,
+      "logps/chosen": -480.1775817871094,
+      "logps/rejected": -522.3671264648438,
+      "loss": 0.4535,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -1.1266310214996338,
+      "rewards/margins": 0.9308539628982544,
+      "rewards/rejected": -2.0574851036071777,
+      "step": 190
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 4.821331504159906e-07,
+      "logits/chosen": -4.0369672775268555,
+      "logits/rejected": -4.075179576873779,
+      "logps/chosen": -509.94189453125,
+      "logps/rejected": -613.8836059570312,
+      "loss": 0.5134,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.5363746881484985,
+      "rewards/margins": 0.8696328401565552,
+      "rewards/rejected": -2.406007766723633,
+      "step": 200
+    },
+    {
+      "epoch": 0.21,
+      "eval_logits/chosen": -3.8175740242004395,
+      "eval_logits/rejected": -3.9154281616210938,
+      "eval_logps/chosen": -545.1560668945312,
+      "eval_logps/rejected": -580.6380004882812,
+      "eval_loss": 0.5025292634963989,
+      "eval_rewards/accuracies": 0.7160000205039978,
+      "eval_rewards/chosen": -1.5934933423995972,
+      "eval_rewards/margins": 0.9089404940605164,
+      "eval_rewards/rejected": -2.5024337768554688,
+      "eval_runtime": 203.658,
+      "eval_samples_per_second": 9.82,
+      "eval_steps_per_second": 0.614,
+      "step": 200
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 4.785842691097342e-07,
+      "logits/chosen": -3.9959654808044434,
+      "logits/rejected": -4.14572811126709,
+      "logps/chosen": -542.1124267578125,
+      "logps/rejected": -513.849853515625,
+      "loss": 0.5114,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -1.1273362636566162,
+      "rewards/margins": 1.0702550411224365,
+      "rewards/rejected": -2.1975910663604736,
+      "step": 210
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 4.7472967660421603e-07,
+      "logits/chosen": -3.90177583694458,
+      "logits/rejected": -3.9888691902160645,
+      "logps/chosen": -448.62091064453125,
+      "logps/rejected": -494.3854064941406,
+      "loss": 0.4834,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -0.7570297718048096,
+      "rewards/margins": 0.9269709587097168,
+      "rewards/rejected": -1.6840009689331055,
+      "step": 220
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 4.705745280752585e-07,
+      "logits/chosen": -3.9159817695617676,
+      "logits/rejected": -4.030351161956787,
+      "logps/chosen": -501.67938232421875,
+      "logps/rejected": -505.40618896484375,
+      "loss": 0.4834,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -1.1143962144851685,
+      "rewards/margins": 0.7790688276290894,
+      "rewards/rejected": -1.8934648036956787,
+      "step": 230
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 4.6612438066572555e-07,
+      "logits/chosen": -3.872812271118164,
+      "logits/rejected": -3.9638118743896484,
+      "logps/chosen": -486.70751953125,
+      "logps/rejected": -500.75518798828125,
+      "loss": 0.4416,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.2733886241912842,
+      "rewards/margins": 0.9924365282058716,
+      "rewards/rejected": -2.2658252716064453,
+      "step": 240
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 4.6138518605333664e-07,
+      "logits/chosen": -3.8824219703674316,
+      "logits/rejected": -3.9245293140411377,
+      "logps/chosen": -505.47711181640625,
+      "logps/rejected": -631.7078247070312,
+      "loss": 0.4569,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -1.439082145690918,
+      "rewards/margins": 1.2419004440307617,
+      "rewards/rejected": -2.680982828140259,
+      "step": 250
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 4.5636328249082514e-07,
+      "logits/chosen": -4.006925106048584,
+      "logits/rejected": -4.075568199157715,
+      "logps/chosen": -515.6424560546875,
+      "logps/rejected": -610.078125,
+      "loss": 0.4646,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -1.567647933959961,
+      "rewards/margins": 0.9515784382820129,
+      "rewards/rejected": -2.519226551055908,
+      "step": 260
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 4.510653863290871e-07,
+      "logits/chosen": -4.042003154754639,
+      "logits/rejected": -4.134153842926025,
+      "logps/chosen": -530.5443115234375,
+      "logps/rejected": -585.4014892578125,
+      "loss": 0.4713,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -1.408792495727539,
+      "rewards/margins": 1.231185793876648,
+      "rewards/rejected": -2.6399781703948975,
+      "step": 270
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 4.4549858303465737e-07,
+      "logits/chosen": -3.96376371383667,
+      "logits/rejected": -3.9938507080078125,
+      "logps/chosen": -508.6741638183594,
+      "logps/rejected": -597.8483276367188,
+      "loss": 0.4507,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -1.4071502685546875,
+      "rewards/margins": 1.0225669145584106,
+      "rewards/rejected": -2.4297173023223877,
+      "step": 280
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 4.396703177135261e-07,
+      "logits/chosen": -3.989154815673828,
+      "logits/rejected": -4.069916248321533,
+      "logps/chosen": -498.7647399902344,
+      "logps/rejected": -517.9674072265625,
+      "loss": 0.452,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -1.4257739782333374,
+      "rewards/margins": 0.752226710319519,
+      "rewards/rejected": -2.1780009269714355,
+      "step": 290
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 4.335883851539693e-07,
+      "logits/chosen": -3.9300217628479004,
+      "logits/rejected": -4.022156238555908,
+      "logps/chosen": -441.078369140625,
+      "logps/rejected": -534.4451904296875,
+      "loss": 0.4489,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -1.3399455547332764,
+      "rewards/margins": 1.284125566482544,
+      "rewards/rejected": -2.6240711212158203,
+      "step": 300
+    },
+    {
+      "epoch": 0.31,
+      "eval_logits/chosen": -3.670259714126587,
+      "eval_logits/rejected": -3.7593655586242676,
+      "eval_logps/chosen": -542.0072021484375,
+      "eval_logps/rejected": -591.3610229492188,
+      "eval_loss": 0.46141302585601807,
+      "eval_rewards/accuracies": 0.7760000228881836,
+      "eval_rewards/chosen": -1.5620052814483643,
+      "eval_rewards/margins": 1.0476588010787964,
+      "eval_rewards/rejected": -2.609663963317871,
+      "eval_runtime": 204.4704,
+      "eval_samples_per_second": 9.781,
+      "eval_steps_per_second": 0.611,
+      "step": 300
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 4.272609194017105e-07,
+      "logits/chosen": -3.7294070720672607,
+      "logits/rejected": -3.789606809616089,
+      "logps/chosen": -506.30218505859375,
+      "logps/rejected": -687.495849609375,
+      "loss": 0.4155,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -1.452553629875183,
+      "rewards/margins": 1.6206657886505127,
+      "rewards/rejected": -3.0732195377349854,
+      "step": 310
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 4.2069638288135547e-07,
+      "logits/chosen": -3.735316753387451,
+      "logits/rejected": -3.802316188812256,
+      "logps/chosen": -556.4932861328125,
+      "logps/rejected": -685.213134765625,
+      "loss": 0.4397,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -1.5745762586593628,
+      "rewards/margins": 1.468933343887329,
+      "rewards/rejected": -3.0435097217559814,
+      "step": 320
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 4.139035550786494e-07,
+      "logits/chosen": -3.783989667892456,
+      "logits/rejected": -3.8774330615997314,
+      "logps/chosen": -530.0714111328125,
+      "logps/rejected": -536.76123046875,
+      "loss": 0.4832,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.6511234045028687,
+      "rewards/margins": 1.1192381381988525,
+      "rewards/rejected": -2.7703614234924316,
+      "step": 330
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 4.0689152079869306e-07,
+      "logits/chosen": -3.9041965007781982,
+      "logits/rejected": -3.946739912033081,
+      "logps/chosen": -430.56365966796875,
+      "logps/rejected": -504.49786376953125,
+      "loss": 0.4937,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -1.466640591621399,
+      "rewards/margins": 0.9467592239379883,
+      "rewards/rejected": -2.4133996963500977,
+      "step": 340
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 3.99669658015821e-07,
+      "logits/chosen": -3.7613883018493652,
+      "logits/rejected": -3.790003538131714,
+      "logps/chosen": -495.4722595214844,
+      "logps/rejected": -638.2640380859375,
+      "loss": 0.4831,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -1.7402899265289307,
+      "rewards/margins": 1.1566716432571411,
+      "rewards/rejected": -2.8969614505767822,
+      "step": 350
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 3.92247625331392e-07,
+      "logits/chosen": -3.937546968460083,
+      "logits/rejected": -3.993028163909912,
+      "logps/chosen": -548.022216796875,
+      "logps/rejected": -609.3221435546875,
+      "loss": 0.4675,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -1.875231385231018,
+      "rewards/margins": 0.993303656578064,
+      "rewards/rejected": -2.868535280227661,
+      "step": 360
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 3.846353490562664e-07,
+      "logits/chosen": -3.8289928436279297,
+      "logits/rejected": -3.8607897758483887,
+      "logps/chosen": -511.55853271484375,
+      "logps/rejected": -626.59814453125,
+      "loss": 0.4382,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.6782903671264648,
+      "rewards/margins": 0.9424189329147339,
+      "rewards/rejected": -2.6207094192504883,
+      "step": 370
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 3.768430099352445e-07,
+      "logits/chosen": -3.8665966987609863,
+      "logits/rejected": -3.9191222190856934,
+      "logps/chosen": -600.85400390625,
+      "logps/rejected": -636.7755126953125,
+      "loss": 0.4414,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.972818374633789,
+      "rewards/margins": 0.9349247217178345,
+      "rewards/rejected": -2.907742977142334,
+      "step": 380
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 3.6888102953122304e-07,
+      "logits/chosen": -3.8797574043273926,
+      "logits/rejected": -3.9397029876708984,
+      "logps/chosen": -541.1205444335938,
+      "logps/rejected": -635.4256591796875,
+      "loss": 0.4324,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -1.8883755207061768,
+      "rewards/margins": 1.541088581085205,
+      "rewards/rejected": -3.4294638633728027,
+      "step": 390
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 3.607600562872785e-07,
+      "logits/chosen": -3.781221389770508,
+      "logits/rejected": -3.8847899436950684,
+      "logps/chosen": -629.6137084960938,
+      "logps/rejected": -632.5733032226562,
+      "loss": 0.4359,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -2.1249642372131348,
+      "rewards/margins": 1.0126584768295288,
+      "rewards/rejected": -3.137622833251953,
+      "step": 400
+    },
+    {
+      "epoch": 0.42,
+      "eval_logits/chosen": -3.6220741271972656,
+      "eval_logits/rejected": -3.702164888381958,
+      "eval_logps/chosen": -594.591796875,
+      "eval_logps/rejected": -651.9946899414062,
+      "eval_loss": 0.4466642141342163,
+      "eval_rewards/accuracies": 0.7680000066757202,
+      "eval_rewards/chosen": -2.087851047515869,
+      "eval_rewards/margins": 1.1281490325927734,
+      "eval_rewards/rejected": -3.2160003185272217,
+      "eval_runtime": 201.9384,
+      "eval_samples_per_second": 9.904,
+      "eval_steps_per_second": 0.619,
+      "step": 400
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 3.5249095128531856e-07,
+      "logits/chosen": -3.7463316917419434,
+      "logits/rejected": -3.8346214294433594,
+      "logps/chosen": -645.6383666992188,
+      "logps/rejected": -699.442626953125,
+      "loss": 0.4482,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -1.9998807907104492,
+      "rewards/margins": 1.0837665796279907,
+      "rewards/rejected": -3.0836472511291504,
+      "step": 410
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 3.4408477372034736e-07,
+      "logits/chosen": -3.732384204864502,
+      "logits/rejected": -3.813812255859375,
+      "logps/chosen": -501.06536865234375,
+      "logps/rejected": -586.9138793945312,
+      "loss": 0.4336,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -1.9944934844970703,
+      "rewards/margins": 1.117249846458435,
+      "rewards/rejected": -3.111743211746216,
+      "step": 420
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 3.3555276610977276e-07,
+      "logits/chosen": -3.799959659576416,
+      "logits/rejected": -3.7980499267578125,
+      "logps/chosen": -466.7644958496094,
+      "logps/rejected": -580.1270751953125,
+      "loss": 0.4684,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -1.807948350906372,
+      "rewards/margins": 1.1490823030471802,
+      "rewards/rejected": -2.9570305347442627,
+      "step": 430
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 3.269063392575352e-07,
+      "logits/chosen": -3.8081250190734863,
+      "logits/rejected": -3.8595759868621826,
+      "logps/chosen": -476.84326171875,
+      "logps/rejected": -597.8814697265625,
+      "loss": 0.4459,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.5761287212371826,
+      "rewards/margins": 1.3166580200195312,
+      "rewards/rejected": -2.892787218093872,
+      "step": 440
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 3.1815705699316964e-07,
+      "logits/chosen": -3.71815824508667,
+      "logits/rejected": -3.7602291107177734,
+      "logps/chosen": -502.3971252441406,
+      "logps/rejected": -605.4251708984375,
+      "loss": 0.4634,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -1.5930547714233398,
+      "rewards/margins": 1.314320683479309,
+      "rewards/rejected": -2.9073758125305176,
+      "step": 450
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 3.0931662070620794e-07,
+      "logits/chosen": -3.741687774658203,
+      "logits/rejected": -3.811591625213623,
+      "logps/chosen": -549.6846923828125,
+      "logps/rejected": -649.2257080078125,
+      "loss": 0.4345,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -1.719300627708435,
+      "rewards/margins": 1.1254762411117554,
+      "rewards/rejected": -2.8447766304016113,
+      "step": 460
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 3.003968536966078e-07,
+      "logits/chosen": -3.716046094894409,
+      "logits/rejected": -3.803307056427002,
+      "logps/chosen": -572.4447631835938,
+      "logps/rejected": -625.49755859375,
+      "loss": 0.4451,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -1.8884143829345703,
+      "rewards/margins": 1.2486246824264526,
+      "rewards/rejected": -3.1370389461517334,
+      "step": 470
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 2.9140968536213693e-07,
+      "logits/chosen": -3.867417573928833,
+      "logits/rejected": -3.8436825275421143,
+      "logps/chosen": -478.40789794921875,
+      "logps/rejected": -613.2086791992188,
+      "loss": 0.4803,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -2.025106906890869,
+      "rewards/margins": 1.0343272686004639,
+      "rewards/rejected": -3.059434175491333,
+      "step": 480
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 2.823671352438608e-07,
+      "logits/chosen": -3.8223514556884766,
+      "logits/rejected": -3.864264965057373,
+      "logps/chosen": -514.0059814453125,
+      "logps/rejected": -626.1476440429688,
+      "loss": 0.4292,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -1.7328517436981201,
+      "rewards/margins": 1.1822339296340942,
+      "rewards/rejected": -2.915085554122925,
+      "step": 490
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 2.73281296951072e-07,
+      "logits/chosen": -3.8355319499969482,
+      "logits/rejected": -3.8868179321289062,
+      "logps/chosen": -587.6881103515625,
+      "logps/rejected": -719.8489990234375,
+      "loss": 0.4271,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -2.061047315597534,
+      "rewards/margins": 1.5319583415985107,
+      "rewards/rejected": -3.593005418777466,
+      "step": 500
+    },
+    {
+      "epoch": 0.52,
+      "eval_logits/chosen": -3.7408037185668945,
+      "eval_logits/rejected": -3.8189327716827393,
+      "eval_logps/chosen": -591.3005981445312,
+      "eval_logps/rejected": -652.2026977539062,
+      "eval_loss": 0.4440613090991974,
+      "eval_rewards/accuracies": 0.7839999794960022,
+      "eval_rewards/chosen": -2.054938793182373,
+      "eval_rewards/margins": 1.1631413698196411,
+      "eval_rewards/rejected": -3.2180800437927246,
+      "eval_runtime": 203.8018,
+      "eval_samples_per_second": 9.813,
+      "eval_steps_per_second": 0.613,
+      "step": 500
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 2.641643219871597e-07,
+      "logits/chosen": -3.8902289867401123,
+      "logits/rejected": -4.004373550415039,
+      "logps/chosen": -559.3378295898438,
+      "logps/rejected": -618.6632690429688,
+      "loss": 0.3992,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -2.054337739944458,
+      "rewards/margins": 1.3023302555084229,
+      "rewards/rejected": -3.3566677570343018,
+      "step": 510
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 2.550284034980507e-07,
+      "logits/chosen": -3.8437469005584717,
+      "logits/rejected": -3.888261318206787,
+      "logps/chosen": -559.3350830078125,
+      "logps/rejected": -661.4388427734375,
+      "loss": 0.4473,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -1.9623550176620483,
+      "rewards/margins": 1.2001652717590332,
+      "rewards/rejected": -3.162520170211792,
+      "step": 520
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 2.4588575996495794e-07,
+      "logits/chosen": -3.9187228679656982,
+      "logits/rejected": -3.973421096801758,
+      "logps/chosen": -602.2457885742188,
+      "logps/rejected": -682.3179931640625,
+      "loss": 0.42,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.9900497198104858,
+      "rewards/margins": 1.2919167280197144,
+      "rewards/rejected": -3.2819664478302,
+      "step": 530
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 2.367486188632446e-07,
+      "logits/chosen": -3.9012649059295654,
+      "logits/rejected": -3.986586093902588,
+      "logps/chosen": -567.7922973632812,
+      "logps/rejected": -682.8541259765625,
+      "loss": 0.4298,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -1.8885074853897095,
+      "rewards/margins": 1.357795000076294,
+      "rewards/rejected": -3.2463021278381348,
+      "step": 540
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 2.276292003092593e-07,
+      "logits/chosen": -3.9662468433380127,
+      "logits/rejected": -4.023472785949707,
+      "logps/chosen": -548.5009765625,
+      "logps/rejected": -603.1859130859375,
+      "loss": 0.4332,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -1.9097919464111328,
+      "rewards/margins": 1.235344648361206,
+      "rewards/rejected": -3.145136594772339,
+      "step": 550
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 2.185397007170141e-07,
+      "logits/chosen": -3.96891713142395,
+      "logits/rejected": -3.992366075515747,
+      "logps/chosen": -527.6655883789062,
+      "logps/rejected": -594.5096435546875,
+      "loss": 0.4661,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -2.0592868328094482,
+      "rewards/margins": 1.0083281993865967,
+      "rewards/rejected": -3.067615032196045,
+      "step": 560
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 2.094922764865619e-07,
+      "logits/chosen": -3.8889777660369873,
+      "logits/rejected": -3.9203734397888184,
+      "logps/chosen": -530.451904296875,
+      "logps/rejected": -649.4442749023438,
+      "loss": 0.4476,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -2.085118532180786,
+      "rewards/margins": 1.273771047592163,
+      "rewards/rejected": -3.3588898181915283,
+      "step": 570
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 2.0049902774588797e-07,
+      "logits/chosen": -3.9504013061523438,
+      "logits/rejected": -4.046439170837402,
+      "logps/chosen": -551.9801635742188,
+      "logps/rejected": -631.9030151367188,
+      "loss": 0.4411,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -2.0798585414886475,
+      "rewards/margins": 1.2492122650146484,
+      "rewards/rejected": -3.329070568084717,
+      "step": 580
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 1.9157198216806238e-07,
+      "logits/chosen": -3.846717119216919,
+      "logits/rejected": -3.8746533393859863,
+      "logps/chosen": -514.6439208984375,
+      "logps/rejected": -652.5971069335938,
+      "loss": 0.4229,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.9687875509262085,
+      "rewards/margins": 1.090827465057373,
+      "rewards/rejected": -3.059614896774292,
+      "step": 590
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 1.8272307888529274e-07,
+      "logits/chosen": -3.835319995880127,
+      "logits/rejected": -3.88202166557312,
+      "logps/chosen": -595.4498291015625,
+      "logps/rejected": -732.9796142578125,
+      "loss": 0.4181,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -1.6396028995513916,
+      "rewards/margins": 1.44252610206604,
+      "rewards/rejected": -3.0821290016174316,
+      "step": 600
+    },
+    {
+      "epoch": 0.63,
+      "eval_logits/chosen": -3.717047929763794,
+      "eval_logits/rejected": -3.7949790954589844,
+      "eval_logps/chosen": -584.56982421875,
+      "eval_logps/rejected": -647.177734375,
+      "eval_loss": 0.43660393357276917,
+      "eval_rewards/accuracies": 0.7760000228881836,
+      "eval_rewards/chosen": -1.9876312017440796,
+      "eval_rewards/margins": 1.1801990270614624,
+      "eval_rewards/rejected": -3.167830467224121,
+      "eval_runtime": 203.6736,
+      "eval_samples_per_second": 9.82,
+      "eval_steps_per_second": 0.614,
+      "step": 600
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 1.7396415252139288e-07,
+      "logits/chosen": -3.9154458045959473,
+      "logits/rejected": -3.974060535430908,
+      "logps/chosen": -533.2646484375,
+      "logps/rejected": -585.8008422851562,
+      "loss": 0.4206,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.9508171081542969,
+      "rewards/margins": 1.1387238502502441,
+      "rewards/rejected": -3.089540958404541,
+      "step": 610
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 1.6530691736402316e-07,
+      "logits/chosen": -3.8785858154296875,
+      "logits/rejected": -3.912694215774536,
+      "logps/chosen": -563.8421630859375,
+      "logps/rejected": -652.9039306640625,
+      "loss": 0.4436,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -2.181243658065796,
+      "rewards/margins": 1.337597131729126,
+      "rewards/rejected": -3.518840789794922,
+      "step": 620
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 1.5676295169786864e-07,
+      "logits/chosen": -3.9640355110168457,
+      "logits/rejected": -4.0246195793151855,
+      "logps/chosen": -543.6168823242188,
+      "logps/rejected": -720.9603271484375,
+      "loss": 0.3924,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": -1.9450584650039673,
+      "rewards/margins": 1.83893620967865,
+      "rewards/rejected": -3.783994674682617,
+      "step": 630
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 1.483436823197092e-07,
+      "logits/chosen": -3.8995678424835205,
+      "logits/rejected": -3.9963104724884033,
+      "logps/chosen": -529.6903686523438,
+      "logps/rejected": -631.6978149414062,
+      "loss": 0.4512,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -1.9272207021713257,
+      "rewards/margins": 1.3727694749832153,
+      "rewards/rejected": -3.29999041557312,
+      "step": 640
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 1.4006036925609243e-07,
+      "logits/chosen": -3.972506046295166,
+      "logits/rejected": -4.015632152557373,
+      "logps/chosen": -595.8973388671875,
+      "logps/rejected": -727.6198120117188,
+      "loss": 0.4132,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -2.108734369277954,
+      "rewards/margins": 1.5102851390838623,
+      "rewards/rejected": -3.6190192699432373,
+      "step": 650
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 1.319240907040458e-07,
+      "logits/chosen": -3.9822299480438232,
+      "logits/rejected": -4.0792555809021,
+      "logps/chosen": -610.338134765625,
+      "logps/rejected": -650.5155639648438,
+      "loss": 0.4142,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -2.233393669128418,
+      "rewards/margins": 1.272410273551941,
+      "rewards/rejected": -3.5058040618896484,
+      "step": 660
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 1.239457282149695e-07,
+      "logits/chosen": -4.008191108703613,
+      "logits/rejected": -4.095564842224121,
+      "logps/chosen": -588.9998168945312,
+      "logps/rejected": -686.9320068359375,
+      "loss": 0.4315,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -2.217425584793091,
+      "rewards/margins": 1.2320573329925537,
+      "rewards/rejected": -3.4494826793670654,
+      "step": 670
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 1.1613595214152711e-07,
+      "logits/chosen": -3.9681782722473145,
+      "logits/rejected": -4.009424686431885,
+      "logps/chosen": -520.8330688476562,
+      "logps/rejected": -595.3616943359375,
+      "loss": 0.4556,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.8725841045379639,
+      "rewards/margins": 1.0434370040893555,
+      "rewards/rejected": -2.9160211086273193,
+      "step": 680
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 1.0850520736699362e-07,
+      "logits/chosen": -3.910592555999756,
+      "logits/rejected": -3.9710609912872314,
+      "logps/chosen": -484.9183044433594,
+      "logps/rejected": -601.7095336914062,
+      "loss": 0.408,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -1.8240716457366943,
+      "rewards/margins": 1.474691390991211,
+      "rewards/rejected": -3.2987632751464844,
+      "step": 690
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 1.0106369933615042e-07,
+      "logits/chosen": -3.870868682861328,
+      "logits/rejected": -3.974864959716797,
+      "logps/chosen": -537.2079467773438,
+      "logps/rejected": -636.1268310546875,
+      "loss": 0.4,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -1.947430968284607,
+      "rewards/margins": 1.452516794204712,
+      "rewards/rejected": -3.3999481201171875,
+      "step": 700
+    },
+    {
+      "epoch": 0.73,
+      "eval_logits/chosen": -3.7969613075256348,
+      "eval_logits/rejected": -3.87385630607605,
+      "eval_logps/chosen": -602.2762451171875,
+      "eval_logps/rejected": -665.6045532226562,
+      "eval_loss": 0.4316871762275696,
+      "eval_rewards/accuracies": 0.7639999985694885,
+      "eval_rewards/chosen": -2.1646955013275146,
+      "eval_rewards/margins": 1.1874041557312012,
+      "eval_rewards/rejected": -3.352099657058716,
+      "eval_runtime": 204.7157,
+      "eval_samples_per_second": 9.77,
+      "eval_steps_per_second": 0.611,
+      "step": 700
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 9.382138040640714e-08,
+      "logits/chosen": -3.92706298828125,
+      "logits/rejected": -3.9605202674865723,
+      "logps/chosen": -567.6884155273438,
+      "logps/rejected": -623.2738647460938,
+      "loss": 0.4277,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -2.1468379497528076,
+      "rewards/margins": 1.0972353219985962,
+      "rewards/rejected": -3.2440733909606934,
+      "step": 710
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 8.678793653740632e-08,
+      "logits/chosen": -3.9516849517822266,
+      "logits/rejected": -4.008336544036865,
+      "logps/chosen": -593.2915649414062,
+      "logps/rejected": -721.9989013671875,
+      "loss": 0.4049,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -2.1563327312469482,
+      "rewards/margins": 1.5128753185272217,
+      "rewards/rejected": -3.669208526611328,
+      "step": 720
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 7.997277433690983e-08,
+      "logits/chosen": -3.8838019371032715,
+      "logits/rejected": -3.978668689727783,
+      "logps/chosen": -610.8298950195312,
+      "logps/rejected": -639.4989624023438,
+      "loss": 0.4293,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -2.255786418914795,
+      "rewards/margins": 1.1647707223892212,
+      "rewards/rejected": -3.4205574989318848,
+      "step": 730
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 7.338500848029602e-08,
+      "logits/chosen": -3.8697731494903564,
+      "logits/rejected": -4.00671911239624,
+      "logps/chosen": -697.6756591796875,
+      "logps/rejected": -759.2227783203125,
+      "loss": 0.4093,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -2.504605293273926,
+      "rewards/margins": 1.4814784526824951,
+      "rewards/rejected": -3.986083507537842,
+      "step": 740
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 6.70334495204884e-08,
+      "logits/chosen": -3.8761909008026123,
+      "logits/rejected": -3.915804386138916,
+      "logps/chosen": -594.4487915039062,
+      "logps/rejected": -745.0926513671875,
+      "loss": 0.4379,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -2.277494430541992,
+      "rewards/margins": 1.3665436506271362,
+      "rewards/rejected": -3.644038438796997,
+      "step": 750
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 6.092659210462231e-08,
+      "logits/chosen": -3.8564085960388184,
+      "logits/rejected": -3.92138671875,
+      "logps/chosen": -573.0128173828125,
+      "logps/rejected": -675.12060546875,
+      "loss": 0.394,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -2.239450693130493,
+      "rewards/margins": 1.328932285308838,
+      "rewards/rejected": -3.568382740020752,
+      "step": 760
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 5.507260361320737e-08,
+      "logits/chosen": -3.944653034210205,
+      "logits/rejected": -3.991550922393799,
+      "logps/chosen": -670.3541870117188,
+      "logps/rejected": -742.4327392578125,
+      "loss": 0.4177,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -2.406642198562622,
+      "rewards/margins": 1.1013365983963013,
+      "rewards/rejected": -3.5079784393310547,
+      "step": 770
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 4.947931323697982e-08,
+      "logits/chosen": -3.9694862365722656,
+      "logits/rejected": -3.969531297683716,
+      "logps/chosen": -480.57232666015625,
+      "logps/rejected": -576.7296142578125,
+      "loss": 0.4341,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -2.20851469039917,
+      "rewards/margins": 0.9807012677192688,
+      "rewards/rejected": -3.189215898513794,
+      "step": 780
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 4.415420150605398e-08,
+      "logits/chosen": -3.9178504943847656,
+      "logits/rejected": -3.8871982097625732,
+      "logps/chosen": -574.5850830078125,
+      "logps/rejected": -722.9913330078125,
+      "loss": 0.4176,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -2.2402052879333496,
+      "rewards/margins": 1.2347979545593262,
+      "rewards/rejected": -3.475003480911255,
+      "step": 790
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 3.9104390285376374e-08,
+      "logits/chosen": -3.819741725921631,
+      "logits/rejected": -3.860565185546875,
+      "logps/chosen": -664.21142578125,
+      "logps/rejected": -710.8525390625,
+      "loss": 0.4123,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -2.2596795558929443,
+      "rewards/margins": 1.2178363800048828,
+      "rewards/rejected": -3.4775161743164062,
+      "step": 800
+    },
+    {
+      "epoch": 0.84,
+      "eval_logits/chosen": -3.7826802730560303,
+      "eval_logits/rejected": -3.8605716228485107,
+      "eval_logps/chosen": -606.1934204101562,
+      "eval_logps/rejected": -675.3074951171875,
+      "eval_loss": 0.42910608649253845,
+      "eval_rewards/accuracies": 0.7680000066757202,
+      "eval_rewards/chosen": -2.2038679122924805,
+      "eval_rewards/margins": 1.245260238647461,
+      "eval_rewards/rejected": -3.4491279125213623,
+      "eval_runtime": 203.5751,
+      "eval_samples_per_second": 9.824,
+      "eval_steps_per_second": 0.614,
+      "step": 800
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 3.433663324986208e-08,
+      "logits/chosen": -3.9174346923828125,
+      "logits/rejected": -3.9697937965393066,
+      "logps/chosen": -566.511962890625,
+      "logps/rejected": -639.9072265625,
+      "loss": 0.4185,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -2.194934606552124,
+      "rewards/margins": 1.1658843755722046,
+      "rewards/rejected": -3.3608193397521973,
+      "step": 810
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 2.9857306851953897e-08,
+      "logits/chosen": -3.928525447845459,
+      "logits/rejected": -4.02262544631958,
+      "logps/chosen": -588.645751953125,
+      "logps/rejected": -649.748046875,
+      "loss": 0.4511,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -2.0678915977478027,
+      "rewards/margins": 1.4058529138565063,
+      "rewards/rejected": -3.4737441539764404,
+      "step": 820
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 2.567240179368185e-08,
+      "logits/chosen": -3.9319825172424316,
+      "logits/rejected": -3.9248032569885254,
+      "logps/chosen": -541.6334228515625,
+      "logps/rejected": -707.2698364257812,
+      "loss": 0.4226,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -2.2310593128204346,
+      "rewards/margins": 1.1511319875717163,
+      "rewards/rejected": -3.3821911811828613,
+      "step": 830
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 2.1787515014630357e-08,
+      "logits/chosen": -3.929730176925659,
+      "logits/rejected": -3.983701705932617,
+      "logps/chosen": -561.4950561523438,
+      "logps/rejected": -643.8230590820312,
+      "loss": 0.457,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -2.112525224685669,
+      "rewards/margins": 1.0804126262664795,
+      "rewards/rejected": -3.1929378509521484,
+      "step": 840
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.820784220652766e-08,
+      "logits/chosen": -3.8999176025390625,
+      "logits/rejected": -3.999703884124756,
+      "logps/chosen": -581.9009399414062,
+      "logps/rejected": -630.4606323242188,
+      "loss": 0.4246,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -1.9299805164337158,
+      "rewards/margins": 1.3858671188354492,
+      "rewards/rejected": -3.315847873687744,
+      "step": 850
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.4938170864468636e-08,
+      "logits/chosen": -3.9592201709747314,
+      "logits/rejected": -3.982034206390381,
+      "logps/chosen": -560.0491333007812,
+      "logps/rejected": -656.3305053710938,
+      "loss": 0.4568,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -2.186509847640991,
+      "rewards/margins": 1.1325315237045288,
+      "rewards/rejected": -3.3190410137176514,
+      "step": 860
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 1.1982873884064465e-08,
+      "logits/chosen": -3.9274532794952393,
+      "logits/rejected": -3.9674625396728516,
+      "logps/chosen": -524.0363159179688,
+      "logps/rejected": -671.90966796875,
+      "loss": 0.424,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -1.886129379272461,
+      "rewards/margins": 1.563849687576294,
+      "rewards/rejected": -3.4499785900115967,
+      "step": 870
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 9.345903713082304e-09,
+      "logits/chosen": -3.8932175636291504,
+      "logits/rejected": -3.9537672996520996,
+      "logps/chosen": -636.6458740234375,
+      "logps/rejected": -725.6226196289062,
+      "loss": 0.4073,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -2.2143332958221436,
+      "rewards/margins": 1.2432187795639038,
+      "rewards/rejected": -3.457551956176758,
+      "step": 880
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 7.030787065396865e-09,
+      "logits/chosen": -3.920933961868286,
+      "logits/rejected": -3.956883192062378,
+      "logps/chosen": -529.4722290039062,
+      "logps/rejected": -692.9610595703125,
+      "loss": 0.431,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -1.8962398767471313,
+      "rewards/margins": 1.446508765220642,
+      "rewards/rejected": -3.3427481651306152,
+      "step": 890
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 5.04062020432286e-09,
+      "logits/chosen": -3.8574442863464355,
+      "logits/rejected": -3.9011623859405518,
+      "logps/chosen": -620.3335571289062,
+      "logps/rejected": -719.7374267578125,
+      "loss": 0.4394,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -2.159029722213745,
+      "rewards/margins": 1.208085298538208,
+      "rewards/rejected": -3.367115020751953,
+      "step": 900
+    },
+    {
+      "epoch": 0.94,
+      "eval_logits/chosen": -3.8000898361206055,
+      "eval_logits/rejected": -3.8776535987854004,
+      "eval_logps/chosen": -599.057373046875,
+      "eval_logps/rejected": -666.7249755859375,
+      "eval_loss": 0.42921942472457886,
+      "eval_rewards/accuracies": 0.7680000066757202,
+      "eval_rewards/chosen": -2.132506847381592,
+      "eval_rewards/margins": 1.230795979499817,
+      "eval_rewards/rejected": -3.3633029460906982,
+      "eval_runtime": 203.7413,
+      "eval_samples_per_second": 9.816,
+      "eval_steps_per_second": 0.614,
+      "step": 900
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 3.3780648016376866e-09,
+      "logits/chosen": -3.9453799724578857,
+      "logits/rejected": -3.986725330352783,
+      "logps/chosen": -560.10205078125,
+      "logps/rejected": -671.8740234375,
+      "loss": 0.4277,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -2.1619820594787598,
+      "rewards/margins": 1.315500259399414,
+      "rewards/rejected": -3.477482557296753,
+      "step": 910
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.0453443778310766e-09,
+      "logits/chosen": -3.936732530593872,
+      "logits/rejected": -3.999469041824341,
+      "logps/chosen": -593.9410400390625,
+      "logps/rejected": -700.7437744140625,
+      "loss": 0.3904,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -2.2095024585723877,
+      "rewards/margins": 1.3912229537963867,
+      "rewards/rejected": -3.600724697113037,
+      "step": 920
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.0442413283435758e-09,
+      "logits/chosen": -3.881744384765625,
+      "logits/rejected": -3.9535934925079346,
+      "logps/chosen": -482.16058349609375,
+      "logps/rejected": -652.9393310546875,
+      "loss": 0.3867,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -1.774622917175293,
+      "rewards/margins": 1.7917388677597046,
+      "rewards/rejected": -3.566361904144287,
+      "step": 930
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 3.760945397705828e-10,
+      "logits/chosen": -3.958568572998047,
+      "logits/rejected": -4.010906219482422,
+      "logps/chosen": -617.6387939453125,
+      "logps/rejected": -726.321044921875,
+      "loss": 0.4054,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -2.300060749053955,
+      "rewards/margins": 1.2705519199371338,
+      "rewards/rejected": -3.570613145828247,
+      "step": 940
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 4.17975992204056e-11,
+      "logits/chosen": -3.931025266647339,
+      "logits/rejected": -3.9499351978302,
+      "logps/chosen": -544.7789916992188,
+      "logps/rejected": -728.7254638671875,
+      "loss": 0.4304,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": -2.2649009227752686,
+      "rewards/margins": 1.7213318347930908,
+      "rewards/rejected": -3.9862327575683594,
+      "step": 950
+    },
+    {
+      "epoch": 1.0,
+      "step": 955,
+      "total_flos": 0.0,
+      "train_loss": 0.46914771214829687,
+      "train_runtime": 18559.523,
+      "train_samples_per_second": 3.294,
+      "train_steps_per_second": 0.051
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 955,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 100,
+  "total_flos": 0.0,
+  "train_batch_size": 4,
+  "trial_name": null,
+  "trial_params": null
+}