Model save

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,79 @@
+---
+library_name: transformers
+license: gemma
+base_model: google/gemma-7b
+tags:
+- trl
+- orpo
+- generated_from_trainer
+model-index:
+- name: gemma-7b-borpo
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# gemma-7b-borpo
+
+This model is a fine-tuned version of [google/gemma-7b](https://huggingface.co/google/gemma-7b) on an unknown dataset.
+It achieves the following results on the evaluation set:
+- Loss: 1.5984
+- Rewards/chosen: -0.0575
+- Rewards/rejected: -0.0699
+- Rewards/accuracies: 0.5899
+- Rewards/margins: 0.0124
+- Logps/rejected: -1.3977
+- Logps/chosen: -1.1506
+- Logits/rejected: 270.9628
+- Logits/chosen: 299.8625
+- Nll Loss: 1.5312
+- Log Odds Ratio: -0.6761
+- Log Odds Chosen: 0.3679
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 5e-06
+- train_batch_size: 2
+- eval_batch_size: 1
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 4
+- gradient_accumulation_steps: 4
+- total_train_batch_size: 32
+- total_eval_batch_size: 4
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: inverse_sqrt
+- lr_scheduler_warmup_steps: 100
+- num_epochs: 3
+
+### Training results
+
+| Training Loss | Epoch  | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen | Nll Loss | Log Odds Ratio | Log Odds Chosen |
+|:-------------:|:------:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|:--------:|:--------------:|:---------------:|
+| 1.4516        | 0.9968 | 157  | 1.4765          | -0.0513        | -0.0577          | 0.5468             | 0.0064          | -1.1547        | -1.0260      | 293.8872        | 321.9495      | 1.4282   | -0.6924        | 0.1911          |
+| 1.0587        | 2.0    | 315  | 1.4250          | -0.0502        | -0.0595          | 0.5468             | 0.0093          | -1.1904        | -1.0035      | 296.0850        | 323.6012      | 1.3729   | -0.6901        | 0.2723          |
+| 0.5897        | 2.9905 | 471  | 1.5984          | -0.0575        | -0.0699          | 0.5899             | 0.0124          | -1.3977        | -1.1506      | 270.9628        | 299.8625      | 1.5312   | -0.6761        | 0.3679          |
+
+
+### Framework versions
+
+- Transformers 4.44.2
+- Pytorch 2.4.0+cu121
+- Datasets 3.0.0
+- Tokenizers 0.19.1

all_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 2.9904761904761905,
+    "total_flos": 0.0,
+    "train_loss": 1.4937715783493788,
+    "train_runtime": 12592.9879,
+    "train_samples": 5034,
+    "train_samples_per_second": 1.199,
+    "train_steps_per_second": 0.037
+}

config.json

@@ -0,0 +1,29 @@
+{
+  "_name_or_path": "google/gemma-7b",
+  "architectures": [
+    "GemmaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 106,
+  "eos_token_id": 107,
+  "head_dim": 256,
+  "hidden_act": "gelu",
+  "hidden_activation": "gelu_pytorch_tanh",
+  "hidden_size": 3072,
+  "initializer_range": 0.02,
+  "intermediate_size": 24576,
+  "max_position_embeddings": 8192,
+  "model_type": "gemma",
+  "num_attention_heads": 16,
+  "num_hidden_layers": 28,
+  "num_key_value_heads": 16,
+  "pad_token_id": 107,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 10000.0,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.44.2",
+  "use_cache": false,
+  "vocab_size": 256000
+}

generation_config.json

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 106,
+  "eos_token_id": 107,
+  "pad_token_id": 107,
+  "transformers_version": "4.44.2"
+}

model-00001-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:feb9e5df7227c57a5f17b3d866c3ebe19e95cc2008c29163aafc45ce64d572a6
+size 4995496656

model-00002-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:315e3392b12dfef4be759999aebfadfead0d6e62af35c51bdc47730de4a9fe44
+size 4982953168

model-00003-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:71410e63d4228e1e273e73b64be4f576c2df7b294e44c58ed37335fa6bda64a1
+size 4982953200

model-00004-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7b87e4a8148ff22fdb3fd1b1f5a5d8b960f51571070581cbfe0b949ef6e68f8d
+size 2113988336

model.safetensors.index.json

@@ -0,0 +1,261 @@
+{
+  "metadata": {
+    "total_size": 17075361792
+  },
+  "weight_map": {
+    "model.embed_tokens.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.norm.weight": "model-00004-of-00004.safetensors"
+  }
+}

runs/Sep21_12-42-46_65ecb96dba42/events.out.tfevents.1726922648.65ecb96dba42.271982.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:011635d02f6fd04bdfb3fda4f1d7a2ec270f98d8b4119ff94717d3117fbbe82b
+size 88906

special_tokens_map.json

@@ -0,0 +1,28 @@
+{
+  "additional_special_tokens": [
+    {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    }
+  ],
+  "bos_token": "<|im_start|>",
+  "eos_token": "<|im_end|>",
+  "pad_token": "<|im_end|>",
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:322a5f52ab5cab196761ab397a022d6fa3a2e1418585e532bb6efb2fedd2ae94
+size 17477501

tokenizer_config.json

@@ -0,0 +1,70 @@
+{
+  "add_bos_token": false,
+  "add_eos_token": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<pad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<eos>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "<bos>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "106": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "107": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>"
+  ],
+  "bos_token": "<|im_start|>",
+  "chat_template": "{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "legacy": null,
+  "model_max_length": 2048,
+  "pad_token": "<|im_end|>",
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "GemmaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}

train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 2.9904761904761905,
+    "total_flos": 0.0,
+    "train_loss": 1.4937715783493788,
+    "train_runtime": 12592.9879,
+    "train_samples": 5034,
+    "train_samples_per_second": 1.199,
+    "train_steps_per_second": 0.037
+}

trainer_state.json

@@ -0,0 +1,1791 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 2.9904761904761905,
+  "eval_steps": 500,
+  "global_step": 471,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.031746031746031744,
+      "grad_norm": 1682.7587890625,
+      "learning_rate": 2.5000000000000004e-07,
+      "log_odds_chosen": 0.019949015229940414,
+      "log_odds_ratio": -1.0337324142456055,
+      "logits/chosen": 250.0281982421875,
+      "logits/rejected": 241.27963256835938,
+      "logps/chosen": -15.414708137512207,
+      "logps/rejected": -15.434652328491211,
+      "loss": 14.8392,
+      "nll_loss": 14.92613697052002,
+      "rewards/accuracies": 0.4000000059604645,
+      "rewards/chosen": -0.7707353830337524,
+      "rewards/margins": 0.0009973436826840043,
+      "rewards/rejected": -0.7717326879501343,
+      "step": 5
+    },
+    {
+      "epoch": 0.06349206349206349,
+      "grad_norm": 1073.85595703125,
+      "learning_rate": 5.000000000000001e-07,
+      "log_odds_chosen": 0.10046229511499405,
+      "log_odds_ratio": -0.9202533960342407,
+      "logits/chosen": 252.1835479736328,
+      "logits/rejected": 275.7188415527344,
+      "logps/chosen": -12.929780960083008,
+      "logps/rejected": -13.030034065246582,
+      "loss": 12.9337,
+      "nll_loss": 12.802912712097168,
+      "rewards/accuracies": 0.3499999940395355,
+      "rewards/chosen": -0.6464890241622925,
+      "rewards/margins": 0.005012729670852423,
+      "rewards/rejected": -0.6515016555786133,
+      "step": 10
+    },
+    {
+      "epoch": 0.09523809523809523,
+      "grad_norm": 566.9100952148438,
+      "learning_rate": 7.5e-07,
+      "log_odds_chosen": -0.22649931907653809,
+      "log_odds_ratio": -1.288556694984436,
+      "logits/chosen": 297.252685546875,
+      "logits/rejected": 288.51727294921875,
+      "logps/chosen": -8.612268447875977,
+      "logps/rejected": -8.385721206665039,
+      "loss": 8.2059,
+      "nll_loss": 8.307785034179688,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.4306134283542633,
+      "rewards/margins": -0.011327351443469524,
+      "rewards/rejected": -0.4192860722541809,
+      "step": 15
+    },
+    {
+      "epoch": 0.12698412698412698,
+      "grad_norm": 190.63539123535156,
+      "learning_rate": 1.0000000000000002e-06,
+      "log_odds_chosen": 0.15076614916324615,
+      "log_odds_ratio": -0.7511703968048096,
+      "logits/chosen": 239.71182250976562,
+      "logits/rejected": 275.3675537109375,
+      "logps/chosen": -4.92425537109375,
+      "logps/rejected": -5.06990385055542,
+      "loss": 5.3341,
+      "nll_loss": 5.193692207336426,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.24621276557445526,
+      "rewards/margins": 0.007282428443431854,
+      "rewards/rejected": -0.2534952163696289,
+      "step": 20
+    },
+    {
+      "epoch": 0.15873015873015872,
+      "grad_norm": 108.96724700927734,
+      "learning_rate": 1.25e-06,
+      "log_odds_chosen": -0.32157501578330994,
+      "log_odds_ratio": -1.014737606048584,
+      "logits/chosen": 249.96826171875,
+      "logits/rejected": 277.22991943359375,
+      "logps/chosen": -2.9985427856445312,
+      "logps/rejected": -2.6938087940216064,
+      "loss": 3.2459,
+      "nll_loss": 3.1284079551696777,
+      "rewards/accuracies": 0.3499999940395355,
+      "rewards/chosen": -0.14992713928222656,
+      "rewards/margins": -0.015236688777804375,
+      "rewards/rejected": -0.13469044864177704,
+      "step": 25
+    },
+    {
+      "epoch": 0.19047619047619047,
+      "grad_norm": 114.80986022949219,
+      "learning_rate": 1.5e-06,
+      "log_odds_chosen": 0.16358330845832825,
+      "log_odds_ratio": -0.7382305860519409,
+      "logits/chosen": 335.34710693359375,
+      "logits/rejected": 316.969970703125,
+      "logps/chosen": -1.8879131078720093,
+      "logps/rejected": -2.03938627243042,
+      "loss": 2.5624,
+      "nll_loss": 2.2552151679992676,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.09439565241336823,
+      "rewards/margins": 0.007573665119707584,
+      "rewards/rejected": -0.10196931660175323,
+      "step": 30
+    },
+    {
+      "epoch": 0.2222222222222222,
+      "grad_norm": 103.77478790283203,
+      "learning_rate": 1.75e-06,
+      "log_odds_chosen": 0.19950130581855774,
+      "log_odds_ratio": -0.6767204403877258,
+      "logits/chosen": 402.70550537109375,
+      "logits/rejected": 401.308837890625,
+      "logps/chosen": -1.784985899925232,
+      "logps/rejected": -1.9822971820831299,
+      "loss": 2.0861,
+      "nll_loss": 2.089512586593628,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.08924929052591324,
+      "rewards/margins": 0.009865568950772285,
+      "rewards/rejected": -0.09911485761404037,
+      "step": 35
+    },
+    {
+      "epoch": 0.25396825396825395,
+      "grad_norm": 87.8357925415039,
+      "learning_rate": 2.0000000000000003e-06,
+      "log_odds_chosen": 0.2945292890071869,
+      "log_odds_ratio": -0.6519163250923157,
+      "logits/chosen": 389.55194091796875,
+      "logits/rejected": 411.52655029296875,
+      "logps/chosen": -1.5404895544052124,
+      "logps/rejected": -1.7902438640594482,
+      "loss": 2.1696,
+      "nll_loss": 1.986445426940918,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.07702446728944778,
+      "rewards/margins": 0.012487736530601978,
+      "rewards/rejected": -0.08951220661401749,
+      "step": 40
+    },
+    {
+      "epoch": 0.2857142857142857,
+      "grad_norm": 43.71113967895508,
+      "learning_rate": 2.25e-06,
+      "log_odds_chosen": 0.16207186877727509,
+      "log_odds_ratio": -0.7175930142402649,
+      "logits/chosen": 377.09368896484375,
+      "logits/rejected": 402.40277099609375,
+      "logps/chosen": -1.3781359195709229,
+      "logps/rejected": -1.5260416269302368,
+      "loss": 1.9061,
+      "nll_loss": 1.8738489151000977,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.06890679895877838,
+      "rewards/margins": 0.007395277731120586,
+      "rewards/rejected": -0.07630207389593124,
+      "step": 45
+    },
+    {
+      "epoch": 0.31746031746031744,
+      "grad_norm": 72.43922424316406,
+      "learning_rate": 2.5e-06,
+      "log_odds_chosen": -0.16915926337242126,
+      "log_odds_ratio": -0.8527873158454895,
+      "logits/chosen": 398.2160949707031,
+      "logits/rejected": 385.0538330078125,
+      "logps/chosen": -1.474123239517212,
+      "logps/rejected": -1.3540009260177612,
+      "loss": 1.8456,
+      "nll_loss": 1.6751617193222046,
+      "rewards/accuracies": 0.4000000059604645,
+      "rewards/chosen": -0.07370616495609283,
+      "rewards/margins": -0.00600611325353384,
+      "rewards/rejected": -0.06770005077123642,
+      "step": 50
+    },
+    {
+      "epoch": 0.3492063492063492,
+      "grad_norm": 191.2710418701172,
+      "learning_rate": 2.7500000000000004e-06,
+      "log_odds_chosen": -0.08151008188724518,
+      "log_odds_ratio": -0.8122876286506653,
+      "logits/chosen": 402.912841796875,
+      "logits/rejected": 368.6970520019531,
+      "logps/chosen": -1.4740947484970093,
+      "logps/rejected": -1.3778045177459717,
+      "loss": 1.711,
+      "nll_loss": 1.6115481853485107,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.07370473444461823,
+      "rewards/margins": -0.004814522340893745,
+      "rewards/rejected": -0.06889022886753082,
+      "step": 55
+    },
+    {
+      "epoch": 0.38095238095238093,
+      "grad_norm": 44.32540512084961,
+      "learning_rate": 3e-06,
+      "log_odds_chosen": 0.23596720397472382,
+      "log_odds_ratio": -0.6497625708580017,
+      "logits/chosen": 352.0681457519531,
+      "logits/rejected": 393.3800354003906,
+      "logps/chosen": -1.2096924781799316,
+      "logps/rejected": -1.3974428176879883,
+      "loss": 1.7459,
+      "nll_loss": 1.6499731540679932,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.0604846253991127,
+      "rewards/margins": 0.009387515485286713,
+      "rewards/rejected": -0.06987214088439941,
+      "step": 60
+    },
+    {
+      "epoch": 0.4126984126984127,
+      "grad_norm": 42.01204299926758,
+      "learning_rate": 3.2500000000000002e-06,
+      "log_odds_chosen": -0.05983946472406387,
+      "log_odds_ratio": -0.833350658416748,
+      "logits/chosen": 355.71978759765625,
+      "logits/rejected": 387.98907470703125,
+      "logps/chosen": -1.2814446687698364,
+      "logps/rejected": -1.2495291233062744,
+      "loss": 1.6973,
+      "nll_loss": 1.529348611831665,
+      "rewards/accuracies": 0.4000000059604645,
+      "rewards/chosen": -0.06407222896814346,
+      "rewards/margins": -0.0015957739669829607,
+      "rewards/rejected": -0.06247646361589432,
+      "step": 65
+    },
+    {
+      "epoch": 0.4444444444444444,
+      "grad_norm": 73.30603790283203,
+      "learning_rate": 3.5e-06,
+      "log_odds_chosen": 0.24904093146324158,
+      "log_odds_ratio": -0.6408222317695618,
+      "logits/chosen": 389.5067443847656,
+      "logits/rejected": 379.82989501953125,
+      "logps/chosen": -1.2020925283432007,
+      "logps/rejected": -1.368801474571228,
+      "loss": 1.6581,
+      "nll_loss": 1.5083438158035278,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.06010463088750839,
+      "rewards/margins": 0.008335437625646591,
+      "rewards/rejected": -0.06844006478786469,
+      "step": 70
+    },
+    {
+      "epoch": 0.47619047619047616,
+      "grad_norm": 39.53456497192383,
+      "learning_rate": 3.7500000000000005e-06,
+      "log_odds_chosen": 0.2291734218597412,
+      "log_odds_ratio": -0.7071748971939087,
+      "logits/chosen": 414.55291748046875,
+      "logits/rejected": 395.8731994628906,
+      "logps/chosen": -1.2737112045288086,
+      "logps/rejected": -1.4267314672470093,
+      "loss": 1.7913,
+      "nll_loss": 1.7599716186523438,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.06368555873632431,
+      "rewards/margins": 0.00765102356672287,
+      "rewards/rejected": -0.07133658230304718,
+      "step": 75
+    },
+    {
+      "epoch": 0.5079365079365079,
+      "grad_norm": 32.64231872558594,
+      "learning_rate": 4.000000000000001e-06,
+      "log_odds_chosen": 0.3214460611343384,
+      "log_odds_ratio": -0.6462022066116333,
+      "logits/chosen": 416.7820739746094,
+      "logits/rejected": 400.43658447265625,
+      "logps/chosen": -1.1434603929519653,
+      "logps/rejected": -1.3834912776947021,
+      "loss": 1.67,
+      "nll_loss": 1.6429141759872437,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.057173021137714386,
+      "rewards/margins": 0.01200154609978199,
+      "rewards/rejected": -0.06917456537485123,
+      "step": 80
+    },
+    {
+      "epoch": 0.5396825396825397,
+      "grad_norm": 30.180082321166992,
+      "learning_rate": 4.25e-06,
+      "log_odds_chosen": 0.202164888381958,
+      "log_odds_ratio": -0.6881243586540222,
+      "logits/chosen": 406.5018310546875,
+      "logits/rejected": 416.58203125,
+      "logps/chosen": -1.2485874891281128,
+      "logps/rejected": -1.4020246267318726,
+      "loss": 1.6165,
+      "nll_loss": 1.5869615077972412,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.06242937967181206,
+      "rewards/margins": 0.007671858184039593,
+      "rewards/rejected": -0.07010124623775482,
+      "step": 85
+    },
+    {
+      "epoch": 0.5714285714285714,
+      "grad_norm": 122.0818862915039,
+      "learning_rate": 4.5e-06,
+      "log_odds_chosen": 0.013370111584663391,
+      "log_odds_ratio": -0.7374812960624695,
+      "logits/chosen": 392.1935729980469,
+      "logits/rejected": 372.4733581542969,
+      "logps/chosen": -1.0436148643493652,
+      "logps/rejected": -1.0640267133712769,
+      "loss": 1.4603,
+      "nll_loss": 1.4258912801742554,
+      "rewards/accuracies": 0.4000000059604645,
+      "rewards/chosen": -0.05218074470758438,
+      "rewards/margins": 0.001020595314912498,
+      "rewards/rejected": -0.0532013364136219,
+      "step": 90
+    },
+    {
+      "epoch": 0.6031746031746031,
+      "grad_norm": 47.91487121582031,
+      "learning_rate": 4.75e-06,
+      "log_odds_chosen": -0.2634345293045044,
+      "log_odds_ratio": -0.944506049156189,
+      "logits/chosen": 402.14569091796875,
+      "logits/rejected": 376.51416015625,
+      "logps/chosen": -1.3972222805023193,
+      "logps/rejected": -1.2059199810028076,
+      "loss": 1.5678,
+      "nll_loss": 1.6941537857055664,
+      "rewards/accuracies": 0.42500001192092896,
+      "rewards/chosen": -0.06986111402511597,
+      "rewards/margins": -0.009565119631588459,
+      "rewards/rejected": -0.060295987874269485,
+      "step": 95
+    },
+    {
+      "epoch": 0.6349206349206349,
+      "grad_norm": 29.921293258666992,
+      "learning_rate": 5e-06,
+      "log_odds_chosen": 0.16289451718330383,
+      "log_odds_ratio": -0.7408555746078491,
+      "logits/chosen": 414.12579345703125,
+      "logits/rejected": 409.37408447265625,
+      "logps/chosen": -1.2401769161224365,
+      "logps/rejected": -1.3893964290618896,
+      "loss": 1.6149,
+      "nll_loss": 1.531909465789795,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.062008846551179886,
+      "rewards/margins": 0.007460971362888813,
+      "rewards/rejected": -0.06946982443332672,
+      "step": 100
+    },
+    {
+      "epoch": 0.6666666666666666,
+      "grad_norm": 27.314743041992188,
+      "learning_rate": 4.8795003647426654e-06,
+      "log_odds_chosen": 0.32554125785827637,
+      "log_odds_ratio": -0.6547573804855347,
+      "logits/chosen": 419.6529235839844,
+      "logits/rejected": 403.6163330078125,
+      "logps/chosen": -1.074582576751709,
+      "logps/rejected": -1.3015466928482056,
+      "loss": 1.4794,
+      "nll_loss": 1.4857242107391357,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.05372912809252739,
+      "rewards/margins": 0.011348200961947441,
+      "rewards/rejected": -0.06507732719182968,
+      "step": 105
+    },
+    {
+      "epoch": 0.6984126984126984,
+      "grad_norm": 37.0976448059082,
+      "learning_rate": 4.767312946227961e-06,
+      "log_odds_chosen": 0.26148396730422974,
+      "log_odds_ratio": -0.6971127390861511,
+      "logits/chosen": 383.90203857421875,
+      "logits/rejected": 392.50604248046875,
+      "logps/chosen": -1.01774001121521,
+      "logps/rejected": -1.1847659349441528,
+      "loss": 1.5732,
+      "nll_loss": 1.6914262771606445,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.05088699609041214,
+      "rewards/margins": 0.00835130549967289,
+      "rewards/rejected": -0.05923830717802048,
+      "step": 110
+    },
+    {
+      "epoch": 0.7301587301587301,
+      "grad_norm": 22.67607879638672,
+      "learning_rate": 4.662524041201569e-06,
+      "log_odds_chosen": 0.5929769277572632,
+      "log_odds_ratio": -0.5206801295280457,
+      "logits/chosen": 403.47882080078125,
+      "logits/rejected": 381.8203125,
+      "logps/chosen": -1.0154887437820435,
+      "logps/rejected": -1.3891681432724,
+      "loss": 1.5678,
+      "nll_loss": 1.5397897958755493,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.050774432718753815,
+      "rewards/margins": 0.01868397556245327,
+      "rewards/rejected": -0.06945841014385223,
+      "step": 115
+    },
+    {
+      "epoch": 0.7619047619047619,
+      "grad_norm": 19.351375579833984,
+      "learning_rate": 4.564354645876385e-06,
+      "log_odds_chosen": 0.18771110475063324,
+      "log_odds_ratio": -0.7007285356521606,
+      "logits/chosen": 403.85211181640625,
+      "logits/rejected": 380.380859375,
+      "logps/chosen": -0.9934407472610474,
+      "logps/rejected": -1.1065788269042969,
+      "loss": 1.4796,
+      "nll_loss": 1.3748087882995605,
+      "rewards/accuracies": 0.44999998807907104,
+      "rewards/chosen": -0.04967203736305237,
+      "rewards/margins": 0.005656905937939882,
+      "rewards/rejected": -0.05532894283533096,
+      "step": 120
+    },
+    {
+      "epoch": 0.7936507936507936,
+      "grad_norm": 82.66716003417969,
+      "learning_rate": 4.47213595499958e-06,
+      "log_odds_chosen": 0.24183085560798645,
+      "log_odds_ratio": -0.7099695801734924,
+      "logits/chosen": 414.0562438964844,
+      "logits/rejected": 379.0338439941406,
+      "logps/chosen": -1.1751952171325684,
+      "logps/rejected": -1.3275480270385742,
+      "loss": 1.557,
+      "nll_loss": 1.7125810384750366,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.05875976011157036,
+      "rewards/margins": 0.007617638912051916,
+      "rewards/rejected": -0.06637740135192871,
+      "step": 125
+    },
+    {
+      "epoch": 0.8253968253968254,
+      "grad_norm": 77.14762115478516,
+      "learning_rate": 4.385290096535147e-06,
+      "log_odds_chosen": -0.02768993005156517,
+      "log_odds_ratio": -0.8408949971199036,
+      "logits/chosen": 408.2793273925781,
+      "logits/rejected": 362.93756103515625,
+      "logps/chosen": -1.183045744895935,
+      "logps/rejected": -1.150722861289978,
+      "loss": 1.5911,
+      "nll_loss": 1.608017921447754,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": -0.05915228649973869,
+      "rewards/margins": -0.0016161398962140083,
+      "rewards/rejected": -0.05753614753484726,
+      "step": 130
+    },
+    {
+      "epoch": 0.8571428571428571,
+      "grad_norm": 38.051902770996094,
+      "learning_rate": 4.303314829119352e-06,
+      "log_odds_chosen": 0.4119636118412018,
+      "log_odds_ratio": -0.6424974203109741,
+      "logits/chosen": 396.2297058105469,
+      "logits/rejected": 415.40289306640625,
+      "logps/chosen": -1.023511290550232,
+      "logps/rejected": -1.3654203414916992,
+      "loss": 1.4791,
+      "nll_loss": 1.5638678073883057,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.0511755645275116,
+      "rewards/margins": 0.017095452174544334,
+      "rewards/rejected": -0.06827102601528168,
+      "step": 135
+    },
+    {
+      "epoch": 0.8888888888888888,
+      "grad_norm": 18.50411605834961,
+      "learning_rate": 4.2257712736425835e-06,
+      "log_odds_chosen": 0.5941201448440552,
+      "log_odds_ratio": -0.629595160484314,
+      "logits/chosen": 382.5362548828125,
+      "logits/rejected": 439.97161865234375,
+      "logps/chosen": -1.0793473720550537,
+      "logps/rejected": -1.5278475284576416,
+      "loss": 1.4732,
+      "nll_loss": 1.4305188655853271,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.053967367857694626,
+      "rewards/margins": 0.022425005212426186,
+      "rewards/rejected": -0.07639236748218536,
+      "step": 140
+    },
+    {
+      "epoch": 0.9206349206349206,
+      "grad_norm": 29.88001251220703,
+      "learning_rate": 4.1522739926869985e-06,
+      "log_odds_chosen": 0.43660497665405273,
+      "log_odds_ratio": -0.6272520422935486,
+      "logits/chosen": 405.6814270019531,
+      "logits/rejected": 429.1936950683594,
+      "logps/chosen": -1.0074222087860107,
+      "logps/rejected": -1.279060959815979,
+      "loss": 1.429,
+      "nll_loss": 1.431378722190857,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.05037111043930054,
+      "rewards/margins": 0.013581933453679085,
+      "rewards/rejected": -0.06395303457975388,
+      "step": 145
+    },
+    {
+      "epoch": 0.9523809523809523,
+      "grad_norm": 21.189523696899414,
+      "learning_rate": 4.082482904638631e-06,
+      "log_odds_chosen": 0.09152127802371979,
+      "log_odds_ratio": -0.7851961851119995,
+      "logits/chosen": 413.9974670410156,
+      "logits/rejected": 395.56390380859375,
+      "logps/chosen": -1.102851390838623,
+      "logps/rejected": -1.1990187168121338,
+      "loss": 1.4701,
+      "nll_loss": 1.5602357387542725,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.05514257028698921,
+      "rewards/margins": 0.004808364901691675,
+      "rewards/rejected": -0.05995092913508415,
+      "step": 150
+    },
+    {
+      "epoch": 0.9841269841269841,
+      "grad_norm": 46.408260345458984,
+      "learning_rate": 4.016096644512495e-06,
+      "log_odds_chosen": 0.1732434332370758,
+      "log_odds_ratio": -0.7032361626625061,
+      "logits/chosen": 387.8311767578125,
+      "logits/rejected": 406.3460693359375,
+      "logps/chosen": -1.0525623559951782,
+      "logps/rejected": -1.2012317180633545,
+      "loss": 1.4516,
+      "nll_loss": 1.3833885192871094,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.05262812227010727,
+      "rewards/margins": 0.007433467544615269,
+      "rewards/rejected": -0.06006159260869026,
+      "step": 155
+    },
+    {
+      "epoch": 0.9968253968253968,
+      "eval_log_odds_chosen": 0.19107408821582794,
+      "eval_log_odds_ratio": -0.6924421191215515,
+      "eval_logits/chosen": 321.94952392578125,
+      "eval_logits/rejected": 293.8872375488281,
+      "eval_logps/chosen": -1.0260426998138428,
+      "eval_logps/rejected": -1.1547330617904663,
+      "eval_loss": 1.4765217304229736,
+      "eval_nll_loss": 1.4281681776046753,
+      "eval_rewards/accuracies": 0.5467625856399536,
+      "eval_rewards/chosen": -0.051302142441272736,
+      "eval_rewards/margins": 0.006434514187276363,
+      "eval_rewards/rejected": -0.05773665010929108,
+      "eval_runtime": 112.3635,
+      "eval_samples_per_second": 4.922,
+      "eval_steps_per_second": 1.237,
+      "step": 157
+    },
+    {
+      "epoch": 1.0158730158730158,
+      "grad_norm": 25.751989364624023,
+      "learning_rate": 3.952847075210474e-06,
+      "log_odds_chosen": 0.12412871420383453,
+      "log_odds_ratio": -0.7145590782165527,
+      "logits/chosen": 387.1231384277344,
+      "logits/rejected": 377.3012390136719,
+      "logps/chosen": -0.9075422286987305,
+      "logps/rejected": -0.9930707812309265,
+      "loss": 1.2728,
+      "nll_loss": 1.2198539972305298,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.04537711292505264,
+      "rewards/margins": 0.004276423715054989,
+      "rewards/rejected": -0.049653537571430206,
+      "step": 160
+    },
+    {
+      "epoch": 1.0476190476190477,
+      "grad_norm": 32.28386688232422,
+      "learning_rate": 3.892494720807615e-06,
+      "log_odds_chosen": 1.30362868309021,
+      "log_odds_ratio": -0.3633436858654022,
+      "logits/chosen": 391.3251953125,
+      "logits/rejected": 433.5608825683594,
+      "logps/chosen": -0.6450636982917786,
+      "logps/rejected": -1.4025341272354126,
+      "loss": 1.0741,
+      "nll_loss": 1.082463264465332,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.03225318342447281,
+      "rewards/margins": 0.0378735177218914,
+      "rewards/rejected": -0.07012669742107391,
+      "step": 165
+    },
+    {
+      "epoch": 1.0793650793650793,
+      "grad_norm": 15.754105567932129,
+      "learning_rate": 3.834824944236852e-06,
+      "log_odds_chosen": 1.1329152584075928,
+      "log_odds_ratio": -0.408731073141098,
+      "logits/chosen": 365.712158203125,
+      "logits/rejected": 407.73284912109375,
+      "logps/chosen": -0.6770078539848328,
+      "logps/rejected": -1.3867766857147217,
+      "loss": 1.0344,
+      "nll_loss": 0.9729477167129517,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.03385039418935776,
+      "rewards/margins": 0.035488441586494446,
+      "rewards/rejected": -0.0693388432264328,
+      "step": 170
+    },
+    {
+      "epoch": 1.1111111111111112,
+      "grad_norm": 38.7343864440918,
+      "learning_rate": 3.7796447300922724e-06,
+      "log_odds_chosen": 1.06243896484375,
+      "log_odds_ratio": -0.3898783326148987,
+      "logits/chosen": 381.17822265625,
+      "logits/rejected": 380.6576232910156,
+      "logps/chosen": -0.6825309991836548,
+      "logps/rejected": -1.2429835796356201,
+      "loss": 1.1273,
+      "nll_loss": 1.1054210662841797,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -0.03412654995918274,
+      "rewards/margins": 0.028022626414895058,
+      "rewards/rejected": -0.062149178236722946,
+      "step": 175
+    },
+    {
+      "epoch": 1.1428571428571428,
+      "grad_norm": 41.509403228759766,
+      "learning_rate": 3.72677996249965e-06,
+      "log_odds_chosen": 0.7714899182319641,
+      "log_odds_ratio": -0.48040905594825745,
+      "logits/chosen": 385.80029296875,
+      "logits/rejected": 383.924560546875,
+      "logps/chosen": -0.8479583859443665,
+      "logps/rejected": -1.3133970499038696,
+      "loss": 1.1486,
+      "nll_loss": 1.1066267490386963,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.042397916316986084,
+      "rewards/margins": 0.023271935060620308,
+      "rewards/rejected": -0.06566984951496124,
+      "step": 180
+    },
+    {
+      "epoch": 1.1746031746031746,
+      "grad_norm": 33.1015510559082,
+      "learning_rate": 3.6760731104690393e-06,
+      "log_odds_chosen": 0.9662767648696899,
+      "log_odds_ratio": -0.42013198137283325,
+      "logits/chosen": 390.68731689453125,
+      "logits/rejected": 372.36456298828125,
+      "logps/chosen": -0.6732779741287231,
+      "logps/rejected": -1.1919136047363281,
+      "loss": 1.1507,
+      "nll_loss": 1.2062638998031616,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -0.03366389125585556,
+      "rewards/margins": 0.025931786745786667,
+      "rewards/rejected": -0.059595685452222824,
+      "step": 185
+    },
+    {
+      "epoch": 1.2063492063492063,
+      "grad_norm": 14.4358491897583,
+      "learning_rate": 3.6273812505500587e-06,
+      "log_odds_chosen": 0.8507558703422546,
+      "log_odds_ratio": -0.5280704498291016,
+      "logits/chosen": 412.74951171875,
+      "logits/rejected": 407.2632751464844,
+      "logps/chosen": -0.7739854454994202,
+      "logps/rejected": -1.3370511531829834,
+      "loss": 1.0636,
+      "nll_loss": 1.1027393341064453,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.03869926929473877,
+      "rewards/margins": 0.028153279796242714,
+      "rewards/rejected": -0.06685255467891693,
+      "step": 190
+    },
+    {
+      "epoch": 1.2380952380952381,
+      "grad_norm": 20.473995208740234,
+      "learning_rate": 3.5805743701971648e-06,
+      "log_odds_chosen": 1.13797128200531,
+      "log_odds_ratio": -0.38396361470222473,
+      "logits/chosen": 414.3319396972656,
+      "logits/rejected": 398.20440673828125,
+      "logps/chosen": -0.6320561170578003,
+      "logps/rejected": -1.2892091274261475,
+      "loss": 1.074,
+      "nll_loss": 1.0247658491134644,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -0.031602807343006134,
+      "rewards/margins": 0.032857660204172134,
+      "rewards/rejected": -0.06446046382188797,
+      "step": 195
+    },
+    {
+      "epoch": 1.2698412698412698,
+      "grad_norm": 17.97160530090332,
+      "learning_rate": 3.5355339059327378e-06,
+      "log_odds_chosen": 0.6695777773857117,
+      "log_odds_ratio": -0.5161210894584656,
+      "logits/chosen": 372.59912109375,
+      "logits/rejected": 398.4673156738281,
+      "logps/chosen": -0.7804095149040222,
+      "logps/rejected": -1.1648569107055664,
+      "loss": 1.0069,
+      "nll_loss": 1.056370735168457,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.03902047872543335,
+      "rewards/margins": 0.019222375005483627,
+      "rewards/rejected": -0.05824284628033638,
+      "step": 200
+    },
+    {
+      "epoch": 1.3015873015873016,
+      "grad_norm": 15.39059066772461,
+      "learning_rate": 3.4921514788478916e-06,
+      "log_odds_chosen": 0.9029768109321594,
+      "log_odds_ratio": -0.45210033655166626,
+      "logits/chosen": 383.56378173828125,
+      "logits/rejected": 386.5069274902344,
+      "logps/chosen": -0.7651145458221436,
+      "logps/rejected": -1.2839988470077515,
+      "loss": 1.1163,
+      "nll_loss": 1.1502236127853394,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.0382557287812233,
+      "rewards/margins": 0.025944212451577187,
+      "rewards/rejected": -0.06419993937015533,
+      "step": 205
+    },
+    {
+      "epoch": 1.3333333333333333,
+      "grad_norm": 14.983006477355957,
+      "learning_rate": 3.450327796711771e-06,
+      "log_odds_chosen": 1.0162965059280396,
+      "log_odds_ratio": -0.42147356271743774,
+      "logits/chosen": 394.2404479980469,
+      "logits/rejected": 420.30450439453125,
+      "logps/chosen": -0.7215703725814819,
+      "logps/rejected": -1.2990248203277588,
+      "loss": 1.056,
+      "nll_loss": 0.9170693159103394,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -0.036078520119190216,
+      "rewards/margins": 0.02887272834777832,
+      "rewards/rejected": -0.06495124846696854,
+      "step": 210
+    },
+    {
+      "epoch": 1.3650793650793651,
+      "grad_norm": 13.9975004196167,
+      "learning_rate": 3.409971697352368e-06,
+      "log_odds_chosen": 0.8547927141189575,
+      "log_odds_ratio": -0.4758794903755188,
+      "logits/chosen": 415.505859375,
+      "logits/rejected": 414.4971618652344,
+      "logps/chosen": -0.7432714700698853,
+      "logps/rejected": -1.159825086593628,
+      "loss": 1.0231,
+      "nll_loss": 0.9773567318916321,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.03716357797384262,
+      "rewards/margins": 0.020827675238251686,
+      "rewards/rejected": -0.05799124762415886,
+      "step": 215
+    },
+    {
+      "epoch": 1.3968253968253967,
+      "grad_norm": 16.370100021362305,
+      "learning_rate": 3.3709993123162106e-06,
+      "log_odds_chosen": 0.9633774757385254,
+      "log_odds_ratio": -0.4300917685031891,
+      "logits/chosen": 405.0268859863281,
+      "logits/rejected": 405.39898681640625,
+      "logps/chosen": -0.6296164393424988,
+      "logps/rejected": -1.0896862745285034,
+      "loss": 1.0243,
+      "nll_loss": 0.9427892565727234,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -0.0314808264374733,
+      "rewards/margins": 0.02300349436700344,
+      "rewards/rejected": -0.05448431894183159,
+      "step": 220
+    },
+    {
+      "epoch": 1.4285714285714286,
+      "grad_norm": 21.732885360717773,
+      "learning_rate": 3.3333333333333333e-06,
+      "log_odds_chosen": 0.8722143173217773,
+      "log_odds_ratio": -0.49761050939559937,
+      "logits/chosen": 373.18170166015625,
+      "logits/rejected": 408.52191162109375,
+      "logps/chosen": -0.7673987150192261,
+      "logps/rejected": -1.3063347339630127,
+      "loss": 1.0781,
+      "nll_loss": 1.0205880403518677,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.03836994245648384,
+      "rewards/margins": 0.026946794241666794,
+      "rewards/rejected": -0.06531673669815063,
+      "step": 225
+    },
+    {
+      "epoch": 1.4603174603174602,
+      "grad_norm": 26.658767700195312,
+      "learning_rate": 3.296902366978936e-06,
+      "log_odds_chosen": 1.3574587106704712,
+      "log_odds_ratio": -0.3206990659236908,
+      "logits/chosen": 397.1745910644531,
+      "logits/rejected": 435.58380126953125,
+      "logps/chosen": -0.5968003869056702,
+      "logps/rejected": -1.3607866764068604,
+      "loss": 1.0374,
+      "nll_loss": 1.0697901248931885,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.02984001860022545,
+      "rewards/margins": 0.03819932043552399,
+      "rewards/rejected": -0.06803934276103973,
+      "step": 230
+    },
+    {
+      "epoch": 1.492063492063492,
+      "grad_norm": 13.732707977294922,
+      "learning_rate": 3.2616403652672114e-06,
+      "log_odds_chosen": 0.7236745357513428,
+      "log_odds_ratio": -0.5223321914672852,
+      "logits/chosen": 354.2861022949219,
+      "logits/rejected": 388.4797668457031,
+      "logps/chosen": -0.7881089448928833,
+      "logps/rejected": -1.2045094966888428,
+      "loss": 1.1557,
+      "nll_loss": 1.0742206573486328,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.0394054539501667,
+      "rewards/margins": 0.020820021629333496,
+      "rewards/rejected": -0.0602254755795002,
+      "step": 235
+    },
+    {
+      "epoch": 1.5238095238095237,
+      "grad_norm": 22.118778228759766,
+      "learning_rate": 3.2274861218395142e-06,
+      "log_odds_chosen": 1.1263599395751953,
+      "log_odds_ratio": -0.43313202261924744,
+      "logits/chosen": 382.83905029296875,
+      "logits/rejected": 387.61065673828125,
+      "logps/chosen": -0.7860811948776245,
+      "logps/rejected": -1.4334101676940918,
+      "loss": 1.1383,
+      "nll_loss": 1.2601253986358643,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.039304058998823166,
+      "rewards/margins": 0.032366447150707245,
+      "rewards/rejected": -0.07167050242424011,
+      "step": 240
+    },
+    {
+      "epoch": 1.5555555555555556,
+      "grad_norm": 22.131145477294922,
+      "learning_rate": 3.1943828249997e-06,
+      "log_odds_chosen": 0.7136174440383911,
+      "log_odds_ratio": -0.5149211883544922,
+      "logits/chosen": 355.15948486328125,
+      "logits/rejected": 387.1158447265625,
+      "logps/chosen": -0.8465463519096375,
+      "logps/rejected": -1.2772014141082764,
+      "loss": 1.0981,
+      "nll_loss": 1.2311415672302246,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.042327314615249634,
+      "rewards/margins": 0.021532755345106125,
+      "rewards/rejected": -0.06386007368564606,
+      "step": 245
+    },
+    {
+      "epoch": 1.5873015873015874,
+      "grad_norm": 19.921703338623047,
+      "learning_rate": 3.1622776601683796e-06,
+      "log_odds_chosen": 1.1693449020385742,
+      "log_odds_ratio": -0.3576763868331909,
+      "logits/chosen": 408.05621337890625,
+      "logits/rejected": 368.35400390625,
+      "logps/chosen": -0.6127609014511108,
+      "logps/rejected": -1.2678160667419434,
+      "loss": 1.0755,
+      "nll_loss": 0.9913685917854309,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.030638042837381363,
+      "rewards/margins": 0.032752759754657745,
+      "rewards/rejected": -0.0633908063173294,
+      "step": 250
+    },
+    {
+      "epoch": 1.619047619047619,
+      "grad_norm": 17.49940299987793,
+      "learning_rate": 3.131121455425748e-06,
+      "log_odds_chosen": 0.8926092982292175,
+      "log_odds_ratio": -0.4545460343360901,
+      "logits/chosen": 421.48529052734375,
+      "logits/rejected": 387.22784423828125,
+      "logps/chosen": -0.6890762448310852,
+      "logps/rejected": -1.1581443548202515,
+      "loss": 1.0777,
+      "nll_loss": 1.068838357925415,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.03445381671190262,
+      "rewards/margins": 0.023453406989574432,
+      "rewards/rejected": -0.057907216250896454,
+      "step": 255
+    },
+    {
+      "epoch": 1.6507936507936507,
+      "grad_norm": 13.594038009643555,
+      "learning_rate": 3.1008683647302113e-06,
+      "log_odds_chosen": 1.0179307460784912,
+      "log_odds_ratio": -0.4250411093235016,
+      "logits/chosen": 393.05145263671875,
+      "logits/rejected": 409.2987365722656,
+      "logps/chosen": -0.7133018970489502,
+      "logps/rejected": -1.2928727865219116,
+      "loss": 1.0383,
+      "nll_loss": 1.0420652627944946,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -0.03566509857773781,
+      "rewards/margins": 0.02897855080664158,
+      "rewards/rejected": -0.06464364379644394,
+      "step": 260
+    },
+    {
+      "epoch": 1.6825396825396826,
+      "grad_norm": 37.5067138671875,
+      "learning_rate": 3.0714755841697565e-06,
+      "log_odds_chosen": 0.7450331449508667,
+      "log_odds_ratio": -0.469769150018692,
+      "logits/chosen": 389.87579345703125,
+      "logits/rejected": 387.38641357421875,
+      "logps/chosen": -0.7393844723701477,
+      "logps/rejected": -1.1401046514511108,
+      "loss": 1.0411,
+      "nll_loss": 1.1084836721420288,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.036969222128391266,
+      "rewards/margins": 0.020036008208990097,
+      "rewards/rejected": -0.05700523406267166,
+      "step": 265
+    },
+    {
+      "epoch": 1.7142857142857144,
+      "grad_norm": 22.182912826538086,
+      "learning_rate": 3.0429030972509227e-06,
+      "log_odds_chosen": 0.946363627910614,
+      "log_odds_ratio": -0.44000881910324097,
+      "logits/chosen": 394.14337158203125,
+      "logits/rejected": 425.6368103027344,
+      "logps/chosen": -0.7661867141723633,
+      "logps/rejected": -1.3347783088684082,
+      "loss": 1.0739,
+      "nll_loss": 1.0289955139160156,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.03830933943390846,
+      "rewards/margins": 0.028429577127099037,
+      "rewards/rejected": -0.06673891097307205,
+      "step": 270
+    },
+    {
+      "epoch": 1.746031746031746,
+      "grad_norm": 23.890846252441406,
+      "learning_rate": 3.0151134457776365e-06,
+      "log_odds_chosen": 0.8923860788345337,
+      "log_odds_ratio": -0.5077377557754517,
+      "logits/chosen": 385.6656799316406,
+      "logits/rejected": 381.90765380859375,
+      "logps/chosen": -0.9714023470878601,
+      "logps/rejected": -1.5488981008529663,
+      "loss": 1.0839,
+      "nll_loss": 1.2609059810638428,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.04857012256979942,
+      "rewards/margins": 0.028874799609184265,
+      "rewards/rejected": -0.07744492590427399,
+      "step": 275
+    },
+    {
+      "epoch": 1.7777777777777777,
+      "grad_norm": 21.916139602661133,
+      "learning_rate": 2.988071523335984e-06,
+      "log_odds_chosen": 0.737246036529541,
+      "log_odds_ratio": -0.4812200665473938,
+      "logits/chosen": 386.85882568359375,
+      "logits/rejected": 401.51556396484375,
+      "logps/chosen": -0.7044110894203186,
+      "logps/rejected": -1.1333626508712769,
+      "loss": 1.1711,
+      "nll_loss": 1.0592542886734009,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.03522055223584175,
+      "rewards/margins": 0.02144758589565754,
+      "rewards/rejected": -0.05666813999414444,
+      "step": 280
+    },
+    {
+      "epoch": 1.8095238095238095,
+      "grad_norm": 52.16273498535156,
+      "learning_rate": 2.961744388795462e-06,
+      "log_odds_chosen": 0.9238273501396179,
+      "log_odds_ratio": -0.533433735370636,
+      "logits/chosen": 430.734375,
+      "logits/rejected": 403.882568359375,
+      "logps/chosen": -0.823104977607727,
+      "logps/rejected": -1.353060007095337,
+      "loss": 1.0421,
+      "nll_loss": 1.0632542371749878,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.041155245155096054,
+      "rewards/margins": 0.026497751474380493,
+      "rewards/rejected": -0.06765300035476685,
+      "step": 285
+    },
+    {
+      "epoch": 1.8412698412698414,
+      "grad_norm": 30.704998016357422,
+      "learning_rate": 2.9361010975735177e-06,
+      "log_odds_chosen": 0.9045122861862183,
+      "log_odds_ratio": -0.44358786940574646,
+      "logits/chosen": 413.3941955566406,
+      "logits/rejected": 385.8519592285156,
+      "logps/chosen": -0.7694595456123352,
+      "logps/rejected": -1.312978982925415,
+      "loss": 1.1363,
+      "nll_loss": 1.063130497932434,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.0384729765355587,
+      "rewards/margins": 0.027175968512892723,
+      "rewards/rejected": -0.06564894318580627,
+      "step": 290
+    },
+    {
+      "epoch": 1.873015873015873,
+      "grad_norm": 14.101789474487305,
+      "learning_rate": 2.9111125486979104e-06,
+      "log_odds_chosen": 0.9633495211601257,
+      "log_odds_ratio": -0.4407424032688141,
+      "logits/chosen": 402.0762634277344,
+      "logits/rejected": 433.57135009765625,
+      "logps/chosen": -0.7939198613166809,
+      "logps/rejected": -1.3206735849380493,
+      "loss": 1.0649,
+      "nll_loss": 1.0656565427780151,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.039695993065834045,
+      "rewards/margins": 0.026337692514061928,
+      "rewards/rejected": -0.06603368371725082,
+      "step": 295
+    },
+    {
+      "epoch": 1.9047619047619047,
+      "grad_norm": 14.737664222717285,
+      "learning_rate": 2.8867513459481293e-06,
+      "log_odds_chosen": 0.8915547132492065,
+      "log_odds_ratio": -0.4351634085178375,
+      "logits/chosen": 394.14019775390625,
+      "logits/rejected": 398.04486083984375,
+      "logps/chosen": -0.6770254373550415,
+      "logps/rejected": -1.136715292930603,
+      "loss": 1.0148,
+      "nll_loss": 1.0366181135177612,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -0.03385127708315849,
+      "rewards/margins": 0.022984493523836136,
+      "rewards/rejected": -0.05683577060699463,
+      "step": 300
+    },
+    {
+      "epoch": 1.9365079365079365,
+      "grad_norm": 27.61500358581543,
+      "learning_rate": 2.862991671569341e-06,
+      "log_odds_chosen": 1.2255324125289917,
+      "log_odds_ratio": -0.3640300929546356,
+      "logits/chosen": 414.54364013671875,
+      "logits/rejected": 399.3366394042969,
+      "logps/chosen": -0.5758011937141418,
+      "logps/rejected": -1.1583391427993774,
+      "loss": 0.9944,
+      "nll_loss": 0.8838338851928711,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.028790060430765152,
+      "rewards/margins": 0.029126901179552078,
+      "rewards/rejected": -0.05791696161031723,
+      "step": 305
+    },
+    {
+      "epoch": 1.9682539682539684,
+      "grad_norm": 22.889514923095703,
+      "learning_rate": 2.839809171235324e-06,
+      "log_odds_chosen": 0.8731952905654907,
+      "log_odds_ratio": -0.4226910471916199,
+      "logits/chosen": 386.8481140136719,
+      "logits/rejected": 391.4911804199219,
+      "logps/chosen": -0.7098206877708435,
+      "logps/rejected": -1.2461992502212524,
+      "loss": 1.068,
+      "nll_loss": 1.1014317274093628,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -0.035491038113832474,
+      "rewards/margins": 0.026818927377462387,
+      "rewards/rejected": -0.06230996176600456,
+      "step": 310
+    },
+    {
+      "epoch": 2.0,
+      "grad_norm": 14.210800170898438,
+      "learning_rate": 2.817180849095055e-06,
+      "log_odds_chosen": 1.0788153409957886,
+      "log_odds_ratio": -0.42032232880592346,
+      "logits/chosen": 411.44830322265625,
+      "logits/rejected": 397.8619384765625,
+      "logps/chosen": -0.7904238104820251,
+      "logps/rejected": -1.4402620792388916,
+      "loss": 1.0587,
+      "nll_loss": 1.1283180713653564,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -0.03952118754386902,
+      "rewards/margins": 0.03249191492795944,
+      "rewards/rejected": -0.07201310247182846,
+      "step": 315
+    },
+    {
+      "epoch": 2.0,
+      "eval_log_odds_chosen": 0.27226492762565613,
+      "eval_log_odds_ratio": -0.6900704503059387,
+      "eval_logits/chosen": 323.6011962890625,
+      "eval_logits/rejected": 296.0849914550781,
+      "eval_logps/chosen": -1.003529667854309,
+      "eval_logps/rejected": -1.1904489994049072,
+      "eval_loss": 1.4250426292419434,
+      "eval_nll_loss": 1.372891902923584,
+      "eval_rewards/accuracies": 0.5467625856399536,
+      "eval_rewards/chosen": -0.0501764751970768,
+      "eval_rewards/margins": 0.009345975704491138,
+      "eval_rewards/rejected": -0.05952245742082596,
+      "eval_runtime": 112.3866,
+      "eval_samples_per_second": 4.921,
+      "eval_steps_per_second": 1.237,
+      "step": 315
+    },
+    {
+      "epoch": 2.0317460317460316,
+      "grad_norm": 15.507851600646973,
+      "learning_rate": 2.7950849718747376e-06,
+      "log_odds_chosen": 2.808565616607666,
+      "log_odds_ratio": -0.13379335403442383,
+      "logits/chosen": 376.21978759765625,
+      "logits/rejected": 409.29669189453125,
+      "logps/chosen": -0.34599769115448,
+      "logps/rejected": -1.854657530784607,
+      "loss": 0.5967,
+      "nll_loss": 0.5440115928649902,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.01729988493025303,
+      "rewards/margins": 0.07543299347162247,
+      "rewards/rejected": -0.09273288398981094,
+      "step": 320
+    },
+    {
+      "epoch": 2.0634920634920633,
+      "grad_norm": 14.76868724822998,
+      "learning_rate": 2.773500981126146e-06,
+      "log_odds_chosen": 2.1307175159454346,
+      "log_odds_ratio": -0.16102290153503418,
+      "logits/chosen": 384.79876708984375,
+      "logits/rejected": 401.5932312011719,
+      "logps/chosen": -0.3530585765838623,
+      "logps/rejected": -1.3687174320220947,
+      "loss": 0.6419,
+      "nll_loss": 0.5659872889518738,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.017652926966547966,
+      "rewards/margins": 0.0507829412817955,
+      "rewards/rejected": -0.06843587011098862,
+      "step": 325
+    },
+    {
+      "epoch": 2.0952380952380953,
+      "grad_norm": 16.631784439086914,
+      "learning_rate": 2.752409412815902e-06,
+      "log_odds_chosen": 2.310250759124756,
+      "log_odds_ratio": -0.15331122279167175,
+      "logits/chosen": 407.6214599609375,
+      "logits/rejected": 376.6868896484375,
+      "logps/chosen": -0.41628751158714294,
+      "logps/rejected": -1.6159404516220093,
+      "loss": 0.6052,
+      "nll_loss": 0.6574462652206421,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.020814374089241028,
+      "rewards/margins": 0.059982649981975555,
+      "rewards/rejected": -0.08079702407121658,
+      "step": 330
+    },
+    {
+      "epoch": 2.126984126984127,
+      "grad_norm": 13.462658882141113,
+      "learning_rate": 2.7317918235407652e-06,
+      "log_odds_chosen": 2.4282920360565186,
+      "log_odds_ratio": -0.13053010404109955,
+      "logits/chosen": 353.52349853515625,
+      "logits/rejected": 368.1820983886719,
+      "logps/chosen": -0.41353076696395874,
+      "logps/rejected": -1.7517569065093994,
+      "loss": 0.5669,
+      "nll_loss": 0.6166477203369141,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.020676542073488235,
+      "rewards/margins": 0.06691131740808487,
+      "rewards/rejected": -0.08758784830570221,
+      "step": 335
+    },
+    {
+      "epoch": 2.1587301587301586,
+      "grad_norm": 14.838258743286133,
+      "learning_rate": 2.711630722733202e-06,
+      "log_odds_chosen": 2.658790111541748,
+      "log_odds_ratio": -0.11425229161977768,
+      "logits/chosen": 371.021484375,
+      "logits/rejected": 371.3381042480469,
+      "logps/chosen": -0.36502301692962646,
+      "logps/rejected": -1.8417142629623413,
+      "loss": 0.5366,
+      "nll_loss": 0.5441598296165466,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.018251152709126472,
+      "rewards/margins": 0.07383455336093903,
+      "rewards/rejected": -0.09208571910858154,
+      "step": 340
+    },
+    {
+      "epoch": 2.1904761904761907,
+      "grad_norm": 13.042370796203613,
+      "learning_rate": 2.691909510290828e-06,
+      "log_odds_chosen": 2.3122012615203857,
+      "log_odds_ratio": -0.1497458517551422,
+      "logits/chosen": 391.87457275390625,
+      "logits/rejected": 377.2791442871094,
+      "logps/chosen": -0.38829633593559265,
+      "logps/rejected": -1.5641629695892334,
+      "loss": 0.5496,
+      "nll_loss": 0.6075534224510193,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.019414816051721573,
+      "rewards/margins": 0.058793336153030396,
+      "rewards/rejected": -0.07820815593004227,
+      "step": 345
+    },
+    {
+      "epoch": 2.2222222222222223,
+      "grad_norm": 13.684109687805176,
+      "learning_rate": 2.6726124191242444e-06,
+      "log_odds_chosen": 2.6677346229553223,
+      "log_odds_ratio": -0.10940170288085938,
+      "logits/chosen": 390.1587829589844,
+      "logits/rejected": 384.64398193359375,
+      "logps/chosen": -0.34501951932907104,
+      "logps/rejected": -1.7201154232025146,
+      "loss": 0.5817,
+      "nll_loss": 0.5362960696220398,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.017250975593924522,
+      "rewards/margins": 0.06875480711460114,
+      "rewards/rejected": -0.08600577712059021,
+      "step": 350
+    },
+    {
+      "epoch": 2.253968253968254,
+      "grad_norm": 12.086108207702637,
+      "learning_rate": 2.6537244621713765e-06,
+      "log_odds_chosen": 2.147876262664795,
+      "log_odds_ratio": -0.16266413033008575,
+      "logits/chosen": 391.79998779296875,
+      "logits/rejected": 370.3006286621094,
+      "logps/chosen": -0.34506654739379883,
+      "logps/rejected": -1.4374301433563232,
+      "loss": 0.5589,
+      "nll_loss": 0.5755339860916138,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.017253328114748,
+      "rewards/margins": 0.054618168622255325,
+      "rewards/rejected": -0.07187150418758392,
+      "step": 355
+    },
+    {
+      "epoch": 2.2857142857142856,
+      "grad_norm": 13.608001708984375,
+      "learning_rate": 2.6352313834736496e-06,
+      "log_odds_chosen": 2.511486768722534,
+      "log_odds_ratio": -0.14479351043701172,
+      "logits/chosen": 381.4727478027344,
+      "logits/rejected": 367.880859375,
+      "logps/chosen": -0.33382540941238403,
+      "logps/rejected": -1.6793336868286133,
+      "loss": 0.6084,
+      "nll_loss": 0.6348077058792114,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.01669127121567726,
+      "rewards/margins": 0.06727541238069534,
+      "rewards/rejected": -0.0839666873216629,
+      "step": 360
+    },
+    {
+      "epoch": 2.317460317460317,
+      "grad_norm": 12.76559829711914,
+      "learning_rate": 2.6171196129510684e-06,
+      "log_odds_chosen": 2.4732162952423096,
+      "log_odds_ratio": -0.13545522093772888,
+      "logits/chosen": 364.4020080566406,
+      "logits/rejected": 389.8330383300781,
+      "logps/chosen": -0.32329806685447693,
+      "logps/rejected": -1.6308807134628296,
+      "loss": 0.5702,
+      "nll_loss": 0.5807278752326965,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.016164902597665787,
+      "rewards/margins": 0.06537913531064987,
+      "rewards/rejected": -0.08154404163360596,
+      "step": 365
+    },
+    {
+      "epoch": 2.3492063492063493,
+      "grad_norm": 11.652225494384766,
+      "learning_rate": 2.599376224550182e-06,
+      "log_odds_chosen": 2.5551929473876953,
+      "log_odds_ratio": -0.12840549647808075,
+      "logits/chosen": 341.6715087890625,
+      "logits/rejected": 366.8011169433594,
+      "logps/chosen": -0.3068418800830841,
+      "logps/rejected": -1.6018937826156616,
+      "loss": 0.5513,
+      "nll_loss": 0.5314286947250366,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.015342095866799355,
+      "rewards/margins": 0.06475259363651276,
+      "rewards/rejected": -0.08009468764066696,
+      "step": 370
+    },
+    {
+      "epoch": 2.380952380952381,
+      "grad_norm": 11.979321479797363,
+      "learning_rate": 2.5819888974716113e-06,
+      "log_odds_chosen": 2.403555393218994,
+      "log_odds_ratio": -0.2127949744462967,
+      "logits/chosen": 380.73468017578125,
+      "logits/rejected": 381.8941650390625,
+      "logps/chosen": -0.3469754755496979,
+      "logps/rejected": -1.4457371234893799,
+      "loss": 0.5535,
+      "nll_loss": 0.6116215586662292,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -0.017348771914839745,
+      "rewards/margins": 0.05493808910250664,
+      "rewards/rejected": -0.07228686660528183,
+      "step": 375
+    },
+    {
+      "epoch": 2.4126984126984126,
+      "grad_norm": 14.060809135437012,
+      "learning_rate": 2.564945880212886e-06,
+      "log_odds_chosen": 2.586824893951416,
+      "log_odds_ratio": -0.11490654945373535,
+      "logits/chosen": 339.2264709472656,
+      "logits/rejected": 333.75042724609375,
+      "logps/chosen": -0.2667796015739441,
+      "logps/rejected": -1.5076611042022705,
+      "loss": 0.555,
+      "nll_loss": 0.563228964805603,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.01333898026496172,
+      "rewards/margins": 0.06204408407211304,
+      "rewards/rejected": -0.07538305968046188,
+      "step": 380
+    },
+    {
+      "epoch": 2.4444444444444446,
+      "grad_norm": 13.985331535339355,
+      "learning_rate": 2.5482359571881276e-06,
+      "log_odds_chosen": 2.4178810119628906,
+      "log_odds_ratio": -0.13190138339996338,
+      "logits/chosen": 361.1400146484375,
+      "logits/rejected": 399.1631164550781,
+      "logps/chosen": -0.3266194462776184,
+      "logps/rejected": -1.4809812307357788,
+      "loss": 0.5341,
+      "nll_loss": 0.5168324708938599,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.01633097417652607,
+      "rewards/margins": 0.05771808698773384,
+      "rewards/rejected": -0.07404907047748566,
+      "step": 385
+    },
+    {
+      "epoch": 2.4761904761904763,
+      "grad_norm": 18.084671020507812,
+      "learning_rate": 2.5318484177091667e-06,
+      "log_odds_chosen": 2.383100986480713,
+      "log_odds_ratio": -0.14292851090431213,
+      "logits/chosen": 384.72064208984375,
+      "logits/rejected": 393.7874755859375,
+      "logps/chosen": -0.32157301902770996,
+      "logps/rejected": -1.5480461120605469,
+      "loss": 0.6103,
+      "nll_loss": 0.5748914480209351,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.01607864908874035,
+      "rewards/margins": 0.061323653906583786,
+      "rewards/rejected": -0.07740230858325958,
+      "step": 390
+    },
+    {
+      "epoch": 2.507936507936508,
+      "grad_norm": 14.601240158081055,
+      "learning_rate": 2.515773027133138e-06,
+      "log_odds_chosen": 2.7805123329162598,
+      "log_odds_ratio": -0.09744001924991608,
+      "logits/chosen": 387.84619140625,
+      "logits/rejected": 379.5089111328125,
+      "logps/chosen": -0.29952239990234375,
+      "logps/rejected": -1.6519800424575806,
+      "loss": 0.6068,
+      "nll_loss": 0.5597054958343506,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.014976121485233307,
+      "rewards/margins": 0.06762289255857468,
+      "rewards/rejected": -0.08259901404380798,
+      "step": 395
+    },
+    {
+      "epoch": 2.5396825396825395,
+      "grad_norm": 12.65790843963623,
+      "learning_rate": 2.5e-06,
+      "log_odds_chosen": 2.2969400882720947,
+      "log_odds_ratio": -0.15153826773166656,
+      "logits/chosen": 380.21063232421875,
+      "logits/rejected": 387.6795959472656,
+      "logps/chosen": -0.4159126281738281,
+      "logps/rejected": -1.649945855140686,
+      "loss": 0.5659,
+      "nll_loss": 0.6253801584243774,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.020795632153749466,
+      "rewards/margins": 0.06170165538787842,
+      "rewards/rejected": -0.08249729126691818,
+      "step": 400
+    },
+    {
+      "epoch": 2.571428571428571,
+      "grad_norm": 14.444682121276855,
+      "learning_rate": 2.484519974999767e-06,
+      "log_odds_chosen": 2.648660182952881,
+      "log_odds_ratio": -0.12657761573791504,
+      "logits/chosen": 393.61175537109375,
+      "logits/rejected": 366.6102600097656,
+      "logps/chosen": -0.3193502426147461,
+      "logps/rejected": -1.483344316482544,
+      "loss": 0.5605,
+      "nll_loss": 0.534288763999939,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.015967514365911484,
+      "rewards/margins": 0.058199692517519,
+      "rewards/rejected": -0.07416721433401108,
+      "step": 405
+    },
+    {
+      "epoch": 2.6031746031746033,
+      "grad_norm": 16.570829391479492,
+      "learning_rate": 2.4693239916239746e-06,
+      "log_odds_chosen": 2.558866500854492,
+      "log_odds_ratio": -0.11873920261859894,
+      "logits/chosen": 402.86456298828125,
+      "logits/rejected": 378.17303466796875,
+      "logps/chosen": -0.3188033699989319,
+      "logps/rejected": -1.5745189189910889,
+      "loss": 0.5904,
+      "nll_loss": 0.5366710424423218,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.015940170735120773,
+      "rewards/margins": 0.06278578191995621,
+      "rewards/rejected": -0.07872594892978668,
+      "step": 410
+    },
+    {
+      "epoch": 2.634920634920635,
+      "grad_norm": 14.746502876281738,
+      "learning_rate": 2.4544034683690802e-06,
+      "log_odds_chosen": 2.3469109535217285,
+      "log_odds_ratio": -0.18333213031291962,
+      "logits/chosen": 368.54840087890625,
+      "logits/rejected": 411.5265197753906,
+      "logps/chosen": -0.3403250277042389,
+      "logps/rejected": -1.5103747844696045,
+      "loss": 0.5125,
+      "nll_loss": 0.4973570704460144,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.017016252502799034,
+      "rewards/margins": 0.05850248783826828,
+      "rewards/rejected": -0.07551874220371246,
+      "step": 415
+    },
+    {
+      "epoch": 2.6666666666666665,
+      "grad_norm": 13.026257514953613,
+      "learning_rate": 2.4397501823713327e-06,
+      "log_odds_chosen": 2.4256062507629395,
+      "log_odds_ratio": -0.13080844283103943,
+      "logits/chosen": 361.3508605957031,
+      "logits/rejected": 377.66217041015625,
+      "logps/chosen": -0.30100154876708984,
+      "logps/rejected": -1.4869256019592285,
+      "loss": 0.5528,
+      "nll_loss": 0.5275372266769409,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.015050077810883522,
+      "rewards/margins": 0.059296198189258575,
+      "rewards/rejected": -0.07434628158807755,
+      "step": 420
+    },
+    {
+      "epoch": 2.6984126984126986,
+      "grad_norm": 13.664224624633789,
+      "learning_rate": 2.4253562503633297e-06,
+      "log_odds_chosen": 2.5698065757751465,
+      "log_odds_ratio": -0.11280637979507446,
+      "logits/chosen": 358.09954833984375,
+      "logits/rejected": 362.4793395996094,
+      "logps/chosen": -0.35316431522369385,
+      "logps/rejected": -1.7075494527816772,
+      "loss": 0.5529,
+      "nll_loss": 0.5961888432502747,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.017658215016126633,
+      "rewards/margins": 0.06771925836801529,
+      "rewards/rejected": -0.08537746965885162,
+      "step": 425
+    },
+    {
+      "epoch": 2.7301587301587302,
+      "grad_norm": 10.971692085266113,
+      "learning_rate": 2.411214110852061e-06,
+      "log_odds_chosen": 2.5541086196899414,
+      "log_odds_ratio": -0.133527934551239,
+      "logits/chosen": 403.82427978515625,
+      "logits/rejected": 415.45831298828125,
+      "logps/chosen": -0.3685951828956604,
+      "logps/rejected": -1.7137283086776733,
+      "loss": 0.6179,
+      "nll_loss": 0.5651038885116577,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.01842975988984108,
+      "rewards/margins": 0.06725665926933289,
+      "rewards/rejected": -0.08568642288446426,
+      "step": 430
+    },
+    {
+      "epoch": 2.761904761904762,
+      "grad_norm": 10.986706733703613,
+      "learning_rate": 2.3973165074269213e-06,
+      "log_odds_chosen": 2.3214781284332275,
+      "log_odds_ratio": -0.13725291192531586,
+      "logits/chosen": 365.35345458984375,
+      "logits/rejected": 361.9271545410156,
+      "logps/chosen": -0.30945223569869995,
+      "logps/rejected": -1.4199343919754028,
+      "loss": 0.5953,
+      "nll_loss": 0.45835572481155396,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.015472611412405968,
+      "rewards/margins": 0.055524103343486786,
+      "rewards/rejected": -0.0709967166185379,
+      "step": 435
+    },
+    {
+      "epoch": 2.7936507936507935,
+      "grad_norm": 10.5348539352417,
+      "learning_rate": 2.3836564731139807e-06,
+      "log_odds_chosen": 2.33512020111084,
+      "log_odds_ratio": -0.15043067932128906,
+      "logits/chosen": 387.08831787109375,
+      "logits/rejected": 410.5296936035156,
+      "logps/chosen": -0.39346036314964294,
+      "logps/rejected": -1.6411031484603882,
+      "loss": 0.5485,
+      "nll_loss": 0.6781072616577148,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.019673021510243416,
+      "rewards/margins": 0.06238213926553726,
+      "rewards/rejected": -0.08205515891313553,
+      "step": 440
+    },
+    {
+      "epoch": 2.825396825396825,
+      "grad_norm": 16.081439971923828,
+      "learning_rate": 2.3702273156998867e-06,
+      "log_odds_chosen": 2.88311505317688,
+      "log_odds_ratio": -0.0930914506316185,
+      "logits/chosen": 375.8430480957031,
+      "logits/rejected": 376.3010559082031,
+      "logps/chosen": -0.26819559931755066,
+      "logps/rejected": -1.6618140935897827,
+      "loss": 0.5927,
+      "nll_loss": 0.5575067400932312,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.013409781269729137,
+      "rewards/margins": 0.06968092918395996,
+      "rewards/rejected": -0.08309070765972137,
+      "step": 445
+    },
+    {
+      "epoch": 2.857142857142857,
+      "grad_norm": 12.238554954528809,
+      "learning_rate": 2.357022603955159e-06,
+      "log_odds_chosen": 2.187112808227539,
+      "log_odds_ratio": -0.17866849899291992,
+      "logits/chosen": 380.0975036621094,
+      "logits/rejected": 422.848876953125,
+      "logps/chosen": -0.3514227867126465,
+      "logps/rejected": -1.5105705261230469,
+      "loss": 0.5869,
+      "nll_loss": 0.5251952409744263,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.017571140080690384,
+      "rewards/margins": 0.05795738101005554,
+      "rewards/rejected": -0.07552852481603622,
+      "step": 450
+    },
+    {
+      "epoch": 2.888888888888889,
+      "grad_norm": 11.653218269348145,
+      "learning_rate": 2.3440361546924774e-06,
+      "log_odds_chosen": 2.4360597133636475,
+      "log_odds_ratio": -0.13144996762275696,
+      "logits/chosen": 347.91571044921875,
+      "logits/rejected": 403.49285888671875,
+      "logps/chosen": -0.3146543800830841,
+      "logps/rejected": -1.518036127090454,
+      "loss": 0.5735,
+      "nll_loss": 0.6055470705032349,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.015732718631625175,
+      "rewards/margins": 0.06016908213496208,
+      "rewards/rejected": -0.07590179890394211,
+      "step": 455
+    },
+    {
+      "epoch": 2.9206349206349205,
+      "grad_norm": 12.282635688781738,
+      "learning_rate": 2.3312620206007847e-06,
+      "log_odds_chosen": 2.144893169403076,
+      "log_odds_ratio": -0.15759766101837158,
+      "logits/chosen": 398.63311767578125,
+      "logits/rejected": 387.18377685546875,
+      "logps/chosen": -0.33198612928390503,
+      "logps/rejected": -1.3607968091964722,
+      "loss": 0.532,
+      "nll_loss": 0.5102331638336182,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.01659930869936943,
+      "rewards/margins": 0.051440536975860596,
+      "rewards/rejected": -0.06803984940052032,
+      "step": 460
+    },
+    {
+      "epoch": 2.9523809523809526,
+      "grad_norm": 16.435625076293945,
+      "learning_rate": 2.3186944788008413e-06,
+      "log_odds_chosen": 2.769465446472168,
+      "log_odds_ratio": -0.11041196435689926,
+      "logits/chosen": 392.8448486328125,
+      "logits/rejected": 378.9862365722656,
+      "logps/chosen": -0.3426569700241089,
+      "logps/rejected": -1.8384767770767212,
+      "loss": 0.5493,
+      "nll_loss": 0.5335457921028137,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.017132848501205444,
+      "rewards/margins": 0.07479099929332733,
+      "rewards/rejected": -0.09192384779453278,
+      "step": 465
+    },
+    {
+      "epoch": 2.984126984126984,
+      "grad_norm": 15.835468292236328,
+      "learning_rate": 2.3063280200722128e-06,
+      "log_odds_chosen": 2.4313647747039795,
+      "log_odds_ratio": -0.13430938124656677,
+      "logits/chosen": 396.10845947265625,
+      "logits/rejected": 391.65936279296875,
+      "logps/chosen": -0.3266219198703766,
+      "logps/rejected": -1.3965808153152466,
+      "loss": 0.5897,
+      "nll_loss": 0.5770766139030457,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.01633109711110592,
+      "rewards/margins": 0.05349794775247574,
+      "rewards/rejected": -0.06982903182506561,
+      "step": 470
+    },
+    {
+      "epoch": 2.9904761904761905,
+      "eval_log_odds_chosen": 0.367882639169693,
+      "eval_log_odds_ratio": -0.6760825514793396,
+      "eval_logits/chosen": 299.86248779296875,
+      "eval_logits/rejected": 270.96282958984375,
+      "eval_logps/chosen": -1.1505995988845825,
+      "eval_logps/rejected": -1.3976730108261108,
+      "eval_loss": 1.59840989112854,
+      "eval_nll_loss": 1.5312451124191284,
+      "eval_rewards/accuracies": 0.5899280309677124,
+      "eval_rewards/chosen": -0.057529982179403305,
+      "eval_rewards/margins": 0.012353661470115185,
+      "eval_rewards/rejected": -0.06988365203142166,
+      "eval_runtime": 112.4929,
+      "eval_samples_per_second": 4.916,
+      "eval_steps_per_second": 1.236,
+      "step": 471
+    },
+    {
+      "epoch": 2.9904761904761905,
+      "step": 471,
+      "total_flos": 0.0,
+      "train_loss": 1.4937715783493788,
+      "train_runtime": 12592.9879,
+      "train_samples_per_second": 1.199,
+      "train_steps_per_second": 0.037
+    }
+  ],
+  "logging_steps": 5,
+  "max_steps": 471,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": false,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e92e0a70b4821a3be729706638722f3cde4a53ab2888e6b40d26c1da02d13a1d
+size 6712