Model save

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,79 @@
+---
+library_name: transformers
+license: gemma
+base_model: google/gemma-7b
+tags:
+- trl
+- orpo
+- generated_from_trainer
+model-index:
+- name: gemma-7b-borpo-low-quality-v5
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# gemma-7b-borpo-low-quality-v5
+
+This model is a fine-tuned version of [google/gemma-7b](https://huggingface.co/google/gemma-7b) on an unknown dataset.
+It achieves the following results on the evaluation set:
+- Loss: 2.0035
+- Rewards/chosen: -0.6455
+- Rewards/rejected: -0.7620
+- Rewards/accuracies: 0.6115
+- Rewards/margins: 0.1164
+- Logps/rejected: -1.5240
+- Logps/chosen: -1.2911
+- Logits/rejected: 259.2041
+- Logits/chosen: 292.7468
+- Nll Loss: 1.6440
+- Log Odds Ratio: -0.6769
+- Log Odds Chosen: 0.3357
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 7.5e-06
+- train_batch_size: 2
+- eval_batch_size: 1
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 4
+- gradient_accumulation_steps: 4
+- total_train_batch_size: 32
+- total_eval_batch_size: 4
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: inverse_sqrt
+- lr_scheduler_warmup_steps: 100
+- num_epochs: 3
+
+### Training results
+
+| Training Loss | Epoch  | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen | Nll Loss | Log Odds Ratio | Log Odds Chosen |
+|:-------------:|:------:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|:--------:|:--------------:|:---------------:|
+| 1.8398        | 0.9955 | 167  | 1.8982          | -0.5762        | -0.6715          | 0.5180             | 0.0953          | -1.3431        | -1.1525      | 302.9095        | 331.1552      | 1.5454   | -0.6749        | 0.2542          |
+| 1.299         | 1.9970 | 335  | 1.8186          | -0.5507        | -0.6510          | 0.5396             | 0.1003          | -1.3021        | -1.1014      | 282.3183        | 313.4974      | 1.4682   | -0.6666        | 0.3074          |
+| 0.6379        | 2.9866 | 501  | 2.0035          | -0.6455        | -0.7620          | 0.6115             | 0.1164          | -1.5240        | -1.2911      | 259.2041        | 292.7468      | 1.6440   | -0.6769        | 0.3357          |
+
+
+### Framework versions
+
+- Transformers 4.45.1
+- Pytorch 2.4.1+cu121
+- Datasets 3.0.1
+- Tokenizers 0.20.0

all_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 2.9865871833084947,
+    "total_flos": 0.0,
+    "train_loss": 1.6206341998543805,
+    "train_runtime": 33576.7022,
+    "train_samples": 5364,
+    "train_samples_per_second": 0.479,
+    "train_steps_per_second": 0.015
+}

config.json

@@ -0,0 +1,29 @@
+{
+  "_name_or_path": "google/gemma-7b",
+  "architectures": [
+    "GemmaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 106,
+  "eos_token_id": 107,
+  "head_dim": 256,
+  "hidden_act": "gelu",
+  "hidden_activation": "gelu_pytorch_tanh",
+  "hidden_size": 3072,
+  "initializer_range": 0.02,
+  "intermediate_size": 24576,
+  "max_position_embeddings": 8192,
+  "model_type": "gemma",
+  "num_attention_heads": 16,
+  "num_hidden_layers": 28,
+  "num_key_value_heads": 16,
+  "pad_token_id": 107,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 10000.0,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.45.1",
+  "use_cache": false,
+  "vocab_size": 256000
+}

generation_config.json

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 106,
+  "eos_token_id": 107,
+  "pad_token_id": 107,
+  "transformers_version": "4.45.1"
+}

model-00001-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9dd1699dd7a0d784f16b511dfcf8489742937b60b899567d162aee0de664f7a6
+size 4995496656

model-00002-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:86eb4ea180ce0cabef3ebf77c59750eca506b639805db83bdfed818efb7e0c5b
+size 4982953168

model-00003-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7b55ba663f7d8f6eda0114a11a9865040acb06634dab9ec329bc506599c49051
+size 4982953200

model-00004-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f1f4b6dbf217e406f590ac1b9692ce7dc54bd643371b8f05b66e4639df87a73c
+size 2113988336

model.safetensors.index.json

@@ -0,0 +1,261 @@
+{
+  "metadata": {
+    "total_size": 17075361792
+  },
+  "weight_map": {
+    "model.embed_tokens.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.norm.weight": "model-00004-of-00004.safetensors"
+  }
+}

runs/Sep29_15-47-27_zizgpu06.cpu.stats.ox.ac.uk/events.out.tfevents.1727621355.zizgpu06.cpu.stats.ox.ac.uk.1837986.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2c0368f6807d48e57645ac5b8a3d7472d74f93ba8c00add9584debbb4a6c97a1
+size 5465

runs/Sep29_15-57-26_zizgpu06.cpu.stats.ox.ac.uk/events.out.tfevents.1727621956.zizgpu06.cpu.stats.ox.ac.uk.1840392.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:201d325d15e8a6e919d2daa4dde8dcb8b3a4418c78db1291850fe7eeb644c068
+size 94171

special_tokens_map.json

@@ -0,0 +1,28 @@
+{
+  "additional_special_tokens": [
+    {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    }
+  ],
+  "bos_token": "<|im_start|>",
+  "eos_token": "<|im_end|>",
+  "pad_token": "<|im_end|>",
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c644094fa37d155fce4eccae9d5a12bb1723fe4439b0b6a2ebbe173755b2c218
+size 34315017

tokenizer_config.json

@@ -0,0 +1,70 @@
+{
+  "add_bos_token": false,
+  "add_eos_token": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<pad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<eos>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "<bos>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "106": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "107": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>"
+  ],
+  "bos_token": "<|im_start|>",
+  "chat_template": "{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "legacy": null,
+  "model_max_length": 2048,
+  "pad_token": "<|im_end|>",
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "GemmaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}

train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 2.9865871833084947,
+    "total_flos": 0.0,
+    "train_loss": 1.6206341998543805,
+    "train_runtime": 33576.7022,
+    "train_samples": 5364,
+    "train_samples_per_second": 0.479,
+    "train_steps_per_second": 0.015
+}

trainer_state.json

@@ -0,0 +1,1899 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 2.9865871833084947,
+  "eval_steps": 500,
+  "global_step": 501,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.029806259314456036,
+      "grad_norm": 1802.2630615234375,
+      "learning_rate": 3.75e-07,
+      "log_odds_chosen": -0.22044436633586884,
+      "log_odds_ratio": -1.0070970058441162,
+      "logits/chosen": 204.41079711914062,
+      "logits/rejected": 203.1068878173828,
+      "logps/chosen": -14.805323600769043,
+      "logps/rejected": -14.584878921508789,
+      "loss": 15.3769,
+      "nll_loss": 14.526931762695312,
+      "rewards/accuracies": 0.375,
+      "rewards/chosen": -7.4026618003845215,
+      "rewards/margins": -0.11022226512432098,
+      "rewards/rejected": -7.2924394607543945,
+      "step": 5
+    },
+    {
+      "epoch": 0.05961251862891207,
+      "grad_norm": 964.4558715820312,
+      "learning_rate": 7.5e-07,
+      "log_odds_chosen": 0.22153806686401367,
+      "log_odds_ratio": -0.7704828977584839,
+      "logits/chosen": 229.1735382080078,
+      "logits/rejected": 234.7494659423828,
+      "logps/chosen": -10.87135124206543,
+      "logps/rejected": -11.092044830322266,
+      "loss": 11.644,
+      "nll_loss": 10.984331130981445,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -5.435675621032715,
+      "rewards/margins": 0.1103477030992508,
+      "rewards/rejected": -5.546022415161133,
+      "step": 10
+    },
+    {
+      "epoch": 0.08941877794336811,
+      "grad_norm": 325.32958984375,
+      "learning_rate": 1.125e-06,
+      "log_odds_chosen": 0.0398867093026638,
+      "log_odds_ratio": -0.7268143892288208,
+      "logits/chosen": 288.3202209472656,
+      "logits/rejected": 268.65325927734375,
+      "logps/chosen": -6.712647914886475,
+      "logps/rejected": -6.752583980560303,
+      "loss": 7.2706,
+      "nll_loss": 6.687516689300537,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -3.3563239574432373,
+      "rewards/margins": 0.019967829808592796,
+      "rewards/rejected": -3.3762919902801514,
+      "step": 15
+    },
+    {
+      "epoch": 0.11922503725782414,
+      "grad_norm": 153.8790740966797,
+      "learning_rate": 1.5e-06,
+      "log_odds_chosen": 0.005181239452213049,
+      "log_odds_ratio": -0.8698928952217102,
+      "logits/chosen": 279.0413513183594,
+      "logits/rejected": 273.7066955566406,
+      "logps/chosen": -4.006048202514648,
+      "logps/rejected": -4.012642860412598,
+      "loss": 4.5269,
+      "nll_loss": 4.109046459197998,
+      "rewards/accuracies": 0.44999998807907104,
+      "rewards/chosen": -2.003024101257324,
+      "rewards/margins": 0.003297415329143405,
+      "rewards/rejected": -2.006321430206299,
+      "step": 20
+    },
+    {
+      "epoch": 0.14903129657228018,
+      "grad_norm": 110.71721649169922,
+      "learning_rate": 1.875e-06,
+      "log_odds_chosen": -0.0372297465801239,
+      "log_odds_ratio": -0.8627766370773315,
+      "logits/chosen": 340.18084716796875,
+      "logits/rejected": 349.11151123046875,
+      "logps/chosen": -2.547694206237793,
+      "logps/rejected": -2.4806361198425293,
+      "loss": 3.1921,
+      "nll_loss": 2.749494791030884,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -1.2738471031188965,
+      "rewards/margins": -0.03352922201156616,
+      "rewards/rejected": -1.2403180599212646,
+      "step": 25
+    },
+    {
+      "epoch": 0.17883755588673622,
+      "grad_norm": 75.32969665527344,
+      "learning_rate": 2.25e-06,
+      "log_odds_chosen": 0.018188167363405228,
+      "log_odds_ratio": -0.8359094858169556,
+      "logits/chosen": 373.6573791503906,
+      "logits/rejected": 400.1236572265625,
+      "logps/chosen": -2.330045223236084,
+      "logps/rejected": -2.3124964237213135,
+      "loss": 2.6112,
+      "nll_loss": 2.3758959770202637,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -1.165022611618042,
+      "rewards/margins": -0.008774325251579285,
+      "rewards/rejected": -1.1562482118606567,
+      "step": 30
+    },
+    {
+      "epoch": 0.20864381520119224,
+      "grad_norm": 45.19425964355469,
+      "learning_rate": 2.625e-06,
+      "log_odds_chosen": 0.25173941254615784,
+      "log_odds_ratio": -0.7035487294197083,
+      "logits/chosen": 391.4384765625,
+      "logits/rejected": 380.1146545410156,
+      "logps/chosen": -1.7061666250228882,
+      "logps/rejected": -1.8836677074432373,
+      "loss": 2.3484,
+      "nll_loss": 1.9240200519561768,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.8530833125114441,
+      "rewards/margins": 0.08875055611133575,
+      "rewards/rejected": -0.9418338537216187,
+      "step": 35
+    },
+    {
+      "epoch": 0.23845007451564829,
+      "grad_norm": 49.53385925292969,
+      "learning_rate": 3e-06,
+      "log_odds_chosen": 0.2747120261192322,
+      "log_odds_ratio": -0.6599891185760498,
+      "logits/chosen": 381.4653625488281,
+      "logits/rejected": 379.7186279296875,
+      "logps/chosen": -1.598753571510315,
+      "logps/rejected": -1.8254070281982422,
+      "loss": 2.2488,
+      "nll_loss": 2.010526418685913,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.7993767857551575,
+      "rewards/margins": 0.11332666873931885,
+      "rewards/rejected": -0.9127035140991211,
+      "step": 40
+    },
+    {
+      "epoch": 0.26825633383010433,
+      "grad_norm": 68.84819030761719,
+      "learning_rate": 3.3750000000000003e-06,
+      "log_odds_chosen": 0.29411372542381287,
+      "log_odds_ratio": -0.6252076625823975,
+      "logits/chosen": 395.29290771484375,
+      "logits/rejected": 404.8909912109375,
+      "logps/chosen": -1.651572823524475,
+      "logps/rejected": -1.8901698589324951,
+      "loss": 2.1795,
+      "nll_loss": 1.7518342733383179,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.8257864117622375,
+      "rewards/margins": 0.11929843574762344,
+      "rewards/rejected": -0.9450849294662476,
+      "step": 45
+    },
+    {
+      "epoch": 0.29806259314456035,
+      "grad_norm": 47.519283294677734,
+      "learning_rate": 3.75e-06,
+      "log_odds_chosen": 0.5501509308815002,
+      "log_odds_ratio": -0.5450042486190796,
+      "logits/chosen": 403.5310363769531,
+      "logits/rejected": 424.2149963378906,
+      "logps/chosen": -1.5113065242767334,
+      "logps/rejected": -1.964956521987915,
+      "loss": 2.1594,
+      "nll_loss": 1.8876510858535767,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.7556532621383667,
+      "rewards/margins": 0.22682484984397888,
+      "rewards/rejected": -0.9824782609939575,
+      "step": 50
+    },
+    {
+      "epoch": 0.32786885245901637,
+      "grad_norm": 36.98734664916992,
+      "learning_rate": 4.125e-06,
+      "log_odds_chosen": 0.26053518056869507,
+      "log_odds_ratio": -0.6607739925384521,
+      "logits/chosen": 388.755859375,
+      "logits/rejected": 381.29827880859375,
+      "logps/chosen": -1.498511552810669,
+      "logps/rejected": -1.7075812816619873,
+      "loss": 2.1979,
+      "nll_loss": 1.9259532690048218,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.7492557764053345,
+      "rewards/margins": 0.1045350432395935,
+      "rewards/rejected": -0.8537906408309937,
+      "step": 55
+    },
+    {
+      "epoch": 0.35767511177347244,
+      "grad_norm": 63.64994430541992,
+      "learning_rate": 4.5e-06,
+      "log_odds_chosen": 0.23358798027038574,
+      "log_odds_ratio": -0.6189488172531128,
+      "logits/chosen": 396.7557678222656,
+      "logits/rejected": 388.23681640625,
+      "logps/chosen": -1.4873348474502563,
+      "logps/rejected": -1.6665433645248413,
+      "loss": 2.0244,
+      "nll_loss": 1.826873540878296,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.7436674237251282,
+      "rewards/margins": 0.08960425853729248,
+      "rewards/rejected": -0.8332716822624207,
+      "step": 60
+    },
+    {
+      "epoch": 0.38748137108792846,
+      "grad_norm": 27.614044189453125,
+      "learning_rate": 4.875e-06,
+      "log_odds_chosen": 0.3461737036705017,
+      "log_odds_ratio": -0.6412385106086731,
+      "logits/chosen": 397.4202575683594,
+      "logits/rejected": 397.5688171386719,
+      "logps/chosen": -1.3429419994354248,
+      "logps/rejected": -1.6044105291366577,
+      "loss": 1.945,
+      "nll_loss": 1.5645616054534912,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.6714709997177124,
+      "rewards/margins": 0.13073426485061646,
+      "rewards/rejected": -0.8022052645683289,
+      "step": 65
+    },
+    {
+      "epoch": 0.4172876304023845,
+      "grad_norm": 86.65094757080078,
+      "learning_rate": 5.25e-06,
+      "log_odds_chosen": 0.41045960783958435,
+      "log_odds_ratio": -0.5924609303474426,
+      "logits/chosen": 376.8070983886719,
+      "logits/rejected": 384.87994384765625,
+      "logps/chosen": -1.5336923599243164,
+      "logps/rejected": -1.8701730966567993,
+      "loss": 1.997,
+      "nll_loss": 1.7713584899902344,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.7668461799621582,
+      "rewards/margins": 0.16824033856391907,
+      "rewards/rejected": -0.9350865483283997,
+      "step": 70
+    },
+    {
+      "epoch": 0.44709388971684055,
+      "grad_norm": 75.32307434082031,
+      "learning_rate": 5.625e-06,
+      "log_odds_chosen": 0.5093179941177368,
+      "log_odds_ratio": -0.5533502101898193,
+      "logits/chosen": 403.20697021484375,
+      "logits/rejected": 389.50067138671875,
+      "logps/chosen": -1.303440809249878,
+      "logps/rejected": -1.7079664468765259,
+      "loss": 1.901,
+      "nll_loss": 1.5844974517822266,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.651720404624939,
+      "rewards/margins": 0.20226280391216278,
+      "rewards/rejected": -0.8539832234382629,
+      "step": 75
+    },
+    {
+      "epoch": 0.47690014903129657,
+      "grad_norm": 52.309078216552734,
+      "learning_rate": 6e-06,
+      "log_odds_chosen": 0.5146449208259583,
+      "log_odds_ratio": -0.5773543119430542,
+      "logits/chosen": 392.9078063964844,
+      "logits/rejected": 413.81451416015625,
+      "logps/chosen": -1.4215633869171143,
+      "logps/rejected": -1.8292858600616455,
+      "loss": 1.9712,
+      "nll_loss": 1.699523687362671,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.7107816934585571,
+      "rewards/margins": 0.20386135578155518,
+      "rewards/rejected": -0.9146429300308228,
+      "step": 80
+    },
+    {
+      "epoch": 0.5067064083457526,
+      "grad_norm": 665.8851318359375,
+      "learning_rate": 6.375e-06,
+      "log_odds_chosen": 0.4371088147163391,
+      "log_odds_ratio": -0.5891958475112915,
+      "logits/chosen": 411.912109375,
+      "logits/rejected": 395.8421325683594,
+      "logps/chosen": -1.437474012374878,
+      "logps/rejected": -1.776432991027832,
+      "loss": 2.0743,
+      "nll_loss": 1.8243602514266968,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.718737006187439,
+      "rewards/margins": 0.16947945952415466,
+      "rewards/rejected": -0.888216495513916,
+      "step": 85
+    },
+    {
+      "epoch": 0.5365126676602087,
+      "grad_norm": 57.34864044189453,
+      "learning_rate": 6.750000000000001e-06,
+      "log_odds_chosen": 0.9314719438552856,
+      "log_odds_ratio": -0.49316421151161194,
+      "logits/chosen": 410.67156982421875,
+      "logits/rejected": 423.86431884765625,
+      "logps/chosen": -1.415283441543579,
+      "logps/rejected": -2.1992759704589844,
+      "loss": 1.8288,
+      "nll_loss": 1.5969065427780151,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.7076417207717896,
+      "rewards/margins": 0.39199626445770264,
+      "rewards/rejected": -1.0996379852294922,
+      "step": 90
+    },
+    {
+      "epoch": 0.5663189269746647,
+      "grad_norm": 65.98050689697266,
+      "learning_rate": 7.1249999999999995e-06,
+      "log_odds_chosen": 0.38506197929382324,
+      "log_odds_ratio": -0.5869079828262329,
+      "logits/chosen": 378.6953430175781,
+      "logits/rejected": 390.01422119140625,
+      "logps/chosen": -1.212702989578247,
+      "logps/rejected": -1.4467040300369263,
+      "loss": 1.8818,
+      "nll_loss": 1.4865398406982422,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.6063514947891235,
+      "rewards/margins": 0.1170005351305008,
+      "rewards/rejected": -0.7233520150184631,
+      "step": 95
+    },
+    {
+      "epoch": 0.5961251862891207,
+      "grad_norm": 42.52804183959961,
+      "learning_rate": 7.5e-06,
+      "log_odds_chosen": 0.6413298845291138,
+      "log_odds_ratio": -0.538486659526825,
+      "logits/chosen": 412.3399353027344,
+      "logits/rejected": 447.61865234375,
+      "logps/chosen": -1.3371002674102783,
+      "logps/rejected": -1.868131399154663,
+      "loss": 1.9035,
+      "nll_loss": 1.6383569240570068,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.6685501337051392,
+      "rewards/margins": 0.2655157446861267,
+      "rewards/rejected": -0.9340656995773315,
+      "step": 100
+    },
+    {
+      "epoch": 0.6259314456035767,
+      "grad_norm": 48.187232971191406,
+      "learning_rate": 7.319250547113999e-06,
+      "log_odds_chosen": 0.7124954462051392,
+      "log_odds_ratio": -0.5506631731987,
+      "logits/chosen": 403.2848205566406,
+      "logits/rejected": 415.0179138183594,
+      "logps/chosen": -1.253138780593872,
+      "logps/rejected": -1.768273115158081,
+      "loss": 1.8545,
+      "nll_loss": 1.6297533512115479,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.626569390296936,
+      "rewards/margins": 0.25756722688674927,
+      "rewards/rejected": -0.8841365575790405,
+      "step": 105
+    },
+    {
+      "epoch": 0.6557377049180327,
+      "grad_norm": 195.05877685546875,
+      "learning_rate": 7.150969419341942e-06,
+      "log_odds_chosen": 0.4548686444759369,
+      "log_odds_ratio": -0.569240927696228,
+      "logits/chosen": 389.09381103515625,
+      "logits/rejected": 386.32513427734375,
+      "logps/chosen": -1.3020490407943726,
+      "logps/rejected": -1.6546863317489624,
+      "loss": 1.8713,
+      "nll_loss": 1.6556203365325928,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.6510245203971863,
+      "rewards/margins": 0.1763187050819397,
+      "rewards/rejected": -0.8273431658744812,
+      "step": 110
+    },
+    {
+      "epoch": 0.6855439642324889,
+      "grad_norm": 53.028961181640625,
+      "learning_rate": 6.993786061802354e-06,
+      "log_odds_chosen": 0.8375183939933777,
+      "log_odds_ratio": -0.4603613018989563,
+      "logits/chosen": 408.5062561035156,
+      "logits/rejected": 414.22332763671875,
+      "logps/chosen": -1.220641851425171,
+      "logps/rejected": -1.8461415767669678,
+      "loss": 1.8867,
+      "nll_loss": 1.583599328994751,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.6103209257125854,
+      "rewards/margins": 0.3127499222755432,
+      "rewards/rejected": -0.9230707883834839,
+      "step": 115
+    },
+    {
+      "epoch": 0.7153502235469449,
+      "grad_norm": 49.91960144042969,
+      "learning_rate": 6.846531968814577e-06,
+      "log_odds_chosen": 0.6342799663543701,
+      "log_odds_ratio": -0.5294886827468872,
+      "logits/chosen": 393.26812744140625,
+      "logits/rejected": 391.69677734375,
+      "logps/chosen": -1.1840012073516846,
+      "logps/rejected": -1.6264234781265259,
+      "loss": 1.9171,
+      "nll_loss": 1.6122639179229736,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.5920006036758423,
+      "rewards/margins": 0.22121115028858185,
+      "rewards/rejected": -0.8132117390632629,
+      "step": 120
+    },
+    {
+      "epoch": 0.7451564828614009,
+      "grad_norm": 26.490934371948242,
+      "learning_rate": 6.708203932499369e-06,
+      "log_odds_chosen": 0.3495193421840668,
+      "log_odds_ratio": -0.6338833570480347,
+      "logits/chosen": 394.9533386230469,
+      "logits/rejected": 406.28662109375,
+      "logps/chosen": -1.23225998878479,
+      "logps/rejected": -1.4777755737304688,
+      "loss": 1.8239,
+      "nll_loss": 1.5472067594528198,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.616129994392395,
+      "rewards/margins": 0.12275775521993637,
+      "rewards/rejected": -0.7388877868652344,
+      "step": 125
+    },
+    {
+      "epoch": 0.7749627421758569,
+      "grad_norm": 45.540184020996094,
+      "learning_rate": 6.577935144802719e-06,
+      "log_odds_chosen": 0.4649319052696228,
+      "log_odds_ratio": -0.5916207432746887,
+      "logits/chosen": 417.3863830566406,
+      "logits/rejected": 405.1355895996094,
+      "logps/chosen": -1.2279858589172363,
+      "logps/rejected": -1.5856168270111084,
+      "loss": 1.8861,
+      "nll_loss": 1.565514326095581,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.6139929294586182,
+      "rewards/margins": 0.17881537973880768,
+      "rewards/rejected": -0.7928084135055542,
+      "step": 130
+    },
+    {
+      "epoch": 0.8047690014903129,
+      "grad_norm": 42.16865921020508,
+      "learning_rate": 6.454972243679028e-06,
+      "log_odds_chosen": 0.46650758385658264,
+      "log_odds_ratio": -0.6158651113510132,
+      "logits/chosen": 421.04730224609375,
+      "logits/rejected": 423.00775146484375,
+      "logps/chosen": -1.509341835975647,
+      "logps/rejected": -1.8879953622817993,
+      "loss": 1.9281,
+      "nll_loss": 1.723570466041565,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.7546709179878235,
+      "rewards/margins": 0.18932664394378662,
+      "rewards/rejected": -0.9439976811408997,
+      "step": 135
+    },
+    {
+      "epoch": 0.834575260804769,
+      "grad_norm": 41.30878448486328,
+      "learning_rate": 6.338656910463875e-06,
+      "log_odds_chosen": 0.27211299538612366,
+      "log_odds_ratio": -0.680214524269104,
+      "logits/chosen": 404.1061096191406,
+      "logits/rejected": 409.54559326171875,
+      "logps/chosen": -1.4520971775054932,
+      "logps/rejected": -1.670898199081421,
+      "loss": 1.9311,
+      "nll_loss": 1.7876545190811157,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.7260485887527466,
+      "rewards/margins": 0.10940053313970566,
+      "rewards/rejected": -0.8354490995407104,
+      "step": 140
+    },
+    {
+      "epoch": 0.8643815201192251,
+      "grad_norm": 58.79216003417969,
+      "learning_rate": 6.228410989030498e-06,
+      "log_odds_chosen": 0.13751578330993652,
+      "log_odds_ratio": -0.6719902753829956,
+      "logits/chosen": 402.82342529296875,
+      "logits/rejected": 405.1291809082031,
+      "logps/chosen": -1.3680763244628906,
+      "logps/rejected": -1.4593302011489868,
+      "loss": 1.9064,
+      "nll_loss": 1.6308927536010742,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.6840381622314453,
+      "rewards/margins": 0.045626964420080185,
+      "rewards/rejected": -0.7296651005744934,
+      "step": 145
+    },
+    {
+      "epoch": 0.8941877794336811,
+      "grad_norm": 28.582611083984375,
+      "learning_rate": 6.123724356957946e-06,
+      "log_odds_chosen": 0.8705786466598511,
+      "log_odds_ratio": -0.5025649666786194,
+      "logits/chosen": 407.1871643066406,
+      "logits/rejected": 423.8992614746094,
+      "logps/chosen": -1.2447984218597412,
+      "logps/rejected": -1.9607397317886353,
+      "loss": 1.8397,
+      "nll_loss": 1.4880900382995605,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.6223992109298706,
+      "rewards/margins": 0.357970654964447,
+      "rewards/rejected": -0.9803698658943176,
+      "step": 150
+    },
+    {
+      "epoch": 0.9239940387481371,
+      "grad_norm": 56.87172317504883,
+      "learning_rate": 6.024144966768742e-06,
+      "log_odds_chosen": 0.32878491282463074,
+      "log_odds_ratio": -0.6313835382461548,
+      "logits/chosen": 388.4563293457031,
+      "logits/rejected": 402.23834228515625,
+      "logps/chosen": -1.365476131439209,
+      "logps/rejected": -1.6416184902191162,
+      "loss": 1.8579,
+      "nll_loss": 1.458609938621521,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.6827380657196045,
+      "rewards/margins": 0.13807126879692078,
+      "rewards/rejected": -0.8208092451095581,
+      "step": 155
+    },
+    {
+      "epoch": 0.9538002980625931,
+      "grad_norm": 52.100013732910156,
+      "learning_rate": 5.929270612815711e-06,
+      "log_odds_chosen": 0.2496444433927536,
+      "log_odds_ratio": -0.6904889345169067,
+      "logits/chosen": 397.15887451171875,
+      "logits/rejected": 440.45587158203125,
+      "logps/chosen": -1.2235276699066162,
+      "logps/rejected": -1.4282524585723877,
+      "loss": 1.8144,
+      "nll_loss": 1.502598762512207,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.6117638349533081,
+      "rewards/margins": 0.10236239433288574,
+      "rewards/rejected": -0.7141262292861938,
+      "step": 160
+    },
+    {
+      "epoch": 0.9836065573770492,
+      "grad_norm": 62.34568786621094,
+      "learning_rate": 5.838742081211422e-06,
+      "log_odds_chosen": 0.25045713782310486,
+      "log_odds_ratio": -0.6064125895500183,
+      "logits/chosen": 407.72943115234375,
+      "logits/rejected": 420.569580078125,
+      "logps/chosen": -1.2549211978912354,
+      "logps/rejected": -1.4284086227416992,
+      "loss": 1.8398,
+      "nll_loss": 1.5022889375686646,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.6274605989456177,
+      "rewards/margins": 0.0867437794804573,
+      "rewards/rejected": -0.7142043113708496,
+      "step": 165
+    },
+    {
+      "epoch": 0.9955290611028316,
+      "eval_log_odds_chosen": 0.25424009561538696,
+      "eval_log_odds_ratio": -0.6749479174613953,
+      "eval_logits/chosen": 331.1551513671875,
+      "eval_logits/rejected": 302.9094543457031,
+      "eval_logps/chosen": -1.1524564027786255,
+      "eval_logps/rejected": -1.343080997467041,
+      "eval_loss": 1.8982083797454834,
+      "eval_nll_loss": 1.5454045534133911,
+      "eval_rewards/accuracies": 0.5179855823516846,
+      "eval_rewards/chosen": -0.5762282013893127,
+      "eval_rewards/margins": 0.09531236439943314,
+      "eval_rewards/rejected": -0.6715404987335205,
+      "eval_runtime": 378.1633,
+      "eval_samples_per_second": 1.462,
+      "eval_steps_per_second": 0.368,
+      "step": 167
+    },
+    {
+      "epoch": 1.0134128166915053,
+      "grad_norm": 19.05902862548828,
+      "learning_rate": 5.752237416355279e-06,
+      "log_odds_chosen": 0.8472374081611633,
+      "log_odds_ratio": -0.5489901304244995,
+      "logits/chosen": 389.7673645019531,
+      "logits/rejected": 412.4244079589844,
+      "logps/chosen": -1.1339842081069946,
+      "logps/rejected": -1.749389410018921,
+      "loss": 1.6458,
+      "nll_loss": 1.2897634506225586,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.5669921040534973,
+      "rewards/margins": 0.3077026307582855,
+      "rewards/rejected": -0.8746947050094604,
+      "step": 170
+    },
+    {
+      "epoch": 1.0432190760059612,
+      "grad_norm": 39.458702087402344,
+      "learning_rate": 5.669467095138408e-06,
+      "log_odds_chosen": 1.5121970176696777,
+      "log_odds_ratio": -0.30622434616088867,
+      "logits/chosen": 365.8326416015625,
+      "logits/rejected": 406.28411865234375,
+      "logps/chosen": -0.7569233775138855,
+      "logps/rejected": -1.806539535522461,
+      "loss": 1.2547,
+      "nll_loss": 1.183380365371704,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -0.37846168875694275,
+      "rewards/margins": 0.5248081088066101,
+      "rewards/rejected": -0.9032697677612305,
+      "step": 175
+    },
+    {
+      "epoch": 1.0730253353204173,
+      "grad_norm": 31.42799949645996,
+      "learning_rate": 5.5901699437494744e-06,
+      "log_odds_chosen": 1.5321556329727173,
+      "log_odds_ratio": -0.2848857045173645,
+      "logits/chosen": 364.0169677734375,
+      "logits/rejected": 339.5393371582031,
+      "logps/chosen": -0.8578670620918274,
+      "logps/rejected": -1.9486223459243774,
+      "loss": 1.3431,
+      "nll_loss": 1.3115805387496948,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -0.4289335310459137,
+      "rewards/margins": 0.5453776717185974,
+      "rewards/rejected": -0.9743111729621887,
+      "step": 180
+    },
+    {
+      "epoch": 1.1028315946348732,
+      "grad_norm": 35.6085090637207,
+      "learning_rate": 5.514109665703559e-06,
+      "log_odds_chosen": 2.223374843597412,
+      "log_odds_ratio": -0.2399306744337082,
+      "logits/chosen": 396.5127868652344,
+      "logits/rejected": 386.45831298828125,
+      "logps/chosen": -0.8754903078079224,
+      "logps/rejected": -2.4870636463165283,
+      "loss": 1.2172,
+      "nll_loss": 1.070143222808838,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -0.4377451539039612,
+      "rewards/margins": 0.8057867288589478,
+      "rewards/rejected": -1.2435318231582642,
+      "step": 185
+    },
+    {
+      "epoch": 1.1326378539493294,
+      "grad_norm": 22.422922134399414,
+      "learning_rate": 5.441071875825088e-06,
+      "log_odds_chosen": 1.2998098134994507,
+      "log_odds_ratio": -0.32330313324928284,
+      "logits/chosen": 364.8489074707031,
+      "logits/rejected": 408.52972412109375,
+      "logps/chosen": -0.870254397392273,
+      "logps/rejected": -1.7498779296875,
+      "loss": 1.2807,
+      "nll_loss": 1.1046603918075562,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.4351271986961365,
+      "rewards/margins": 0.4398117661476135,
+      "rewards/rejected": -0.87493896484375,
+      "step": 190
+    },
+    {
+      "epoch": 1.1624441132637853,
+      "grad_norm": 25.98566246032715,
+      "learning_rate": 5.370861555295747e-06,
+      "log_odds_chosen": 2.002765417098999,
+      "log_odds_ratio": -0.24665918946266174,
+      "logits/chosen": 384.2296142578125,
+      "logits/rejected": 398.67205810546875,
+      "logps/chosen": -0.8954353332519531,
+      "logps/rejected": -2.385274648666382,
+      "loss": 1.2781,
+      "nll_loss": 1.1818829774856567,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.44771766662597656,
+      "rewards/margins": 0.7449195981025696,
+      "rewards/rejected": -1.192637324333191,
+      "step": 195
+    },
+    {
+      "epoch": 1.1922503725782414,
+      "grad_norm": 23.273193359375,
+      "learning_rate": 5.303300858899106e-06,
+      "log_odds_chosen": 2.1724612712860107,
+      "log_odds_ratio": -0.26438647508621216,
+      "logits/chosen": 413.26361083984375,
+      "logits/rejected": 396.8680419921875,
+      "logps/chosen": -0.730914831161499,
+      "logps/rejected": -2.3943963050842285,
+      "loss": 1.2803,
+      "nll_loss": 1.0805118083953857,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -0.3654574155807495,
+      "rewards/margins": 0.83174067735672,
+      "rewards/rejected": -1.1971981525421143,
+      "step": 200
+    },
+    {
+      "epoch": 1.2220566318926975,
+      "grad_norm": 27.806596755981445,
+      "learning_rate": 5.238227218271837e-06,
+      "log_odds_chosen": 2.4484591484069824,
+      "log_odds_ratio": -0.22746221721172333,
+      "logits/chosen": 365.72119140625,
+      "logits/rejected": 359.6267395019531,
+      "logps/chosen": -0.758979320526123,
+      "logps/rejected": -2.573237180709839,
+      "loss": 1.2398,
+      "nll_loss": 1.1106595993041992,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.3794896602630615,
+      "rewards/margins": 0.9071289896965027,
+      "rewards/rejected": -1.2866185903549194,
+      "step": 205
+    },
+    {
+      "epoch": 1.2518628912071534,
+      "grad_norm": 22.43109130859375,
+      "learning_rate": 5.175491695067656e-06,
+      "log_odds_chosen": 2.793255567550659,
+      "log_odds_ratio": -0.23325839638710022,
+      "logits/chosen": 377.73443603515625,
+      "logits/rejected": 404.66644287109375,
+      "logps/chosen": -0.7964418530464172,
+      "logps/rejected": -3.0091164112091064,
+      "loss": 1.2371,
+      "nll_loss": 1.0737245082855225,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -0.3982209265232086,
+      "rewards/margins": 1.106337308883667,
+      "rewards/rejected": -1.5045582056045532,
+      "step": 210
+    },
+    {
+      "epoch": 1.2816691505216096,
+      "grad_norm": 49.615081787109375,
+      "learning_rate": 5.114957546028551e-06,
+      "log_odds_chosen": 2.5913193225860596,
+      "log_odds_ratio": -0.23048730194568634,
+      "logits/chosen": 401.5857238769531,
+      "logits/rejected": 382.7472229003906,
+      "logps/chosen": -0.7830225229263306,
+      "logps/rejected": -2.782843828201294,
+      "loss": 1.2555,
+      "nll_loss": 1.1490776538848877,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -0.3915112614631653,
+      "rewards/margins": 0.9999106526374817,
+      "rewards/rejected": -1.391421914100647,
+      "step": 215
+    },
+    {
+      "epoch": 1.3114754098360657,
+      "grad_norm": 37.056453704833984,
+      "learning_rate": 5.056498968474315e-06,
+      "log_odds_chosen": 1.4505621194839478,
+      "log_odds_ratio": -0.3243867754936218,
+      "logits/chosen": 393.34063720703125,
+      "logits/rejected": 387.152099609375,
+      "logps/chosen": -0.8490570783615112,
+      "logps/rejected": -1.8110103607177734,
+      "loss": 1.2897,
+      "nll_loss": 1.1027252674102783,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -0.4245285391807556,
+      "rewards/margins": 0.48097652196884155,
+      "rewards/rejected": -0.9055051803588867,
+      "step": 220
+    },
+    {
+      "epoch": 1.3412816691505216,
+      "grad_norm": 22.3747615814209,
+      "learning_rate": 4.9999999999999996e-06,
+      "log_odds_chosen": 1.0405768156051636,
+      "log_odds_ratio": -0.37774857878685,
+      "logits/chosen": 391.2264099121094,
+      "logits/rejected": 382.2864685058594,
+      "logps/chosen": -1.0447379350662231,
+      "logps/rejected": -1.8071935176849365,
+      "loss": 1.2781,
+      "nll_loss": 1.2478437423706055,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.5223689675331116,
+      "rewards/margins": 0.38122788071632385,
+      "rewards/rejected": -0.9035967588424683,
+      "step": 225
+    },
+    {
+      "epoch": 1.3710879284649775,
+      "grad_norm": 22.041149139404297,
+      "learning_rate": 4.945353550468404e-06,
+      "log_odds_chosen": 2.0077407360076904,
+      "log_odds_ratio": -0.25288665294647217,
+      "logits/chosen": 370.07696533203125,
+      "logits/rejected": 385.3541564941406,
+      "logps/chosen": -0.90089350938797,
+      "logps/rejected": -2.3661916255950928,
+      "loss": 1.2539,
+      "nll_loss": 0.989953339099884,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -0.450446754693985,
+      "rewards/margins": 0.7326492667198181,
+      "rewards/rejected": -1.1830958127975464,
+      "step": 230
+    },
+    {
+      "epoch": 1.4008941877794336,
+      "grad_norm": 33.446224212646484,
+      "learning_rate": 4.8924605479008165e-06,
+      "log_odds_chosen": 1.8689000606536865,
+      "log_odds_ratio": -0.24617977440357208,
+      "logits/chosen": 391.69732666015625,
+      "logits/rejected": 402.88836669921875,
+      "logps/chosen": -0.8436087369918823,
+      "logps/rejected": -2.213329553604126,
+      "loss": 1.2797,
+      "nll_loss": 1.1102927923202515,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -0.42180436849594116,
+      "rewards/margins": 0.684860348701477,
+      "rewards/rejected": -1.106664776802063,
+      "step": 235
+    },
+    {
+      "epoch": 1.4307004470938898,
+      "grad_norm": 19.861835479736328,
+      "learning_rate": 4.8412291827592715e-06,
+      "log_odds_chosen": 1.848902940750122,
+      "log_odds_ratio": -0.3047091066837311,
+      "logits/chosen": 415.73486328125,
+      "logits/rejected": 420.25909423828125,
+      "logps/chosen": -0.8824371099472046,
+      "logps/rejected": -2.3095035552978516,
+      "loss": 1.2891,
+      "nll_loss": 1.1334781646728516,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -0.4412185549736023,
+      "rewards/margins": 0.7135332226753235,
+      "rewards/rejected": -1.1547517776489258,
+      "step": 240
+    },
+    {
+      "epoch": 1.4605067064083457,
+      "grad_norm": 31.12167739868164,
+      "learning_rate": 4.79157423749955e-06,
+      "log_odds_chosen": 1.9949026107788086,
+      "log_odds_ratio": -0.24574021995067596,
+      "logits/chosen": 409.1098937988281,
+      "logits/rejected": 394.39593505859375,
+      "logps/chosen": -0.7639743089675903,
+      "logps/rejected": -2.1869406700134277,
+      "loss": 1.312,
+      "nll_loss": 1.2122286558151245,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -0.38198715448379517,
+      "rewards/margins": 0.7114832997322083,
+      "rewards/rejected": -1.0934703350067139,
+      "step": 245
+    },
+    {
+      "epoch": 1.4903129657228018,
+      "grad_norm": 20.75017547607422,
+      "learning_rate": 4.74341649025257e-06,
+      "log_odds_chosen": 2.6609864234924316,
+      "log_odds_ratio": -0.25564372539520264,
+      "logits/chosen": 380.4820251464844,
+      "logits/rejected": 385.7266845703125,
+      "logps/chosen": -0.804089367389679,
+      "logps/rejected": -2.9035263061523438,
+      "loss": 1.2332,
+      "nll_loss": 1.0138413906097412,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.4020446836948395,
+      "rewards/margins": 1.0497183799743652,
+      "rewards/rejected": -1.4517631530761719,
+      "step": 250
+    },
+    {
+      "epoch": 1.520119225037258,
+      "grad_norm": 26.518312454223633,
+      "learning_rate": 4.696682183138622e-06,
+      "log_odds_chosen": 2.0447866916656494,
+      "log_odds_ratio": -0.22285214066505432,
+      "logits/chosen": 399.64971923828125,
+      "logits/rejected": 400.3153076171875,
+      "logps/chosen": -0.8079016804695129,
+      "logps/rejected": -2.2692599296569824,
+      "loss": 1.2551,
+      "nll_loss": 1.0453466176986694,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -0.40395084023475647,
+      "rewards/margins": 0.7306792140007019,
+      "rewards/rejected": -1.1346299648284912,
+      "step": 255
+    },
+    {
+      "epoch": 1.5499254843517138,
+      "grad_norm": 18.25040626525879,
+      "learning_rate": 4.6513025470953165e-06,
+      "log_odds_chosen": 1.7279441356658936,
+      "log_odds_ratio": -0.3020802140235901,
+      "logits/chosen": 380.86590576171875,
+      "logits/rejected": 420.64007568359375,
+      "logps/chosen": -0.8852326273918152,
+      "logps/rejected": -2.209996461868286,
+      "loss": 1.2132,
+      "nll_loss": 1.1014916896820068,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -0.4426163136959076,
+      "rewards/margins": 0.6623818278312683,
+      "rewards/rejected": -1.104998230934143,
+      "step": 260
+    },
+    {
+      "epoch": 1.5797317436661698,
+      "grad_norm": 17.494176864624023,
+      "learning_rate": 4.607213376254634e-06,
+      "log_odds_chosen": 1.6819546222686768,
+      "log_odds_ratio": -0.3340403437614441,
+      "logits/chosen": 388.51068115234375,
+      "logits/rejected": 409.89971923828125,
+      "logps/chosen": -0.8424839973449707,
+      "logps/rejected": -2.041083574295044,
+      "loss": 1.3572,
+      "nll_loss": 1.1232020854949951,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.42124199867248535,
+      "rewards/margins": 0.5992996692657471,
+      "rewards/rejected": -1.020541787147522,
+      "step": 265
+    },
+    {
+      "epoch": 1.6095380029806259,
+      "grad_norm": 34.47447967529297,
+      "learning_rate": 4.564354645876384e-06,
+      "log_odds_chosen": 1.859960913658142,
+      "log_odds_ratio": -0.2928032875061035,
+      "logits/chosen": 372.35052490234375,
+      "logits/rejected": 382.39117431640625,
+      "logps/chosen": -0.9743008613586426,
+      "logps/rejected": -2.432216167449951,
+      "loss": 1.3166,
+      "nll_loss": 1.270026683807373,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -0.4871504306793213,
+      "rewards/margins": 0.7289576530456543,
+      "rewards/rejected": -1.2161080837249756,
+      "step": 270
+    },
+    {
+      "epoch": 1.639344262295082,
+      "grad_norm": 21.639741897583008,
+      "learning_rate": 4.522670168666455e-06,
+      "log_odds_chosen": 1.463616967201233,
+      "log_odds_ratio": -0.3027709126472473,
+      "logits/chosen": 362.27044677734375,
+      "logits/rejected": 350.851318359375,
+      "logps/chosen": -0.7592640519142151,
+      "logps/rejected": -1.6749483346939087,
+      "loss": 1.2876,
+      "nll_loss": 1.0598416328430176,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -0.37963202595710754,
+      "rewards/margins": 0.45784205198287964,
+      "rewards/rejected": -0.8374741673469543,
+      "step": 275
+    },
+    {
+      "epoch": 1.669150521609538,
+      "grad_norm": 32.02434158325195,
+      "learning_rate": 4.482107285003976e-06,
+      "log_odds_chosen": 1.598225474357605,
+      "log_odds_ratio": -0.3603498339653015,
+      "logits/chosen": 410.98687744140625,
+      "logits/rejected": 397.2725830078125,
+      "logps/chosen": -0.8709872364997864,
+      "logps/rejected": -1.9590165615081787,
+      "loss": 1.285,
+      "nll_loss": 1.1194555759429932,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -0.4354936182498932,
+      "rewards/margins": 0.5440146923065186,
+      "rewards/rejected": -0.9795082807540894,
+      "step": 280
+    },
+    {
+      "epoch": 1.698956780923994,
+      "grad_norm": 18.135896682739258,
+      "learning_rate": 4.442616583193192e-06,
+      "log_odds_chosen": 2.131542921066284,
+      "log_odds_ratio": -0.2719998061656952,
+      "logits/chosen": 375.759033203125,
+      "logits/rejected": 377.7268981933594,
+      "logps/chosen": -0.7118241786956787,
+      "logps/rejected": -2.2590417861938477,
+      "loss": 1.2113,
+      "nll_loss": 1.054491400718689,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -0.35591208934783936,
+      "rewards/margins": 0.7736088037490845,
+      "rewards/rejected": -1.1295208930969238,
+      "step": 285
+    },
+    {
+      "epoch": 1.7287630402384502,
+      "grad_norm": 16.36552619934082,
+      "learning_rate": 4.404151646360276e-06,
+      "log_odds_chosen": 2.539419651031494,
+      "log_odds_ratio": -0.2885150611400604,
+      "logits/chosen": 395.09832763671875,
+      "logits/rejected": 429.5597229003906,
+      "logps/chosen": -0.8963809013366699,
+      "logps/rejected": -2.95475697517395,
+      "loss": 1.2765,
+      "nll_loss": 1.1113842725753784,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -0.44819045066833496,
+      "rewards/margins": 1.0291881561279297,
+      "rewards/rejected": -1.477378487586975,
+      "step": 290
+    },
+    {
+      "epoch": 1.758569299552906,
+      "grad_norm": 20.132436752319336,
+      "learning_rate": 4.366668823046865e-06,
+      "log_odds_chosen": 1.553693413734436,
+      "log_odds_ratio": -0.3133796155452728,
+      "logits/chosen": 371.56475830078125,
+      "logits/rejected": 413.505615234375,
+      "logps/chosen": -0.9781205058097839,
+      "logps/rejected": -2.0690226554870605,
+      "loss": 1.2963,
+      "nll_loss": 1.1234586238861084,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.48906025290489197,
+      "rewards/margins": 0.5454510450363159,
+      "rewards/rejected": -1.0345113277435303,
+      "step": 295
+    },
+    {
+      "epoch": 1.788375558867362,
+      "grad_norm": 26.103137969970703,
+      "learning_rate": 4.330127018922194e-06,
+      "log_odds_chosen": 2.1462044715881348,
+      "log_odds_ratio": -0.24198386073112488,
+      "logits/chosen": 409.97161865234375,
+      "logits/rejected": 383.2239990234375,
+      "logps/chosen": -0.7521153688430786,
+      "logps/rejected": -2.3262906074523926,
+      "loss": 1.1963,
+      "nll_loss": 1.022304654121399,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -0.3760576844215393,
+      "rewards/margins": 0.7870877385139465,
+      "rewards/rejected": -1.1631453037261963,
+      "step": 300
+    },
+    {
+      "epoch": 1.8181818181818183,
+      "grad_norm": 20.291587829589844,
+      "learning_rate": 4.294487507354011e-06,
+      "log_odds_chosen": 1.8153495788574219,
+      "log_odds_ratio": -0.3603833019733429,
+      "logits/chosen": 399.9996032714844,
+      "logits/rejected": 405.7534484863281,
+      "logps/chosen": -1.029065728187561,
+      "logps/rejected": -2.513669490814209,
+      "loss": 1.2618,
+      "nll_loss": 1.2645294666290283,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.5145328640937805,
+      "rewards/margins": 0.742301881313324,
+      "rewards/rejected": -1.2568347454071045,
+      "step": 305
+    },
+    {
+      "epoch": 1.8479880774962743,
+      "grad_norm": 19.601377487182617,
+      "learning_rate": 4.259713756852986e-06,
+      "log_odds_chosen": 3.238518238067627,
+      "log_odds_ratio": -0.22569267451763153,
+      "logits/chosen": 380.0532531738281,
+      "logits/rejected": 386.49920654296875,
+      "logps/chosen": -0.8288024067878723,
+      "logps/rejected": -3.505189895629883,
+      "loss": 1.3118,
+      "nll_loss": 1.1829016208648682,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -0.41440120339393616,
+      "rewards/margins": 1.338193655014038,
+      "rewards/rejected": -1.7525949478149414,
+      "step": 310
+    },
+    {
+      "epoch": 1.8777943368107302,
+      "grad_norm": 19.882904052734375,
+      "learning_rate": 4.225771273642583e-06,
+      "log_odds_chosen": 1.2986266613006592,
+      "log_odds_ratio": -0.3742119073867798,
+      "logits/chosen": 354.4183044433594,
+      "logits/rejected": 373.57757568359375,
+      "logps/chosen": -1.0900706052780151,
+      "logps/rejected": -2.065725326538086,
+      "loss": 1.3407,
+      "nll_loss": 1.3649649620056152,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -0.5450353026390076,
+      "rewards/margins": 0.4878273606300354,
+      "rewards/rejected": -1.032862663269043,
+      "step": 315
+    },
+    {
+      "epoch": 1.9076005961251863,
+      "grad_norm": 21.79350471496582,
+      "learning_rate": 4.192627457812106e-06,
+      "log_odds_chosen": 2.3978288173675537,
+      "log_odds_ratio": -0.24583116173744202,
+      "logits/chosen": 379.4220886230469,
+      "logits/rejected": 400.0596618652344,
+      "logps/chosen": -0.7697328329086304,
+      "logps/rejected": -2.6053688526153564,
+      "loss": 1.1966,
+      "nll_loss": 1.0758590698242188,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -0.3848664164543152,
+      "rewards/margins": 0.9178179502487183,
+      "rewards/rejected": -1.3026844263076782,
+      "step": 320
+    },
+    {
+      "epoch": 1.9374068554396424,
+      "grad_norm": 25.364540100097656,
+      "learning_rate": 4.1602514716892185e-06,
+      "log_odds_chosen": 2.511842727661133,
+      "log_odds_ratio": -0.21811020374298096,
+      "logits/chosen": 377.62139892578125,
+      "logits/rejected": 409.9249572753906,
+      "logps/chosen": -0.8184257745742798,
+      "logps/rejected": -2.717911720275879,
+      "loss": 1.2025,
+      "nll_loss": 1.0009828805923462,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.4092128872871399,
+      "rewards/margins": 0.9497429728507996,
+      "rewards/rejected": -1.3589558601379395,
+      "step": 325
+    },
+    {
+      "epoch": 1.9672131147540983,
+      "grad_norm": 24.96152114868164,
+      "learning_rate": 4.128614119223853e-06,
+      "log_odds_chosen": 1.425945520401001,
+      "log_odds_ratio": -0.2635498642921448,
+      "logits/chosen": 370.4853515625,
+      "logits/rejected": 376.5760192871094,
+      "logps/chosen": -0.8058730959892273,
+      "logps/rejected": -1.770816445350647,
+      "loss": 1.257,
+      "nll_loss": 0.9558019638061523,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.40293654799461365,
+      "rewards/margins": 0.48247161507606506,
+      "rewards/rejected": -0.8854082226753235,
+      "step": 330
+    },
+    {
+      "epoch": 1.9970193740685542,
+      "grad_norm": 17.690673828125,
+      "learning_rate": 4.097687735311148e-06,
+      "log_odds_chosen": 1.0468049049377441,
+      "log_odds_ratio": -0.43120455741882324,
+      "logits/chosen": 390.3360900878906,
+      "logits/rejected": 377.1321105957031,
+      "logps/chosen": -1.1694064140319824,
+      "logps/rejected": -1.9213879108428955,
+      "loss": 1.299,
+      "nll_loss": 1.3618862628936768,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.5847032070159912,
+      "rewards/margins": 0.3759908080101013,
+      "rewards/rejected": -0.9606939554214478,
+      "step": 335
+    },
+    {
+      "epoch": 1.9970193740685542,
+      "eval_log_odds_chosen": 0.30739960074424744,
+      "eval_log_odds_ratio": -0.6665536761283875,
+      "eval_logits/chosen": 313.4974365234375,
+      "eval_logits/rejected": 282.3182678222656,
+      "eval_logps/chosen": -1.1013935804367065,
+      "eval_logps/rejected": -1.3020689487457275,
+      "eval_loss": 1.8186225891113281,
+      "eval_nll_loss": 1.4682071208953857,
+      "eval_rewards/accuracies": 0.5395683646202087,
+      "eval_rewards/chosen": -0.5506967902183533,
+      "eval_rewards/margins": 0.10033769905567169,
+      "eval_rewards/rejected": -0.6510344743728638,
+      "eval_runtime": 378.4505,
+      "eval_samples_per_second": 1.461,
+      "eval_steps_per_second": 0.367,
+      "step": 335
+    },
+    {
+      "epoch": 2.0268256333830106,
+      "grad_norm": 19.424360275268555,
+      "learning_rate": 4.0674460840998035e-06,
+      "log_odds_chosen": 3.000514507293701,
+      "log_odds_ratio": -0.10731367021799088,
+      "logits/chosen": 394.8528747558594,
+      "logits/rejected": 368.841796875,
+      "logps/chosen": -0.4828576147556305,
+      "logps/rejected": -2.412517547607422,
+      "loss": 0.7587,
+      "nll_loss": 0.8271492123603821,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.24142880737781525,
+      "rewards/margins": 0.9648300409317017,
+      "rewards/rejected": -1.206258773803711,
+      "step": 340
+    },
+    {
+      "epoch": 2.0566318926974665,
+      "grad_norm": 13.576729774475098,
+      "learning_rate": 4.037864265436242e-06,
+      "log_odds_chosen": 3.940922498703003,
+      "log_odds_ratio": -0.07164354622364044,
+      "logits/chosen": 342.024658203125,
+      "logits/rejected": 342.9552307128906,
+      "logps/chosen": -0.48198527097702026,
+      "logps/rejected": -3.1786067485809326,
+      "loss": 0.6501,
+      "nll_loss": 0.6376822590827942,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -0.24099263548851013,
+      "rewards/margins": 1.3483107089996338,
+      "rewards/rejected": -1.5893033742904663,
+      "step": 345
+    },
+    {
+      "epoch": 2.0864381520119224,
+      "grad_norm": 13.920299530029297,
+      "learning_rate": 4.008918628686366e-06,
+      "log_odds_chosen": 3.9218780994415283,
+      "log_odds_ratio": -0.04757709056138992,
+      "logits/chosen": 334.36187744140625,
+      "logits/rejected": 374.3761291503906,
+      "logps/chosen": -0.4404733180999756,
+      "logps/rejected": -3.2984976768493652,
+      "loss": 0.6334,
+      "nll_loss": 0.6365537643432617,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.2202366590499878,
+      "rewards/margins": 1.4290122985839844,
+      "rewards/rejected": -1.6492488384246826,
+      "step": 350
+    },
+    {
+      "epoch": 2.1162444113263787,
+      "grad_norm": 12.827689170837402,
+      "learning_rate": 3.980586693257065e-06,
+      "log_odds_chosen": 4.072226047515869,
+      "log_odds_ratio": -0.05848418548703194,
+      "logits/chosen": 343.1537170410156,
+      "logits/rejected": 358.5604248046875,
+      "logps/chosen": -0.41472020745277405,
+      "logps/rejected": -3.3263726234436035,
+      "loss": 0.6337,
+      "nll_loss": 0.6818060278892517,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.20736010372638702,
+      "rewards/margins": 1.4558260440826416,
+      "rewards/rejected": -1.6631863117218018,
+      "step": 355
+    },
+    {
+      "epoch": 2.1460506706408347,
+      "grad_norm": 14.782392501831055,
+      "learning_rate": 3.952847075210474e-06,
+      "log_odds_chosen": 4.195507049560547,
+      "log_odds_ratio": -0.03957627713680267,
+      "logits/chosen": 346.2444152832031,
+      "logits/rejected": 393.8316955566406,
+      "logps/chosen": -0.4012891352176666,
+      "logps/rejected": -3.2268481254577637,
+      "loss": 0.62,
+      "nll_loss": 0.5778239965438843,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.2006445676088333,
+      "rewards/margins": 1.4127795696258545,
+      "rewards/rejected": -1.6134240627288818,
+      "step": 360
+    },
+    {
+      "epoch": 2.1758569299552906,
+      "grad_norm": 13.0780029296875,
+      "learning_rate": 3.925679419426603e-06,
+      "log_odds_chosen": 3.8077125549316406,
+      "log_odds_ratio": -0.06930537521839142,
+      "logits/chosen": 345.62933349609375,
+      "logits/rejected": 324.48724365234375,
+      "logps/chosen": -0.416290283203125,
+      "logps/rejected": -3.0446524620056152,
+      "loss": 0.6211,
+      "nll_loss": 0.5697866678237915,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.2081451416015625,
+      "rewards/margins": 1.3141810894012451,
+      "rewards/rejected": -1.5223262310028076,
+      "step": 365
+    },
+    {
+      "epoch": 2.2056631892697465,
+      "grad_norm": 13.796915054321289,
+      "learning_rate": 3.8990643368252725e-06,
+      "log_odds_chosen": 3.758181095123291,
+      "log_odds_ratio": -0.08863835036754608,
+      "logits/chosen": 309.7760009765625,
+      "logits/rejected": 327.28875732421875,
+      "logps/chosen": -0.4810813367366791,
+      "logps/rejected": -3.211531400680542,
+      "loss": 0.6805,
+      "nll_loss": 0.6794728636741638,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.24054066836833954,
+      "rewards/margins": 1.365225076675415,
+      "rewards/rejected": -1.605765700340271,
+      "step": 370
+    },
+    {
+      "epoch": 2.235469448584203,
+      "grad_norm": 12.518247604370117,
+      "learning_rate": 3.872983346207417e-06,
+      "log_odds_chosen": 3.9930922985076904,
+      "log_odds_ratio": -0.07049717009067535,
+      "logits/chosen": 364.5960388183594,
+      "logits/rejected": 381.1838684082031,
+      "logps/chosen": -0.47075748443603516,
+      "logps/rejected": -3.3296546936035156,
+      "loss": 0.6191,
+      "nll_loss": 0.6227518320083618,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.23537874221801758,
+      "rewards/margins": 1.4294484853744507,
+      "rewards/rejected": -1.6648273468017578,
+      "step": 375
+    },
+    {
+      "epoch": 2.2652757078986587,
+      "grad_norm": 23.583436965942383,
+      "learning_rate": 3.847418820319329e-06,
+      "log_odds_chosen": 3.55949330329895,
+      "log_odds_ratio": -0.048723362386226654,
+      "logits/chosen": 364.0585632324219,
+      "logits/rejected": 337.66424560546875,
+      "logps/chosen": -0.41810473799705505,
+      "logps/rejected": -2.7338831424713135,
+      "loss": 0.6367,
+      "nll_loss": 0.6199051737785339,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.20905236899852753,
+      "rewards/margins": 1.1578892469406128,
+      "rewards/rejected": -1.3669415712356567,
+      "step": 380
+    },
+    {
+      "epoch": 2.2950819672131146,
+      "grad_norm": 12.274104118347168,
+      "learning_rate": 3.822353935782191e-06,
+      "log_odds_chosen": 4.226048946380615,
+      "log_odds_ratio": -0.04617181420326233,
+      "logits/chosen": 359.08154296875,
+      "logits/rejected": 344.7435607910156,
+      "logps/chosen": -0.36728304624557495,
+      "logps/rejected": -3.1797947883605957,
+      "loss": 0.5982,
+      "nll_loss": 0.536335289478302,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.18364152312278748,
+      "rewards/margins": 1.4062559604644775,
+      "rewards/rejected": -1.5898973941802979,
+      "step": 385
+    },
+    {
+      "epoch": 2.3248882265275705,
+      "grad_norm": 15.543499946594238,
+      "learning_rate": 3.79777262656375e-06,
+      "log_odds_chosen": 3.8094513416290283,
+      "log_odds_ratio": -0.03931097313761711,
+      "logits/chosen": 373.632568359375,
+      "logits/rejected": 393.21295166015625,
+      "logps/chosen": -0.46378618478775024,
+      "logps/rejected": -3.2327797412872314,
+      "loss": 0.667,
+      "nll_loss": 0.6460134387016296,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.23189309239387512,
+      "rewards/margins": 1.3844965696334839,
+      "rewards/rejected": -1.6163898706436157,
+      "step": 390
+    },
+    {
+      "epoch": 2.354694485842027,
+      "grad_norm": 13.162529945373535,
+      "learning_rate": 3.773659540699707e-06,
+      "log_odds_chosen": 3.9874942302703857,
+      "log_odds_ratio": -0.05443572998046875,
+      "logits/chosen": 368.2042236328125,
+      "logits/rejected": 346.73590087890625,
+      "logps/chosen": -0.3340614438056946,
+      "logps/rejected": -2.9790287017822266,
+      "loss": 0.5966,
+      "nll_loss": 0.5552054643630981,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.1670307219028473,
+      "rewards/margins": 1.3224836587905884,
+      "rewards/rejected": -1.4895143508911133,
+      "step": 395
+    },
+    {
+      "epoch": 2.384500745156483,
+      "grad_norm": 11.184009552001953,
+      "learning_rate": 3.75e-06,
+      "log_odds_chosen": 4.703558444976807,
+      "log_odds_ratio": -0.06566660106182098,
+      "logits/chosen": 368.7803955078125,
+      "logits/rejected": 383.49188232421875,
+      "logps/chosen": -0.4126485288143158,
+      "logps/rejected": -3.8882851600646973,
+      "loss": 0.619,
+      "nll_loss": 0.5621041059494019,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -0.2063242644071579,
+      "rewards/margins": 1.7378184795379639,
+      "rewards/rejected": -1.9441425800323486,
+      "step": 400
+    },
+    {
+      "epoch": 2.4143070044709387,
+      "grad_norm": 12.980825424194336,
+      "learning_rate": 3.72677996249965e-06,
+      "log_odds_chosen": 4.317046165466309,
+      "log_odds_ratio": -0.07360969483852386,
+      "logits/chosen": 422.6468200683594,
+      "logits/rejected": 380.22998046875,
+      "logps/chosen": -0.43779826164245605,
+      "logps/rejected": -3.444517135620117,
+      "loss": 0.6456,
+      "nll_loss": 0.6048913598060608,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.21889913082122803,
+      "rewards/margins": 1.5033594369888306,
+      "rewards/rejected": -1.7222585678100586,
+      "step": 405
+    },
+    {
+      "epoch": 2.444113263785395,
+      "grad_norm": 11.386286735534668,
+      "learning_rate": 3.7039859874359616e-06,
+      "log_odds_chosen": 3.6852588653564453,
+      "log_odds_ratio": -0.06955203413963318,
+      "logits/chosen": 365.75726318359375,
+      "logits/rejected": 373.08935546875,
+      "logps/chosen": -0.4394207000732422,
+      "logps/rejected": -2.8892974853515625,
+      "loss": 0.6307,
+      "nll_loss": 0.6479864120483398,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.2197103500366211,
+      "rewards/margins": 1.2249382734298706,
+      "rewards/rejected": -1.4446487426757812,
+      "step": 410
+    },
+    {
+      "epoch": 2.473919523099851,
+      "grad_norm": 18.22783088684082,
+      "learning_rate": 3.68160520255362e-06,
+      "log_odds_chosen": 4.396167755126953,
+      "log_odds_ratio": -0.04461110755801201,
+      "logits/chosen": 371.11077880859375,
+      "logits/rejected": 393.236328125,
+      "logps/chosen": -0.4007198214530945,
+      "logps/rejected": -3.539832353591919,
+      "loss": 0.6134,
+      "nll_loss": 0.5493707656860352,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.20035991072654724,
+      "rewards/margins": 1.5695561170578003,
+      "rewards/rejected": -1.7699161767959595,
+      "step": 415
+    },
+    {
+      "epoch": 2.503725782414307,
+      "grad_norm": 12.961064338684082,
+      "learning_rate": 3.6596252735569993e-06,
+      "log_odds_chosen": 3.7051539421081543,
+      "log_odds_ratio": -0.06346116960048676,
+      "logits/chosen": 368.0414123535156,
+      "logits/rejected": 338.7422180175781,
+      "logps/chosen": -0.38283970952033997,
+      "logps/rejected": -2.9152991771698,
+      "loss": 0.6314,
+      "nll_loss": 0.6913596987724304,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.19141985476016998,
+      "rewards/margins": 1.2662298679351807,
+      "rewards/rejected": -1.4576495885849,
+      "step": 420
+    },
+    {
+      "epoch": 2.533532041728763,
+      "grad_norm": 14.195425033569336,
+      "learning_rate": 3.6380343755449946e-06,
+      "log_odds_chosen": 4.560431480407715,
+      "log_odds_ratio": -0.03129279986023903,
+      "logits/chosen": 362.96746826171875,
+      "logits/rejected": 349.57958984375,
+      "logps/chosen": -0.4239065647125244,
+      "logps/rejected": -3.7524616718292236,
+      "loss": 0.6005,
+      "nll_loss": 0.6241891980171204,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.2119532823562622,
+      "rewards/margins": 1.66427743434906,
+      "rewards/rejected": -1.8762308359146118,
+      "step": 425
+    },
+    {
+      "epoch": 2.563338301043219,
+      "grad_norm": 13.435834884643555,
+      "learning_rate": 3.6168211662780912e-06,
+      "log_odds_chosen": 4.4234299659729,
+      "log_odds_ratio": -0.03766634315252304,
+      "logits/chosen": 361.0891418457031,
+      "logits/rejected": 362.5020446777344,
+      "logps/chosen": -0.3212960362434387,
+      "logps/rejected": -3.3532252311706543,
+      "loss": 0.6194,
+      "nll_loss": 0.5339005589485168,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.16064801812171936,
+      "rewards/margins": 1.5159647464752197,
+      "rewards/rejected": -1.6766126155853271,
+      "step": 430
+    },
+    {
+      "epoch": 2.593144560357675,
+      "grad_norm": 13.10000228881836,
+      "learning_rate": 3.5959747611403815e-06,
+      "log_odds_chosen": 3.667933702468872,
+      "log_odds_ratio": -0.049441076815128326,
+      "logits/chosen": 370.7209777832031,
+      "logits/rejected": 329.81048583984375,
+      "logps/chosen": -0.3787091374397278,
+      "logps/rejected": -2.830317735671997,
+      "loss": 0.6284,
+      "nll_loss": 0.5626553297042847,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.1893545687198639,
+      "rewards/margins": 1.2258042097091675,
+      "rewards/rejected": -1.4151588678359985,
+      "step": 435
+    },
+    {
+      "epoch": 2.6229508196721314,
+      "grad_norm": 16.336698532104492,
+      "learning_rate": 3.575484709670971e-06,
+      "log_odds_chosen": 4.056857585906982,
+      "log_odds_ratio": -0.035340312868356705,
+      "logits/chosen": 350.6004333496094,
+      "logits/rejected": 356.38323974609375,
+      "logps/chosen": -0.3078530430793762,
+      "logps/rejected": -2.8841094970703125,
+      "loss": 0.6605,
+      "nll_loss": 0.6005590558052063,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.1539265215396881,
+      "rewards/margins": 1.288128137588501,
+      "rewards/rejected": -1.4420547485351562,
+      "step": 440
+    },
+    {
+      "epoch": 2.6527570789865873,
+      "grad_norm": 16.911279678344727,
+      "learning_rate": 3.5553409735498294e-06,
+      "log_odds_chosen": 4.540930271148682,
+      "log_odds_ratio": -0.021845344454050064,
+      "logits/chosen": 333.2252197265625,
+      "logits/rejected": 360.4376525878906,
+      "logps/chosen": -0.3848707973957062,
+      "logps/rejected": -3.6115005016326904,
+      "loss": 0.6305,
+      "nll_loss": 0.616925060749054,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.1924353986978531,
+      "rewards/margins": 1.6133148670196533,
+      "rewards/rejected": -1.8057502508163452,
+      "step": 445
+    },
+    {
+      "epoch": 2.682563338301043,
+      "grad_norm": 12.66332721710205,
+      "learning_rate": 3.535533905932738e-06,
+      "log_odds_chosen": 3.8172874450683594,
+      "log_odds_ratio": -0.04109153896570206,
+      "logits/chosen": 363.5258483886719,
+      "logits/rejected": 355.9951171875,
+      "logps/chosen": -0.442394495010376,
+      "logps/rejected": -3.0551743507385254,
+      "loss": 0.6594,
+      "nll_loss": 0.6155456304550171,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.221197247505188,
+      "rewards/margins": 1.3063898086547852,
+      "rewards/rejected": -1.5275871753692627,
+      "step": 450
+    },
+    {
+      "epoch": 2.712369597615499,
+      "grad_norm": 11.626214981079102,
+      "learning_rate": 3.516054232038716e-06,
+      "log_odds_chosen": 3.7958292961120605,
+      "log_odds_ratio": -0.05676658824086189,
+      "logits/chosen": 394.8970947265625,
+      "logits/rejected": 370.40692138671875,
+      "logps/chosen": -0.49928539991378784,
+      "logps/rejected": -3.0073413848876953,
+      "loss": 0.6971,
+      "nll_loss": 0.6259102821350098,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.24964269995689392,
+      "rewards/margins": 1.2540279626846313,
+      "rewards/rejected": -1.5036706924438477,
+      "step": 455
+    },
+    {
+      "epoch": 2.742175856929955,
+      "grad_norm": 12.148904800415039,
+      "learning_rate": 3.496893030901177e-06,
+      "log_odds_chosen": 4.385091781616211,
+      "log_odds_ratio": -0.0362153984606266,
+      "logits/chosen": 390.84942626953125,
+      "logits/rejected": 403.558837890625,
+      "logps/chosen": -0.35050949454307556,
+      "logps/rejected": -3.5067138671875,
+      "loss": 0.64,
+      "nll_loss": 0.6797647476196289,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.17525474727153778,
+      "rewards/margins": 1.5781023502349854,
+      "rewards/rejected": -1.75335693359375,
+      "step": 460
+    },
+    {
+      "epoch": 2.7719821162444114,
+      "grad_norm": 16.0198974609375,
+      "learning_rate": 3.478041718201262e-06,
+      "log_odds_chosen": 4.281471252441406,
+      "log_odds_ratio": -0.04437633976340294,
+      "logits/chosen": 379.65142822265625,
+      "logits/rejected": 373.5622863769531,
+      "logps/chosen": -0.3376389443874359,
+      "logps/rejected": -3.2862961292266846,
+      "loss": 0.6475,
+      "nll_loss": 0.6045721769332886,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.16881947219371796,
+      "rewards/margins": 1.4743287563323975,
+      "rewards/rejected": -1.6431480646133423,
+      "step": 465
+    },
+    {
+      "epoch": 2.8017883755588673,
+      "grad_norm": 20.6240291595459,
+      "learning_rate": 3.459492030108319e-06,
+      "log_odds_chosen": 3.6953399181365967,
+      "log_odds_ratio": -0.07698026299476624,
+      "logits/chosen": 384.45758056640625,
+      "logits/rejected": 344.5814208984375,
+      "logps/chosen": -0.43422994017601013,
+      "logps/rejected": -2.9317374229431152,
+      "loss": 0.6539,
+      "nll_loss": 0.6051168441772461,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.21711497008800507,
+      "rewards/margins": 1.2487537860870361,
+      "rewards/rejected": -1.4658687114715576,
+      "step": 470
+    },
+    {
+      "epoch": 2.8315946348733236,
+      "grad_norm": 16.241731643676758,
+      "learning_rate": 3.441236008058426e-06,
+      "log_odds_chosen": 4.172859191894531,
+      "log_odds_ratio": -0.03734353557229042,
+      "logits/chosen": 359.6179504394531,
+      "logits/rejected": 372.67559814453125,
+      "logps/chosen": -0.3877125382423401,
+      "logps/rejected": -3.2986769676208496,
+      "loss": 0.6017,
+      "nll_loss": 0.5468435287475586,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.19385626912117004,
+      "rewards/margins": 1.4554822444915771,
+      "rewards/rejected": -1.6493384838104248,
+      "step": 475
+    },
+    {
+      "epoch": 2.8614008941877795,
+      "grad_norm": 13.836338996887207,
+      "learning_rate": 3.4232659844072883e-06,
+      "log_odds_chosen": 5.078559875488281,
+      "log_odds_ratio": -0.040774669498205185,
+      "logits/chosen": 369.23388671875,
+      "logits/rejected": 398.89654541015625,
+      "logps/chosen": -0.4116280674934387,
+      "logps/rejected": -4.2671427726745605,
+      "loss": 0.5823,
+      "nll_loss": 0.5545191764831543,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.20581403374671936,
+      "rewards/margins": 1.9277572631835938,
+      "rewards/rejected": -2.1335713863372803,
+      "step": 480
+    },
+    {
+      "epoch": 2.8912071535022354,
+      "grad_norm": 13.629563331604004,
+      "learning_rate": 3.4055745688987482e-06,
+      "log_odds_chosen": 3.756863832473755,
+      "log_odds_ratio": -0.05358726903796196,
+      "logits/chosen": 360.4635925292969,
+      "logits/rejected": 379.5230407714844,
+      "logps/chosen": -0.4683550000190735,
+      "logps/rejected": -3.0478286743164062,
+      "loss": 0.6356,
+      "nll_loss": 0.5991590023040771,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.23417750000953674,
+      "rewards/margins": 1.2897369861602783,
+      "rewards/rejected": -1.5239143371582031,
+      "step": 485
+    },
+    {
+      "epoch": 2.9210134128166914,
+      "grad_norm": 11.519225120544434,
+      "learning_rate": 3.3881546358946924e-06,
+      "log_odds_chosen": 4.434014320373535,
+      "log_odds_ratio": -0.048062875866889954,
+      "logits/chosen": 385.712158203125,
+      "logits/rejected": 336.68524169921875,
+      "logps/chosen": -0.45937252044677734,
+      "logps/rejected": -3.5976157188415527,
+      "loss": 0.6968,
+      "nll_loss": 0.5958720445632935,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.22968626022338867,
+      "rewards/margins": 1.5691215991973877,
+      "rewards/rejected": -1.7988078594207764,
+      "step": 490
+    },
+    {
+      "epoch": 2.9508196721311473,
+      "grad_norm": 16.139705657958984,
+      "learning_rate": 3.37099931231621e-06,
+      "log_odds_chosen": 4.680273532867432,
+      "log_odds_ratio": -0.062451474368572235,
+      "logits/chosen": 375.66119384765625,
+      "logits/rejected": 394.88690185546875,
+      "logps/chosen": -0.4953223764896393,
+      "logps/rejected": -3.950305938720703,
+      "loss": 0.6257,
+      "nll_loss": 0.628521203994751,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.24766118824481964,
+      "rewards/margins": 1.7274919748306274,
+      "rewards/rejected": -1.9751529693603516,
+      "step": 495
+    },
+    {
+      "epoch": 2.9806259314456036,
+      "grad_norm": 15.226314544677734,
+      "learning_rate": 3.3541019662496845e-06,
+      "log_odds_chosen": 4.290003776550293,
+      "log_odds_ratio": -0.04303581267595291,
+      "logits/chosen": 379.34765625,
+      "logits/rejected": 365.36773681640625,
+      "logps/chosen": -0.3512369692325592,
+      "logps/rejected": -3.2438788414001465,
+      "loss": 0.6379,
+      "nll_loss": 0.5231742858886719,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.1756184846162796,
+      "rewards/margins": 1.44632089138031,
+      "rewards/rejected": -1.6219394207000732,
+      "step": 500
+    },
+    {
+      "epoch": 2.9865871833084947,
+      "eval_log_odds_chosen": 0.33574143052101135,
+      "eval_log_odds_ratio": -0.6769095063209534,
+      "eval_logits/chosen": 292.74676513671875,
+      "eval_logits/rejected": 259.2041320800781,
+      "eval_logps/chosen": -1.2910676002502441,
+      "eval_logps/rejected": -1.5239651203155518,
+      "eval_loss": 2.003471851348877,
+      "eval_nll_loss": 1.6439613103866577,
+      "eval_rewards/accuracies": 0.6115108132362366,
+      "eval_rewards/chosen": -0.6455338001251221,
+      "eval_rewards/margins": 0.1164487823843956,
+      "eval_rewards/rejected": -0.7619825601577759,
+      "eval_runtime": 378.0521,
+      "eval_samples_per_second": 1.463,
+      "eval_steps_per_second": 0.368,
+      "step": 501
+    },
+    {
+      "epoch": 2.9865871833084947,
+      "step": 501,
+      "total_flos": 0.0,
+      "train_loss": 1.6206341998543805,
+      "train_runtime": 33576.7022,
+      "train_samples_per_second": 0.479,
+      "train_steps_per_second": 0.015
+    }
+  ],
+  "logging_steps": 5,
+  "max_steps": 501,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": false,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:02af6198c2807aea3cc5677207c4b2017143053c66b9c1841c4dce2ecc54ecf1
+size 6840