Model save

README.md

@@ -2,17 +2,10 @@
 license: apache-2.0
 library_name: peft
 tags:
-- alignment-handbook
 - trl
 - dpo
 - generated_from_trainer
-- trl
-- dpo
-- alignment-handbook
-- generated_from_trainer
 base_model: mistralai/Mistral-7B-v0.1
-datasets:
-- HuggingFaceH4/ultrafeedback_binarized
 model-index:
 - name: zephyr-7b
   results: []
@@ -23,19 +16,19 @@ should probably proofread and complete it, then remove this comment. -->
 
 # zephyr-7b
 
-This model is a fine-tuned version of [alignment-handbook/zephyr-7b-sft-qlora](https://huggingface.co/alignment-handbook/zephyr-7b-sft-qlora) on the HuggingFaceH4/ultrafeedback_binarized dataset.
+This model is a fine-tuned version of [mistralai/Mistral-7B-v0.1](https://huggingface.co/mistralai/Mistral-7B-v0.1) on the None dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.6152
-- Rewards/chosen: -0.5055
-- Rewards/rejected: -0.8740
-- Rewards/accuracies: 0.3789
-- Rewards/margins: 0.3685
-- Logps/rejected: -164.5882
-- Logps/chosen: -114.4584
-- Logits/rejected: 1.5984
-- Logits/chosen: 1.5070
-- Use Label: 0.0
-- Pred Label: 0.0
+- Loss: 0.5769
+- Rewards/chosen: -0.6646
+- Rewards/rejected: -1.1353
+- Rewards/accuracies: 0.3711
+- Rewards/margins: 0.4707
+- Logps/rejected: -190.7267
+- Logps/chosen: -130.3719
+- Logits/rejected: 1.8500
+- Logits/chosen: 1.7576
+- Use Label: 6517.1875
+- Pred Label: 782.8125
 
 ## Model description
 
@@ -72,10 +65,10 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen | Use Label | Pred Label |
 |:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|:---------:|:----------:|
-| 0.6551        | 0.21  | 100  | 0.6526          | -0.2364        | -0.3728          | 0.3359             | 0.1364          | -114.4721      | -87.5525     | -1.7460         | -1.7620       | 0.0       | 0.0        |
-| 0.6376        | 0.42  | 200  | 0.6289          | -0.3405        | -0.6072          | 0.3672             | 0.2667          | -137.9142      | -97.9614     | 0.0432          | -0.0238       | 0.0       | 0.0        |
-| 0.6196        | 0.63  | 300  | 0.6189          | -0.3871        | -0.7293          | 0.375              | 0.3422          | -150.1250      | -102.6218    | 1.1831          | 1.0945        | 0.0       | 0.0        |
-| 0.6139        | 0.84  | 400  | 0.6157          | -0.4865        | -0.8500          | 0.3711             | 0.3636          | -162.1976      | -112.5605    | 1.5453          | 1.4533        | 0.0       | 0.0        |
+| 0.6531        | 0.21  | 100  | 0.6528          | -0.1643        | -0.2945          | 0.3633             | 0.1303          | -106.6470      | -80.3385     | -1.7198         | -1.7354       | 1725.3125 | 6.6875     |
+| 0.6041        | 0.42  | 200  | 0.5936          | -0.7144        | -1.1047          | 0.3516             | 0.3903          | -187.6596      | -135.3474    | 0.9784          | 0.8864        | 3420.5938 | 167.4062   |
+| 0.5763        | 0.63  | 300  | 0.5773          | -0.7930        | -1.2317          | 0.3516             | 0.4387          | -200.3615      | -143.2137    | 1.7526          | 1.6599        | 4991.2812 | 452.7188   |
+| 0.5836        | 0.84  | 400  | 0.5769          | -0.6646        | -1.1353          | 0.3711             | 0.4707          | -190.7267      | -130.3719    | 1.8500          | 1.7576        | 6517.1875 | 782.8125   |
 
 
 ### Framework versions

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5a370d44f4bd7644f05dddbceaabd4c6255b0c5236b8eded581cb67596ef082a
+oid sha256:96028cf9e913d832d0a70759eea27b9a5c849327ec7dfdcb4154ba5214730296
 size 671150064

all_results.json

@@ -1,23 +1,8 @@
 {
     "epoch": 1.0,
-    "eval_logits/chosen": 1.507018804550171,
-    "eval_logits/rejected": 1.598363995552063,
-    "eval_logps/chosen": -114.45843505859375,
-    "eval_logps/rejected": -164.58822631835938,
-    "eval_loss": 0.6152364015579224,
-    "eval_pred_label": 0.0,
-    "eval_rewards/accuracies": 0.37890625,
-    "eval_rewards/chosen": -0.5054618120193481,
-    "eval_rewards/margins": 0.3684910833835602,
-    "eval_rewards/rejected": -0.8739528656005859,
-    "eval_runtime": 125.1233,
-    "eval_samples": 2000,
-    "eval_samples_per_second": 15.984,
-    "eval_steps_per_second": 0.256,
-    "eval_use_label": 0.0,
-    "train_loss": 0.6357159084743924,
-    "train_runtime": 9601.7268,
+    "train_loss": 0.6110695428068533,
+    "train_runtime": 9999.3279,
     "train_samples": 61135,
-    "train_samples_per_second": 6.367,
-    "train_steps_per_second": 0.05
+    "train_samples_per_second": 6.114,
+    "train_steps_per_second": 0.048
 }

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.6357159084743924,
-    "train_runtime": 9601.7268,
+    "train_loss": 0.6110695428068533,
+    "train_runtime": 9999.3279,
     "train_samples": 61135,
-    "train_samples_per_second": 6.367,
-    "train_steps_per_second": 0.05
+    "train_samples_per_second": 6.114,
+    "train_steps_per_second": 0.048
 }

trainer_state.json

@@ -23,887 +23,887 @@
       "rewards/margins": 0.0,
       "rewards/rejected": 0.0,
       "step": 1,
-      "use_label": 0.0
+      "use_label": 10.0
     },
     {
       "epoch": 0.02,
       "grad_norm": 0.4609375,
       "learning_rate": 1.0416666666666667e-06,
-      "logits/chosen": -2.2421462535858154,
-      "logits/rejected": -2.2770614624023438,
-      "logps/chosen": -51.98179626464844,
-      "logps/rejected": -64.9604263305664,
+      "logits/chosen": -2.2421748638153076,
+      "logits/rejected": -2.2769579887390137,
+      "logps/chosen": -51.987098693847656,
+      "logps/rejected": -64.96717071533203,
       "loss": 0.6929,
       "pred_label": 0.0,
-      "rewards/accuracies": 0.2222222238779068,
-      "rewards/chosen": 0.001975727966055274,
-      "rewards/margins": 0.00047667179023846984,
-      "rewards/rejected": 0.001499056350439787,
+      "rewards/accuracies": 0.25,
+      "rewards/chosen": 0.0019227324519306421,
+      "rewards/margins": 0.0004911368596367538,
+      "rewards/rejected": 0.0014315954176709056,
       "step": 10,
-      "use_label": 0.0
+      "use_label": 90.0
     },
     {
       "epoch": 0.04,
       "grad_norm": 0.39453125,
       "learning_rate": 2.0833333333333334e-06,
-      "logits/chosen": -2.2520272731781006,
-      "logits/rejected": -2.255510091781616,
-      "logps/chosen": -62.492515563964844,
-      "logps/rejected": -72.63607788085938,
+      "logits/chosen": -2.2521612644195557,
+      "logits/rejected": -2.255767822265625,
+      "logps/chosen": -62.4937629699707,
+      "logps/rejected": -72.63874816894531,
       "loss": 0.6919,
       "pred_label": 0.0,
       "rewards/accuracies": 0.2874999940395355,
-      "rewards/chosen": 0.01601376011967659,
-      "rewards/margins": 0.0011284304782748222,
-      "rewards/rejected": 0.014885328710079193,
+      "rewards/chosen": 0.01600126549601555,
+      "rewards/margins": 0.0011427802965044975,
+      "rewards/rejected": 0.0148584870621562,
       "step": 20,
-      "use_label": 0.0
+      "use_label": 242.0
     },
     {
       "epoch": 0.06,
-      "grad_norm": 0.5078125,
+      "grad_norm": 0.51171875,
       "learning_rate": 3.125e-06,
-      "logits/chosen": -2.3422012329101562,
-      "logits/rejected": -2.3548905849456787,
-      "logps/chosen": -79.14694213867188,
-      "logps/rejected": -98.82722473144531,
-      "loss": 0.6898,
-      "pred_label": 0.0,
-      "rewards/accuracies": 0.2874999940395355,
-      "rewards/chosen": 0.030949687585234642,
-      "rewards/margins": 0.0029636542312800884,
-      "rewards/rejected": 0.027986034750938416,
+      "logits/chosen": -2.3423426151275635,
+      "logits/rejected": -2.3549609184265137,
+      "logps/chosen": -79.10475158691406,
+      "logps/rejected": -98.8157958984375,
+      "loss": 0.6897,
+      "pred_label": 0.0,
+      "rewards/accuracies": 0.28125,
+      "rewards/chosen": 0.03137165680527687,
+      "rewards/margins": 0.0032712810207158327,
+      "rewards/rejected": 0.028100375086069107,
       "step": 30,
-      "use_label": 0.0
+      "use_label": 402.0
     },
     {
       "epoch": 0.08,
-      "grad_norm": 0.515625,
+      "grad_norm": 0.51953125,
       "learning_rate": 4.166666666666667e-06,
-      "logits/chosen": -2.322833776473999,
-      "logits/rejected": -2.3010501861572266,
-      "logps/chosen": -82.85880279541016,
-      "logps/rejected": -82.40392303466797,
+      "logits/chosen": -2.323338270187378,
+      "logits/rejected": -2.3015079498291016,
+      "logps/chosen": -82.85453796386719,
+      "logps/rejected": -82.39984893798828,
       "loss": 0.6866,
       "pred_label": 0.0,
       "rewards/accuracies": 0.2874999940395355,
-      "rewards/chosen": 0.033333443105220795,
-      "rewards/margins": 0.011918319389224052,
-      "rewards/rejected": 0.021415119990706444,
+      "rewards/chosen": 0.03337595611810684,
+      "rewards/margins": 0.011919925920665264,
+      "rewards/rejected": 0.021456023678183556,
       "step": 40,
-      "use_label": 0.0
+      "use_label": 562.0
     },
     {
       "epoch": 0.1,
-      "grad_norm": 0.67578125,
+      "grad_norm": 0.671875,
       "learning_rate": 4.999731868769027e-06,
-      "logits/chosen": -2.241189956665039,
-      "logits/rejected": -2.263849973678589,
-      "logps/chosen": -67.93062591552734,
-      "logps/rejected": -81.85546875,
+      "logits/chosen": -2.2404515743255615,
+      "logits/rejected": -2.262972354888916,
+      "logps/chosen": -67.89888000488281,
+      "logps/rejected": -81.8695068359375,
       "loss": 0.6805,
       "pred_label": 0.0,
       "rewards/accuracies": 0.32499998807907104,
-      "rewards/chosen": 0.009002490900456905,
-      "rewards/margins": 0.03016103245317936,
-      "rewards/rejected": -0.02115854248404503,
+      "rewards/chosen": 0.009319942444562912,
+      "rewards/margins": 0.030618786811828613,
+      "rewards/rejected": -0.0212988443672657,
       "step": 50,
-      "use_label": 0.0
+      "use_label": 722.0
     },
     {
       "epoch": 0.13,
-      "grad_norm": 1.09375,
+      "grad_norm": 1.0234375,
       "learning_rate": 4.9903533134293035e-06,
-      "logits/chosen": -2.218756914138794,
-      "logits/rejected": -2.1594481468200684,
-      "logps/chosen": -62.0407600402832,
-      "logps/rejected": -71.9369888305664,
-      "loss": 0.6748,
+      "logits/chosen": -2.2157275676727295,
+      "logits/rejected": -2.155928134918213,
+      "logps/chosen": -63.64031982421875,
+      "logps/rejected": -73.28236389160156,
+      "loss": 0.6752,
       "pred_label": 0.0,
       "rewards/accuracies": 0.3062500059604645,
-      "rewards/chosen": -0.0231451578438282,
-      "rewards/margins": 0.04653460532426834,
-      "rewards/rejected": -0.06967976689338684,
+      "rewards/chosen": -0.03914070501923561,
+      "rewards/margins": 0.04399287328124046,
+      "rewards/rejected": -0.08313358575105667,
       "step": 60,
-      "use_label": 0.0
+      "use_label": 882.0
     },
     {
       "epoch": 0.15,
-      "grad_norm": 0.8984375,
+      "grad_norm": 0.859375,
       "learning_rate": 4.967625656594782e-06,
-      "logits/chosen": -2.08909273147583,
-      "logits/rejected": -2.088801383972168,
-      "logps/chosen": -68.09326171875,
-      "logps/rejected": -81.9454116821289,
-      "loss": 0.6684,
+      "logits/chosen": -2.114478588104248,
+      "logits/rejected": -2.1126065254211426,
+      "logps/chosen": -70.76527404785156,
+      "logps/rejected": -83.94652557373047,
+      "loss": 0.6712,
       "pred_label": 0.0,
       "rewards/accuracies": 0.25,
-      "rewards/chosen": -0.12382155656814575,
-      "rewards/margins": 0.03761869668960571,
-      "rewards/rejected": -0.16144026815891266,
+      "rewards/chosen": -0.15054164826869965,
+      "rewards/margins": 0.030909737572073936,
+      "rewards/rejected": -0.18145139515399933,
       "step": 70,
-      "use_label": 0.0
+      "use_label": 1042.0
     },
     {
       "epoch": 0.17,
-      "grad_norm": 1.15625,
+      "grad_norm": 1.1640625,
       "learning_rate": 4.93167072587771e-06,
-      "logits/chosen": -2.20400071144104,
-      "logits/rejected": -2.1452622413635254,
-      "logps/chosen": -55.867881774902344,
-      "logps/rejected": -70.91771697998047,
-      "loss": 0.6588,
-      "pred_label": 0.0,
-      "rewards/accuracies": 0.26875001192092896,
-      "rewards/chosen": -0.0733698159456253,
-      "rewards/margins": 0.10403277724981308,
-      "rewards/rejected": -0.17740261554718018,
+      "logits/chosen": -2.2166943550109863,
+      "logits/rejected": -2.1609182357788086,
+      "logps/chosen": -54.8065185546875,
+      "logps/rejected": -69.45613861083984,
+      "loss": 0.6589,
+      "pred_label": 0.4749999940395355,
+      "rewards/accuracies": 0.26249998807907104,
+      "rewards/chosen": -0.06275613605976105,
+      "rewards/margins": 0.10003063827753067,
+      "rewards/rejected": -0.16278676688671112,
       "step": 80,
-      "use_label": 0.0
+      "use_label": 1201.5250244140625
     },
     {
       "epoch": 0.19,
-      "grad_norm": 1.0546875,
+      "grad_norm": 1.8125,
       "learning_rate": 4.882681251368549e-06,
-      "logits/chosen": -1.991231918334961,
-      "logits/rejected": -1.9964717626571655,
-      "logps/chosen": -72.28443908691406,
-      "logps/rejected": -90.79218292236328,
-      "loss": 0.6587,
-      "pred_label": 0.0,
-      "rewards/accuracies": 0.30000001192092896,
-      "rewards/chosen": -0.13902384042739868,
-      "rewards/margins": 0.08125626295804977,
-      "rewards/rejected": -0.22028008103370667,
+      "logits/chosen": -1.9692049026489258,
+      "logits/rejected": -1.9792039394378662,
+      "logps/chosen": -76.60871887207031,
+      "logps/rejected": -96.53330993652344,
+      "loss": 0.6564,
+      "pred_label": 2.0999999046325684,
+      "rewards/accuracies": 0.29374998807907104,
+      "rewards/chosen": -0.18226662278175354,
+      "rewards/margins": 0.09542477130889893,
+      "rewards/rejected": -0.27769142389297485,
       "step": 90,
-      "use_label": 0.0
+      "use_label": 1359.9000244140625
     },
     {
       "epoch": 0.21,
-      "grad_norm": 2.359375,
+      "grad_norm": 2.171875,
       "learning_rate": 4.8209198325401815e-06,
-      "logits/chosen": -1.9231764078140259,
-      "logits/rejected": -1.9043807983398438,
-      "logps/chosen": -103.5636978149414,
-      "logps/rejected": -96.08602142333984,
-      "loss": 0.6551,
-      "pred_label": 0.0,
-      "rewards/accuracies": 0.35624998807907104,
-      "rewards/chosen": -0.2353379726409912,
-      "rewards/margins": 0.08685441315174103,
-      "rewards/rejected": -0.32219237089157104,
+      "logits/chosen": -1.9027693271636963,
+      "logits/rejected": -1.8775581121444702,
+      "logps/chosen": -92.94733428955078,
+      "logps/rejected": -84.73824310302734,
+      "loss": 0.6531,
+      "pred_label": 4.0,
+      "rewards/accuracies": 0.32499998807907104,
+      "rewards/chosen": -0.12917451560497284,
+      "rewards/margins": 0.07954015582799911,
+      "rewards/rejected": -0.20871467888355255,
       "step": 100,
-      "use_label": 0.0
+      "use_label": 1518.0
     },
     {
       "epoch": 0.21,
-      "eval_logits/chosen": -1.762041687965393,
-      "eval_logits/rejected": -1.7460479736328125,
-      "eval_logps/chosen": -87.55253601074219,
-      "eval_logps/rejected": -114.47212219238281,
-      "eval_loss": 0.652633547782898,
-      "eval_pred_label": 0.0,
-      "eval_rewards/accuracies": 0.3359375,
-      "eval_rewards/chosen": -0.23640292882919312,
-      "eval_rewards/margins": 0.136388897895813,
-      "eval_rewards/rejected": -0.3727918267250061,
-      "eval_runtime": 125.4491,
-      "eval_samples_per_second": 15.943,
-      "eval_steps_per_second": 0.255,
-      "eval_use_label": 0.0,
+      "eval_logits/chosen": -1.7353737354278564,
+      "eval_logits/rejected": -1.7198325395584106,
+      "eval_logps/chosen": -80.33845520019531,
+      "eval_logps/rejected": -106.64702606201172,
+      "eval_loss": 0.6527961492538452,
+      "eval_pred_label": 6.6875,
+      "eval_rewards/accuracies": 0.36328125,
+      "eval_rewards/chosen": -0.1642620712518692,
+      "eval_rewards/margins": 0.13027876615524292,
+      "eval_rewards/rejected": -0.2945408225059509,
+      "eval_runtime": 125.2319,
+      "eval_samples_per_second": 15.97,
+      "eval_steps_per_second": 0.256,
+      "eval_use_label": 1725.3125,
       "step": 100
     },
     {
       "epoch": 0.23,
-      "grad_norm": 1.59375,
+      "grad_norm": 2.0,
       "learning_rate": 4.746717530629565e-06,
-      "logits/chosen": -1.7847106456756592,
-      "logits/rejected": -1.7590484619140625,
-      "logps/chosen": -85.73925018310547,
-      "logps/rejected": -106.20509338378906,
-      "loss": 0.6557,
-      "pred_label": 0.0,
-      "rewards/accuracies": 0.3499999940395355,
-      "rewards/chosen": -0.14638465642929077,
-      "rewards/margins": 0.12975916266441345,
-      "rewards/rejected": -0.2761438190937042,
+      "logits/chosen": -1.7974278926849365,
+      "logits/rejected": -1.7697474956512451,
+      "logps/chosen": -89.79286193847656,
+      "logps/rejected": -113.6241455078125,
+      "loss": 0.6479,
+      "pred_label": 9.199999809265137,
+      "rewards/accuracies": 0.36250001192092896,
+      "rewards/chosen": -0.18692079186439514,
+      "rewards/margins": 0.16341358423233032,
+      "rewards/rejected": -0.3503343462944031,
       "step": 110,
-      "use_label": 0.0
+      "use_label": 1928.800048828125
     },
     {
       "epoch": 0.25,
-      "grad_norm": 1.828125,
+      "grad_norm": 2.890625,
       "learning_rate": 4.660472094042121e-06,
-      "logits/chosen": -1.1902318000793457,
-      "logits/rejected": -1.0542975664138794,
-      "logps/chosen": -108.4779052734375,
-      "logps/rejected": -127.95109558105469,
-      "loss": 0.6493,
-      "pred_label": 0.0,
-      "rewards/accuracies": 0.36250001192092896,
-      "rewards/chosen": -0.38532325625419617,
-      "rewards/margins": 0.1649974286556244,
-      "rewards/rejected": -0.5503206849098206,
+      "logits/chosen": -1.454304814338684,
+      "logits/rejected": -1.3457725048065186,
+      "logps/chosen": -109.3675537109375,
+      "logps/rejected": -133.90725708007812,
+      "loss": 0.6432,
+      "pred_label": 14.949999809265137,
+      "rewards/accuracies": 0.34375,
+      "rewards/chosen": -0.3942197263240814,
+      "rewards/margins": 0.21566259860992432,
+      "rewards/rejected": -0.6098822951316833,
       "step": 120,
-      "use_label": 0.0
+      "use_label": 2083.050048828125
     },
     {
       "epoch": 0.27,
-      "grad_norm": 1.9375,
+      "grad_norm": 2.5625,
       "learning_rate": 4.5626458262912745e-06,
-      "logits/chosen": -0.818010687828064,
-      "logits/rejected": -0.7847374081611633,
-      "logps/chosen": -109.61775207519531,
-      "logps/rejected": -133.42086791992188,
-      "loss": 0.6524,
-      "pred_label": 0.0,
-      "rewards/accuracies": 0.33125001192092896,
-      "rewards/chosen": -0.43839359283447266,
-      "rewards/margins": 0.16735044121742249,
-      "rewards/rejected": -0.6057440638542175,
+      "logits/chosen": -1.0859026908874512,
+      "logits/rejected": -1.0426993370056152,
+      "logps/chosen": -112.0394515991211,
+      "logps/rejected": -139.61097717285156,
+      "loss": 0.6391,
+      "pred_label": 21.049999237060547,
+      "rewards/accuracies": 0.36250001192092896,
+      "rewards/chosen": -0.4626106321811676,
+      "rewards/margins": 0.20503444969654083,
+      "rewards/rejected": -0.6676451563835144,
       "step": 130,
-      "use_label": 0.0
+      "use_label": 2236.949951171875
     },
     {
       "epoch": 0.29,
-      "grad_norm": 1.71875,
+      "grad_norm": 2.484375,
       "learning_rate": 4.453763107901676e-06,
-      "logits/chosen": -0.7395650148391724,
-      "logits/rejected": -0.8444339036941528,
-      "logps/chosen": -116.97528076171875,
-      "logps/rejected": -130.2399139404297,
-      "loss": 0.6381,
-      "pred_label": 0.0,
-      "rewards/accuracies": 0.33125001192092896,
-      "rewards/chosen": -0.3622770607471466,
-      "rewards/margins": 0.1490650475025177,
-      "rewards/rejected": -0.5113420486450195,
+      "logits/chosen": -0.735418975353241,
+      "logits/rejected": -0.8380192518234253,
+      "logps/chosen": -138.07081604003906,
+      "logps/rejected": -150.91665649414062,
+      "loss": 0.6252,
+      "pred_label": 31.399999618530273,
+      "rewards/accuracies": 0.30000001192092896,
+      "rewards/chosen": -0.5732325315475464,
+      "rewards/margins": 0.1448771208524704,
+      "rewards/rejected": -0.7181096076965332,
       "step": 140,
-      "use_label": 0.0
+      "use_label": 2386.60009765625
     },
     {
       "epoch": 0.31,
-      "grad_norm": 2.125,
+      "grad_norm": 3.859375,
       "learning_rate": 4.33440758555951e-06,
-      "logits/chosen": -0.6497868299484253,
-      "logits/rejected": -0.6378159523010254,
-      "logps/chosen": -89.60552978515625,
-      "logps/rejected": -115.42192077636719,
-      "loss": 0.6379,
-      "pred_label": 0.0,
-      "rewards/accuracies": 0.3187499940395355,
-      "rewards/chosen": -0.2445882111787796,
-      "rewards/margins": 0.23124215006828308,
-      "rewards/rejected": -0.4758303761482239,
+      "logits/chosen": -0.48231878876686096,
+      "logits/rejected": -0.43882569670677185,
+      "logps/chosen": -117.69664001464844,
+      "logps/rejected": -150.86083984375,
+      "loss": 0.6219,
+      "pred_label": 43.45000076293945,
+      "rewards/accuracies": 0.32499998807907104,
+      "rewards/chosen": -0.5254992246627808,
+      "rewards/margins": 0.3047201633453369,
+      "rewards/rejected": -0.8302194476127625,
       "step": 150,
-      "use_label": 0.0
+      "use_label": 2534.550048828125
     },
     {
       "epoch": 0.33,
-      "grad_norm": 2.15625,
+      "grad_norm": 2.890625,
       "learning_rate": 4.205219043576955e-06,
-      "logits/chosen": -0.3159053921699524,
-      "logits/rejected": -0.33064812421798706,
-      "logps/chosen": -99.68696594238281,
-      "logps/rejected": -129.45729064941406,
-      "loss": 0.6317,
-      "pred_label": 0.0,
-      "rewards/accuracies": 0.2874999940395355,
-      "rewards/chosen": -0.35356926918029785,
-      "rewards/margins": 0.16687795519828796,
-      "rewards/rejected": -0.5204472541809082,
+      "logits/chosen": -0.15186011791229248,
+      "logits/rejected": -0.17336201667785645,
+      "logps/chosen": -128.78500366210938,
+      "logps/rejected": -159.26498413085938,
+      "loss": 0.5982,
+      "pred_label": 58.25,
+      "rewards/accuracies": 0.2750000059604645,
+      "rewards/chosen": -0.6445494294166565,
+      "rewards/margins": 0.17397476732730865,
+      "rewards/rejected": -0.818524181842804,
       "step": 160,
-      "use_label": 0.0
+      "use_label": 2679.75
     },
     {
       "epoch": 0.36,
-      "grad_norm": 2.4375,
+      "grad_norm": 3.328125,
       "learning_rate": 4.066889974440757e-06,
-      "logits/chosen": 0.14531800150871277,
-      "logits/rejected": 0.18166163563728333,
-      "logps/chosen": -95.45491027832031,
-      "logps/rejected": -125.1463623046875,
-      "loss": 0.6291,
-      "pred_label": 0.0,
-      "rewards/accuracies": 0.29374998807907104,
-      "rewards/chosen": -0.39946848154067993,
-      "rewards/margins": 0.20978550612926483,
-      "rewards/rejected": -0.609254002571106,
+      "logits/chosen": 0.14322622120380402,
+      "logits/rejected": 0.18100713193416595,
+      "logps/chosen": -108.39127349853516,
+      "logps/rejected": -140.55824279785156,
+      "loss": 0.5938,
+      "pred_label": 79.57499694824219,
+      "rewards/accuracies": 0.3062500059604645,
+      "rewards/chosen": -0.5288320779800415,
+      "rewards/margins": 0.23454061150550842,
+      "rewards/rejected": -0.7633727192878723,
       "step": 170,
-      "use_label": 0.0
+      "use_label": 2818.425048828125
     },
     {
       "epoch": 0.38,
-      "grad_norm": 2.453125,
+      "grad_norm": 3.0,
       "learning_rate": 3.92016186682789e-06,
-      "logits/chosen": -0.3282355070114136,
-      "logits/rejected": -0.21966704726219177,
-      "logps/chosen": -108.00712585449219,
-      "logps/rejected": -128.67587280273438,
-      "loss": 0.649,
-      "pred_label": 0.0,
+      "logits/chosen": -0.20601686835289001,
+      "logits/rejected": -0.09364790469408035,
+      "logps/chosen": -105.94217681884766,
+      "logps/rejected": -130.695556640625,
+      "loss": 0.6262,
+      "pred_label": 100.2750015258789,
       "rewards/accuracies": 0.35624998807907104,
-      "rewards/chosen": -0.4521949887275696,
-      "rewards/margins": 0.27172034978866577,
-      "rewards/rejected": -0.7239152789115906,
+      "rewards/chosen": -0.4315454065799713,
+      "rewards/margins": 0.3125666677951813,
+      "rewards/rejected": -0.7441121339797974,
       "step": 180,
-      "use_label": 0.0
+      "use_label": 2957.72509765625
     },
     {
       "epoch": 0.4,
-      "grad_norm": 1.84375,
+      "grad_norm": 2.734375,
       "learning_rate": 3.7658212309857576e-06,
-      "logits/chosen": -0.889633297920227,
-      "logits/rejected": -0.6851574778556824,
-      "logps/chosen": -91.25111389160156,
-      "logps/rejected": -118.9649887084961,
-      "loss": 0.6461,
-      "pred_label": 0.0,
-      "rewards/accuracies": 0.33125001192092896,
-      "rewards/chosen": -0.32139474153518677,
-      "rewards/margins": 0.22424864768981934,
-      "rewards/rejected": -0.5456433892250061,
+      "logits/chosen": -0.34412023425102234,
+      "logits/rejected": -0.07299783080816269,
+      "logps/chosen": -107.5626449584961,
+      "logps/rejected": -141.1322479248047,
+      "loss": 0.6092,
+      "pred_label": 121.05000305175781,
+      "rewards/accuracies": 0.34375,
+      "rewards/chosen": -0.48451024293899536,
+      "rewards/margins": 0.28280580043792725,
+      "rewards/rejected": -0.7673160433769226,
       "step": 190,
-      "use_label": 0.0
+      "use_label": 3096.949951171875
     },
     {
       "epoch": 0.42,
-      "grad_norm": 1.9453125,
+      "grad_norm": 6.5,
       "learning_rate": 3.604695382782159e-06,
-      "logits/chosen": -0.8204952478408813,
-      "logits/rejected": -0.7186430096626282,
-      "logps/chosen": -112.41142272949219,
-      "logps/rejected": -120.7835693359375,
-      "loss": 0.6376,
-      "pred_label": 0.0,
-      "rewards/accuracies": 0.3125,
-      "rewards/chosen": -0.30735117197036743,
-      "rewards/margins": 0.169038325548172,
-      "rewards/rejected": -0.47638946771621704,
+      "logits/chosen": 0.03128425031900406,
+      "logits/rejected": 0.20205454528331757,
+      "logps/chosen": -145.35342407226562,
+      "logps/rejected": -162.05667114257812,
+      "loss": 0.6041,
+      "pred_label": 135.89999389648438,
+      "rewards/accuracies": 0.30000001192092896,
+      "rewards/chosen": -0.6367710828781128,
+      "rewards/margins": 0.25234952569007874,
+      "rewards/rejected": -0.8891205787658691,
       "step": 200,
-      "use_label": 0.0
+      "use_label": 3242.10009765625
     },
     {
       "epoch": 0.42,
-      "eval_logits/chosen": -0.023804781958460808,
-      "eval_logits/rejected": 0.04317883029580116,
-      "eval_logps/chosen": -97.96138000488281,
-      "eval_logps/rejected": -137.9141845703125,
-      "eval_loss": 0.6288520693778992,
-      "eval_pred_label": 0.0,
-      "eval_rewards/accuracies": 0.3671875,
-      "eval_rewards/chosen": -0.34049129486083984,
-      "eval_rewards/margins": 0.26672109961509705,
-      "eval_rewards/rejected": -0.6072123646736145,
-      "eval_runtime": 125.433,
-      "eval_samples_per_second": 15.945,
+      "eval_logits/chosen": 0.886444091796875,
+      "eval_logits/rejected": 0.9784458875656128,
+      "eval_logps/chosen": -135.34742736816406,
+      "eval_logps/rejected": -187.65963745117188,
+      "eval_loss": 0.5936154723167419,
+      "eval_pred_label": 167.40625,
+      "eval_rewards/accuracies": 0.3515625,
+      "eval_rewards/chosen": -0.7143516540527344,
+      "eval_rewards/margins": 0.3903152644634247,
+      "eval_rewards/rejected": -1.1046667098999023,
+      "eval_runtime": 125.3006,
+      "eval_samples_per_second": 15.962,
       "eval_steps_per_second": 0.255,
-      "eval_use_label": 0.0,
+      "eval_use_label": 3420.59375,
       "step": 200
     },
     {
       "epoch": 0.44,
-      "grad_norm": 2.265625,
+      "grad_norm": 3.796875,
       "learning_rate": 3.437648009023905e-06,
-      "logits/chosen": -0.05805685371160507,
-      "logits/rejected": -0.06056814268231392,
-      "logps/chosen": -88.78871154785156,
-      "logps/rejected": -124.3318862915039,
-      "loss": 0.6218,
-      "pred_label": 0.0,
-      "rewards/accuracies": 0.375,
-      "rewards/chosen": -0.3281395435333252,
-      "rewards/margins": 0.28538644313812256,
-      "rewards/rejected": -0.613525927066803,
+      "logits/chosen": 0.6729141473770142,
+      "logits/rejected": 0.6579598188400269,
+      "logps/chosen": -119.19351959228516,
+      "logps/rejected": -159.00997924804688,
+      "loss": 0.5936,
+      "pred_label": 201.6999969482422,
+      "rewards/accuracies": 0.3499999940395355,
+      "rewards/chosen": -0.63218754529953,
+      "rewards/margins": 0.3281194567680359,
+      "rewards/rejected": -0.9603070020675659,
       "step": 210,
-      "use_label": 0.0
+      "use_label": 3592.300048828125
     },
     {
       "epoch": 0.46,
-      "grad_norm": 2.21875,
+      "grad_norm": 4.5,
       "learning_rate": 3.265574537815398e-06,
-      "logits/chosen": -0.1400775909423828,
-      "logits/rejected": -0.005620801355689764,
-      "logps/chosen": -133.7158660888672,
-      "logps/rejected": -136.84619140625,
-      "loss": 0.627,
-      "pred_label": 0.0,
-      "rewards/accuracies": 0.30000001192092896,
-      "rewards/chosen": -0.5408719778060913,
-      "rewards/margins": 0.16390959918498993,
-      "rewards/rejected": -0.7047815918922424,
+      "logits/chosen": 0.2854166626930237,
+      "logits/rejected": 0.4488348066806793,
+      "logps/chosen": -148.92379760742188,
+      "logps/rejected": -161.19557189941406,
+      "loss": 0.5938,
+      "pred_label": 225.52499389648438,
+      "rewards/accuracies": 0.3125,
+      "rewards/chosen": -0.6929510235786438,
+      "rewards/margins": 0.2553243637084961,
+      "rewards/rejected": -0.9482753872871399,
       "step": 220,
-      "use_label": 0.0
+      "use_label": 3728.47509765625
     },
     {
       "epoch": 0.48,
-      "grad_norm": 1.8515625,
+      "grad_norm": 2.28125,
       "learning_rate": 3.089397338773569e-06,
-      "logits/chosen": 0.16266627609729767,
-      "logits/rejected": 0.2626825273036957,
-      "logps/chosen": -93.3644027709961,
-      "logps/rejected": -119.67996978759766,
-      "loss": 0.6261,
-      "pred_label": 0.0,
-      "rewards/accuracies": 0.3187499940395355,
-      "rewards/chosen": -0.28929832577705383,
-      "rewards/margins": 0.27991363406181335,
-      "rewards/rejected": -0.5692119598388672,
+      "logits/chosen": 0.00020002425299026072,
+      "logits/rejected": 0.1493436098098755,
+      "logps/chosen": -103.05213928222656,
+      "logps/rejected": -136.05099487304688,
+      "loss": 0.597,
+      "pred_label": 252.02499389648438,
+      "rewards/accuracies": 0.30000001192092896,
+      "rewards/chosen": -0.3861756920814514,
+      "rewards/margins": 0.3467464745044708,
+      "rewards/rejected": -0.7329221963882446,
       "step": 230,
-      "use_label": 0.0
+      "use_label": 3861.97509765625
     },
     {
       "epoch": 0.5,
-      "grad_norm": 1.8984375,
+      "grad_norm": 3.125,
       "learning_rate": 2.9100607788275547e-06,
-      "logits/chosen": 0.854693591594696,
-      "logits/rejected": 0.7261193990707397,
-      "logps/chosen": -99.00528717041016,
-      "logps/rejected": -135.73580932617188,
-      "loss": 0.6295,
-      "pred_label": 0.0,
+      "logits/chosen": 0.49308425188064575,
+      "logits/rejected": 0.44487372040748596,
+      "logps/chosen": -109.46275329589844,
+      "logps/rejected": -153.8666534423828,
+      "loss": 0.584,
+      "pred_label": 275.375,
       "rewards/accuracies": 0.3687500059604645,
-      "rewards/chosen": -0.2997274696826935,
-      "rewards/margins": 0.3153937757015228,
-      "rewards/rejected": -0.6151211857795715,
+      "rewards/chosen": -0.40430212020874023,
+      "rewards/margins": 0.3921273946762085,
+      "rewards/rejected": -0.7964295148849487,
       "step": 240,
-      "use_label": 0.0
+      "use_label": 3998.625
     },
     {
       "epoch": 0.52,
-      "grad_norm": 2.03125,
+      "grad_norm": 2.0625,
       "learning_rate": 2.72852616010567e-06,
-      "logits/chosen": 0.6816203594207764,
-      "logits/rejected": 0.7033491134643555,
-      "logps/chosen": -119.7255859375,
-      "logps/rejected": -144.8857421875,
-      "loss": 0.6376,
-      "pred_label": 0.0,
-      "rewards/accuracies": 0.3812499940395355,
-      "rewards/chosen": -0.4632648825645447,
-      "rewards/margins": 0.2932681143283844,
-      "rewards/rejected": -0.7565330266952515,
+      "logits/chosen": 0.3891890347003937,
+      "logits/rejected": 0.47166162729263306,
+      "logps/chosen": -122.5915298461914,
+      "logps/rejected": -153.12493896484375,
+      "loss": 0.5998,
+      "pred_label": 301.875,
+      "rewards/accuracies": 0.36250001192092896,
+      "rewards/chosen": -0.4919242262840271,
+      "rewards/margins": 0.3470008671283722,
+      "rewards/rejected": -0.8389250636100769,
       "step": 250,
-      "use_label": 0.0
+      "use_label": 4132.125
     },
     {
       "epoch": 0.54,
-      "grad_norm": 1.8984375,
+      "grad_norm": 2.171875,
       "learning_rate": 2.5457665670441937e-06,
-      "logits/chosen": 0.5938165187835693,
-      "logits/rejected": 0.5592354536056519,
-      "logps/chosen": -110.32804870605469,
-      "logps/rejected": -146.76275634765625,
-      "loss": 0.6162,
-      "pred_label": 0.0,
-      "rewards/accuracies": 0.34375,
-      "rewards/chosen": -0.44222426414489746,
-      "rewards/margins": 0.2809238135814667,
-      "rewards/rejected": -0.7231480479240417,
+      "logits/chosen": 0.4214790463447571,
+      "logits/rejected": 0.4202333092689514,
+      "logps/chosen": -116.09378814697266,
+      "logps/rejected": -156.8458251953125,
+      "loss": 0.592,
+      "pred_label": 326.3500061035156,
+      "rewards/accuracies": 0.32499998807907104,
+      "rewards/chosen": -0.4998815953731537,
+      "rewards/margins": 0.324097216129303,
+      "rewards/rejected": -0.8239787817001343,
       "step": 260,
-      "use_label": 0.0
+      "use_label": 4267.64990234375
     },
     {
       "epoch": 0.57,
-      "grad_norm": 2.90625,
+      "grad_norm": 3.40625,
       "learning_rate": 2.3627616503391813e-06,
-      "logits/chosen": 0.6390979290008545,
-      "logits/rejected": 0.5789315700531006,
-      "logps/chosen": -123.83528137207031,
-      "logps/rejected": -144.61489868164062,
-      "loss": 0.6162,
-      "pred_label": 0.0,
-      "rewards/accuracies": 0.36250001192092896,
-      "rewards/chosen": -0.5091949701309204,
-      "rewards/margins": 0.24320097267627716,
-      "rewards/rejected": -0.7523959279060364,
+      "logits/chosen": 0.9609361886978149,
+      "logits/rejected": 0.8760908246040344,
+      "logps/chosen": -142.81573486328125,
+      "logps/rejected": -170.10379028320312,
+      "loss": 0.5888,
+      "pred_label": 343.04998779296875,
+      "rewards/accuracies": 0.3375000059604645,
+      "rewards/chosen": -0.698999285697937,
+      "rewards/margins": 0.30828553438186646,
+      "rewards/rejected": -1.0072848796844482,
       "step": 270,
-      "use_label": 0.0
+      "use_label": 4410.9501953125
     },
     {
       "epoch": 0.59,
-      "grad_norm": 2.34375,
+      "grad_norm": 2.234375,
       "learning_rate": 2.1804923757009885e-06,
-      "logits/chosen": 0.8771865963935852,
-      "logits/rejected": 1.0158352851867676,
-      "logps/chosen": -118.5296859741211,
-      "logps/rejected": -138.31729125976562,
-      "loss": 0.6357,
-      "pred_label": 0.0,
-      "rewards/accuracies": 0.30000001192092896,
-      "rewards/chosen": -0.5302416086196899,
-      "rewards/margins": 0.2237352430820465,
-      "rewards/rejected": -0.7539768218994141,
+      "logits/chosen": 1.1657536029815674,
+      "logits/rejected": 1.3259608745574951,
+      "logps/chosen": -131.4703826904297,
+      "logps/rejected": -156.4979248046875,
+      "loss": 0.6007,
+      "pred_label": 361.5,
+      "rewards/accuracies": 0.3375000059604645,
+      "rewards/chosen": -0.6596485376358032,
+      "rewards/margins": 0.27613458037376404,
+      "rewards/rejected": -0.9357832074165344,
       "step": 280,
-      "use_label": 0.0
+      "use_label": 4552.5
     },
     {
       "epoch": 0.61,
-      "grad_norm": 2.59375,
+      "grad_norm": 2.875,
       "learning_rate": 1.9999357655598894e-06,
-      "logits/chosen": 0.44083184003829956,
-      "logits/rejected": 0.41123947501182556,
-      "logps/chosen": -112.27372741699219,
-      "logps/rejected": -146.95498657226562,
-      "loss": 0.6228,
-      "pred_label": 0.0,
-      "rewards/accuracies": 0.30000001192092896,
-      "rewards/chosen": -0.4572528004646301,
-      "rewards/margins": 0.24868395924568176,
-      "rewards/rejected": -0.7059367299079895,
+      "logits/chosen": 0.9594011306762695,
+      "logits/rejected": 0.9126796722412109,
+      "logps/chosen": -144.55104064941406,
+      "logps/rejected": -183.51065063476562,
+      "loss": 0.5899,
+      "pred_label": 386.07501220703125,
+      "rewards/accuracies": 0.3187499940395355,
+      "rewards/chosen": -0.7800258994102478,
+      "rewards/margins": 0.2914672791957855,
+      "rewards/rejected": -1.0714932680130005,
       "step": 290,
-      "use_label": 0.0
+      "use_label": 4687.9248046875
     },
     {
       "epoch": 0.63,
-      "grad_norm": 2.34375,
+      "grad_norm": 3.1875,
       "learning_rate": 1.8220596619089576e-06,
-      "logits/chosen": 0.6273639798164368,
-      "logits/rejected": 0.5140804052352905,
-      "logps/chosen": -123.02046966552734,
-      "logps/rejected": -168.80987548828125,
-      "loss": 0.6196,
-      "pred_label": 0.0,
-      "rewards/accuracies": 0.40625,
-      "rewards/chosen": -0.4542613625526428,
-      "rewards/margins": 0.2926333546638489,
-      "rewards/rejected": -0.7468947172164917,
+      "logits/chosen": 1.2753574848175049,
+      "logits/rejected": 1.1057071685791016,
+      "logps/chosen": -165.4674072265625,
+      "logps/rejected": -223.6466064453125,
+      "loss": 0.5763,
+      "pred_label": 409.9750061035156,
+      "rewards/accuracies": 0.4124999940395355,
+      "rewards/chosen": -0.8787307739257812,
+      "rewards/margins": 0.41653138399124146,
+      "rewards/rejected": -1.295262098312378,
       "step": 300,
-      "use_label": 0.0
+      "use_label": 4824.02490234375
     },
     {
       "epoch": 0.63,
-      "eval_logits/chosen": 1.0944873094558716,
-      "eval_logits/rejected": 1.1831356287002563,
-      "eval_logps/chosen": -102.62176513671875,
-      "eval_logps/rejected": -150.12503051757812,
-      "eval_loss": 0.618873655796051,
-      "eval_pred_label": 0.0,
-      "eval_rewards/accuracies": 0.375,
-      "eval_rewards/chosen": -0.3870951533317566,
-      "eval_rewards/margins": 0.34222573041915894,
-      "eval_rewards/rejected": -0.7293209433555603,
-      "eval_runtime": 125.4362,
-      "eval_samples_per_second": 15.944,
+      "eval_logits/chosen": 1.6598718166351318,
+      "eval_logits/rejected": 1.7526323795318604,
+      "eval_logps/chosen": -143.2136993408203,
+      "eval_logps/rejected": -200.36146545410156,
+      "eval_loss": 0.5773172974586487,
+      "eval_pred_label": 452.71875,
+      "eval_rewards/accuracies": 0.3515625,
+      "eval_rewards/chosen": -0.7930145263671875,
+      "eval_rewards/margins": 0.4386705756187439,
+      "eval_rewards/rejected": -1.2316851615905762,
+      "eval_runtime": 125.3512,
+      "eval_samples_per_second": 15.955,
       "eval_steps_per_second": 0.255,
-      "eval_use_label": 0.0,
+      "eval_use_label": 4991.28125,
       "step": 300
     },
     {
       "epoch": 0.65,
-      "grad_norm": 1.8515625,
+      "grad_norm": 3.0625,
       "learning_rate": 1.647817538357072e-06,
-      "logits/chosen": 0.8131985664367676,
-      "logits/rejected": 0.8752232789993286,
-      "logps/chosen": -91.52378845214844,
-      "logps/rejected": -139.95840454101562,
-      "loss": 0.5999,
-      "pred_label": 0.0,
-      "rewards/accuracies": 0.3687500059604645,
-      "rewards/chosen": -0.3592718541622162,
-      "rewards/margins": 0.3578081727027893,
-      "rewards/rejected": -0.7170799970626831,
+      "logits/chosen": 1.3793504238128662,
+      "logits/rejected": 1.4072078466415405,
+      "logps/chosen": -126.9173583984375,
+      "logps/rejected": -186.46255493164062,
+      "loss": 0.5633,
+      "pred_label": 494.42498779296875,
+      "rewards/accuracies": 0.36250001192092896,
+      "rewards/chosen": -0.7132076025009155,
+      "rewards/margins": 0.4689141809940338,
+      "rewards/rejected": -1.1821218729019165,
       "step": 310,
-      "use_label": 0.0
+      "use_label": 5155.5751953125
     },
     {
       "epoch": 0.67,
-      "grad_norm": 2.40625,
+      "grad_norm": 4.21875,
       "learning_rate": 1.4781433892011132e-06,
-      "logits/chosen": 0.9751952886581421,
-      "logits/rejected": 1.1630818843841553,
-      "logps/chosen": -135.82566833496094,
-      "logps/rejected": -168.11805725097656,
-      "loss": 0.6109,
-      "pred_label": 0.0,
-      "rewards/accuracies": 0.3687500059604645,
-      "rewards/chosen": -0.6275521516799927,
-      "rewards/margins": 0.3816707730293274,
-      "rewards/rejected": -1.0092228651046753,
+      "logits/chosen": 1.2615296840667725,
+      "logits/rejected": 1.4717950820922852,
+      "logps/chosen": -163.67529296875,
+      "logps/rejected": -205.421142578125,
+      "loss": 0.5761,
+      "pred_label": 523.5250244140625,
+      "rewards/accuracies": 0.3812499940395355,
+      "rewards/chosen": -0.9060484766960144,
+      "rewards/margins": 0.4762052893638611,
+      "rewards/rejected": -1.382253885269165,
       "step": 320,
-      "use_label": 0.0
+      "use_label": 5286.47509765625
     },
     {
       "epoch": 0.69,
-      "grad_norm": 1.984375,
+      "grad_norm": 2.359375,
       "learning_rate": 1.3139467229135999e-06,
-      "logits/chosen": 1.3293979167938232,
-      "logits/rejected": 1.3260401487350464,
-      "logps/chosen": -135.96664428710938,
-      "logps/rejected": -166.52359008789062,
-      "loss": 0.6295,
-      "pred_label": 0.0,
+      "logits/chosen": 1.4169238805770874,
+      "logits/rejected": 1.4296729564666748,
+      "logps/chosen": -150.2149200439453,
+      "logps/rejected": -186.73570251464844,
+      "loss": 0.5799,
+      "pred_label": 550.125,
       "rewards/accuracies": 0.33125001192092896,
-      "rewards/chosen": -0.6585850715637207,
-      "rewards/margins": 0.3205706775188446,
-      "rewards/rejected": -0.9791557192802429,
+      "rewards/chosen": -0.8010675311088562,
+      "rewards/margins": 0.3802093267440796,
+      "rewards/rejected": -1.1812770366668701,
       "step": 330,
-      "use_label": 0.0
+      "use_label": 5419.875
     },
     {
       "epoch": 0.71,
-      "grad_norm": 2.09375,
+      "grad_norm": 3.15625,
       "learning_rate": 1.1561076868822756e-06,
-      "logits/chosen": 0.7383319139480591,
-      "logits/rejected": 0.6407849192619324,
-      "logps/chosen": -150.60504150390625,
-      "logps/rejected": -166.74940490722656,
-      "loss": 0.6247,
-      "pred_label": 0.0,
-      "rewards/accuracies": 0.3499999940395355,
-      "rewards/chosen": -0.658658504486084,
-      "rewards/margins": 0.24373307824134827,
-      "rewards/rejected": -0.9023915529251099,
+      "logits/chosen": 0.9984269142150879,
+      "logits/rejected": 0.9373771548271179,
+      "logps/chosen": -161.85842895507812,
+      "logps/rejected": -182.74703979492188,
+      "loss": 0.5933,
+      "pred_label": 567.2000122070312,
+      "rewards/accuracies": 0.32499998807907104,
+      "rewards/chosen": -0.771192193031311,
+      "rewards/margins": 0.2911759614944458,
+      "rewards/rejected": -1.0623681545257568,
       "step": 340,
-      "use_label": 0.0
+      "use_label": 5562.7998046875
     },
     {
       "epoch": 0.73,
-      "grad_norm": 2.21875,
+      "grad_norm": 2.59375,
       "learning_rate": 1.0054723495346484e-06,
-      "logits/chosen": 0.6359546184539795,
-      "logits/rejected": 0.7167641520500183,
-      "logps/chosen": -163.8385772705078,
-      "logps/rejected": -195.6297607421875,
-      "loss": 0.6138,
-      "pred_label": 0.0,
+      "logits/chosen": 0.83796626329422,
+      "logits/rejected": 0.8520887494087219,
+      "logps/chosen": -176.03054809570312,
+      "logps/rejected": -217.10214233398438,
+      "loss": 0.5863,
+      "pred_label": 598.5750122070312,
       "rewards/accuracies": 0.36250001192092896,
-      "rewards/chosen": -0.7442194819450378,
-      "rewards/margins": 0.3593973219394684,
-      "rewards/rejected": -1.103616714477539,
+      "rewards/chosen": -0.86613929271698,
+      "rewards/margins": 0.4522012174129486,
+      "rewards/rejected": -1.318340539932251,
       "step": 350,
-      "use_label": 0.0
+      "use_label": 5691.4248046875
     },
     {
       "epoch": 0.75,
-      "grad_norm": 1.859375,
+      "grad_norm": 2.234375,
       "learning_rate": 8.628481651367876e-07,
-      "logits/chosen": 0.7298086881637573,
-      "logits/rejected": 0.8517257571220398,
-      "logps/chosen": -119.41548156738281,
-      "logps/rejected": -165.3460235595703,
-      "loss": 0.6137,
-      "pred_label": 0.0,
-      "rewards/accuracies": 0.3812499940395355,
-      "rewards/chosen": -0.5577787160873413,
-      "rewards/margins": 0.37339919805526733,
-      "rewards/rejected": -0.9311779141426086,
+      "logits/chosen": 0.7010875940322876,
+      "logits/rejected": 0.8413160443305969,
+      "logps/chosen": -126.9655532836914,
+      "logps/rejected": -182.5807342529297,
+      "loss": 0.5885,
+      "pred_label": 629.0,
+      "rewards/accuracies": 0.39375001192092896,
+      "rewards/chosen": -0.6332792043685913,
+      "rewards/margins": 0.47024598717689514,
+      "rewards/rejected": -1.103525161743164,
       "step": 360,
-      "use_label": 0.0
+      "use_label": 5821.0
     },
     {
       "epoch": 0.77,
-      "grad_norm": 2.421875,
+      "grad_norm": 2.4375,
       "learning_rate": 7.289996455765749e-07,
-      "logits/chosen": 0.8383787274360657,
-      "logits/rejected": 0.9305205345153809,
-      "logps/chosen": -111.84449768066406,
-      "logps/rejected": -153.93136596679688,
-      "loss": 0.6125,
-      "pred_label": 0.0,
-      "rewards/accuracies": 0.3499999940395355,
-      "rewards/chosen": -0.46409696340560913,
-      "rewards/margins": 0.39606258273124695,
-      "rewards/rejected": -0.8601595759391785,
+      "logits/chosen": 0.8454801440238953,
+      "logits/rejected": 0.9659041166305542,
+      "logps/chosen": -120.26502990722656,
+      "logps/rejected": -170.44923400878906,
+      "loss": 0.585,
+      "pred_label": 655.625,
+      "rewards/accuracies": 0.34375,
+      "rewards/chosen": -0.5483022928237915,
+      "rewards/margins": 0.4770358204841614,
+      "rewards/rejected": -1.0253381729125977,
       "step": 370,
-      "use_label": 0.0
+      "use_label": 5954.375
     },
     {
       "epoch": 0.8,
-      "grad_norm": 1.8984375,
+      "grad_norm": 3.5625,
       "learning_rate": 6.046442623320145e-07,
-      "logits/chosen": 0.5329448580741882,
-      "logits/rejected": 0.513522744178772,
-      "logps/chosen": -116.62841796875,
-      "logps/rejected": -165.17893981933594,
-      "loss": 0.6191,
-      "pred_label": 0.0,
+      "logits/chosen": 0.7346574664115906,
+      "logits/rejected": 0.7028430104255676,
+      "logps/chosen": -131.0785675048828,
+      "logps/rejected": -188.57435607910156,
+      "loss": 0.589,
+      "pred_label": 685.5250244140625,
       "rewards/accuracies": 0.3187499940395355,
-      "rewards/chosen": -0.5079302787780762,
-      "rewards/margins": 0.2802185118198395,
-      "rewards/rejected": -0.7881487607955933,
+      "rewards/chosen": -0.6524317264556885,
+      "rewards/margins": 0.3696710765361786,
+      "rewards/rejected": -1.0221028327941895,
       "step": 380,
-      "use_label": 0.0
+      "use_label": 6084.47509765625
     },
     {
       "epoch": 0.82,
-      "grad_norm": 2.4375,
+      "grad_norm": 3.3125,
       "learning_rate": 4.904486005914027e-07,
-      "logits/chosen": 0.8266662359237671,
-      "logits/rejected": 0.5234752893447876,
-      "logps/chosen": -159.83407592773438,
-      "logps/rejected": -186.96768188476562,
-      "loss": 0.6085,
-      "pred_label": 0.0,
-      "rewards/accuracies": 0.38749998807907104,
-      "rewards/chosen": -0.6701575517654419,
-      "rewards/margins": 0.36982032656669617,
-      "rewards/rejected": -1.039977788925171,
+      "logits/chosen": 1.1143369674682617,
+      "logits/rejected": 0.8643951416015625,
+      "logps/chosen": -179.11276245117188,
+      "logps/rejected": -220.11068725585938,
+      "loss": 0.5727,
+      "pred_label": 717.625,
+      "rewards/accuracies": 0.375,
+      "rewards/chosen": -0.8629444241523743,
+      "rewards/margins": 0.508463442325592,
+      "rewards/rejected": -1.3714077472686768,
       "step": 390,
-      "use_label": 0.0
+      "use_label": 6212.375
     },
     {
       "epoch": 0.84,
-      "grad_norm": 2.46875,
+      "grad_norm": 3.25,
       "learning_rate": 3.8702478614051353e-07,
-      "logits/chosen": 0.511390745639801,
-      "logits/rejected": 0.6720080971717834,
-      "logps/chosen": -116.7987060546875,
-      "logps/rejected": -141.3931884765625,
-      "loss": 0.6139,
-      "pred_label": 0.0,
-      "rewards/accuracies": 0.3812499940395355,
-      "rewards/chosen": -0.4430771768093109,
-      "rewards/margins": 0.3362268805503845,
-      "rewards/rejected": -0.779304027557373,
+      "logits/chosen": 0.8043449521064758,
+      "logits/rejected": 0.9917415380477905,
+      "logps/chosen": -130.07017517089844,
+      "logps/rejected": -163.469970703125,
+      "loss": 0.5836,
+      "pred_label": 747.5750122070312,
+      "rewards/accuracies": 0.375,
+      "rewards/chosen": -0.5757918953895569,
+      "rewards/margins": 0.42427974939346313,
+      "rewards/rejected": -1.0000715255737305,
       "step": 400,
-      "use_label": 0.0
+      "use_label": 6342.4248046875
     },
     {
       "epoch": 0.84,
-      "eval_logits/chosen": 1.4532994031906128,
-      "eval_logits/rejected": 1.5453113317489624,
-      "eval_logps/chosen": -112.56050109863281,
-      "eval_logps/rejected": -162.19764709472656,
-      "eval_loss": 0.6157013177871704,
-      "eval_pred_label": 0.0,
+      "eval_logits/chosen": 1.75760817527771,
+      "eval_logits/rejected": 1.8499951362609863,
+      "eval_logps/chosen": -130.3719482421875,
+      "eval_logps/rejected": -190.7267303466797,
+      "eval_loss": 0.5768851041793823,
+      "eval_pred_label": 782.8125,
       "eval_rewards/accuracies": 0.37109375,
-      "eval_rewards/chosen": -0.4864824414253235,
-      "eval_rewards/margins": 0.36356455087661743,
-      "eval_rewards/rejected": -0.8500469923019409,
-      "eval_runtime": 125.4203,
-      "eval_samples_per_second": 15.946,
-      "eval_steps_per_second": 0.255,
-      "eval_use_label": 0.0,
+      "eval_rewards/chosen": -0.6645968556404114,
+      "eval_rewards/margins": 0.4707409739494324,
+      "eval_rewards/rejected": -1.1353378295898438,
+      "eval_runtime": 147.391,
+      "eval_samples_per_second": 13.569,
+      "eval_steps_per_second": 0.217,
+      "eval_use_label": 6517.1875,
       "step": 400
     },
     {
       "epoch": 0.86,
-      "grad_norm": 2.203125,
+      "grad_norm": 3.390625,
       "learning_rate": 2.9492720416985004e-07,
-      "logits/chosen": 0.8359997868537903,
-      "logits/rejected": 0.8144146800041199,
-      "logps/chosen": -110.30177307128906,
-      "logps/rejected": -143.6800079345703,
-      "loss": 0.6222,
-      "pred_label": 0.0,
-      "rewards/accuracies": 0.39375001192092896,
-      "rewards/chosen": -0.4887877404689789,
-      "rewards/margins": 0.3508199453353882,
-      "rewards/rejected": -0.8396075963973999,
+      "logits/chosen": 1.1002473831176758,
+      "logits/rejected": 1.1428117752075195,
+      "logps/chosen": -126.85247802734375,
+      "logps/rejected": -170.77365112304688,
+      "loss": 0.5838,
+      "pred_label": 822.75,
+      "rewards/accuracies": 0.3687500059604645,
+      "rewards/chosen": -0.6542948484420776,
+      "rewards/margins": 0.4562492370605469,
+      "rewards/rejected": -1.1105440855026245,
       "step": 410,
-      "use_label": 0.0
+      "use_label": 6683.25
     },
     {
       "epoch": 0.88,
-      "grad_norm": 1.984375,
+      "grad_norm": 2.609375,
       "learning_rate": 2.1464952759020857e-07,
-      "logits/chosen": 1.027252435684204,
-      "logits/rejected": 0.9827619791030884,
-      "logps/chosen": -106.49784851074219,
-      "logps/rejected": -116.97566223144531,
-      "loss": 0.6216,
-      "pred_label": 0.0,
-      "rewards/accuracies": 0.2750000059604645,
-      "rewards/chosen": -0.4555872976779938,
-      "rewards/margins": 0.20033884048461914,
-      "rewards/rejected": -0.6559261083602905,
+      "logits/chosen": 1.3246395587921143,
+      "logits/rejected": 1.2824434041976929,
+      "logps/chosen": -122.80003356933594,
+      "logps/rejected": -138.56423950195312,
+      "loss": 0.5822,
+      "pred_label": 846.4249877929688,
+      "rewards/accuracies": 0.26249998807907104,
+      "rewards/chosen": -0.6186091303825378,
+      "rewards/margins": 0.25320303440093994,
+      "rewards/rejected": -0.8718121647834778,
       "step": 420,
-      "use_label": 0.0
+      "use_label": 6819.5751953125
     },
     {
       "epoch": 0.9,
-      "grad_norm": 1.96875,
+      "grad_norm": 2.09375,
       "learning_rate": 1.4662207078575685e-07,
-      "logits/chosen": 0.9206047058105469,
-      "logits/rejected": 0.8673297166824341,
-      "logps/chosen": -151.376220703125,
-      "logps/rejected": -178.04725646972656,
-      "loss": 0.5986,
-      "pred_label": 0.0,
-      "rewards/accuracies": 0.45625001192092896,
-      "rewards/chosen": -0.5210937261581421,
-      "rewards/margins": 0.46580758690834045,
-      "rewards/rejected": -0.9869012832641602,
+      "logits/chosen": 1.270193099975586,
+      "logits/rejected": 1.253873348236084,
+      "logps/chosen": -171.46336364746094,
+      "logps/rejected": -207.75607299804688,
+      "loss": 0.564,
+      "pred_label": 873.5499877929688,
+      "rewards/accuracies": 0.4625000059604645,
+      "rewards/chosen": -0.7219651341438293,
+      "rewards/margins": 0.5620242357254028,
+      "rewards/rejected": -1.283989429473877,
       "step": 430,
-      "use_label": 0.0
+      "use_label": 6952.4501953125
     },
     {
       "epoch": 0.92,
-      "grad_norm": 2.125,
+      "grad_norm": 2.578125,
       "learning_rate": 9.120948298936422e-08,
-      "logits/chosen": 0.9004503488540649,
-      "logits/rejected": 1.0573413372039795,
-      "logps/chosen": -119.21500396728516,
-      "logps/rejected": -165.19241333007812,
-      "loss": 0.6064,
-      "pred_label": 0.0,
-      "rewards/accuracies": 0.35624998807907104,
-      "rewards/chosen": -0.5231102705001831,
-      "rewards/margins": 0.37818416953086853,
-      "rewards/rejected": -0.9012944102287292,
+      "logits/chosen": 1.221411943435669,
+      "logits/rejected": 1.397247552871704,
+      "logps/chosen": -136.4575653076172,
+      "logps/rejected": -193.40870666503906,
+      "loss": 0.5736,
+      "pred_label": 905.5750122070312,
+      "rewards/accuracies": 0.36250001192092896,
+      "rewards/chosen": -0.6955360770225525,
+      "rewards/margins": 0.4879213869571686,
+      "rewards/rejected": -1.183457612991333,
       "step": 440,
-      "use_label": 0.0
+      "use_label": 7080.4248046875
     },
     {
       "epoch": 0.94,
-      "grad_norm": 2.46875,
+      "grad_norm": 4.78125,
       "learning_rate": 4.870879364444109e-08,
-      "logits/chosen": 1.300728440284729,
-      "logits/rejected": 1.0580918788909912,
-      "logps/chosen": -129.29281616210938,
-      "logps/rejected": -178.3690948486328,
-      "loss": 0.6111,
-      "pred_label": 0.0,
-      "rewards/accuracies": 0.3499999940395355,
-      "rewards/chosen": -0.570349931716919,
-      "rewards/margins": 0.3304445147514343,
-      "rewards/rejected": -0.9007943868637085,
+      "logits/chosen": 1.6054052114486694,
+      "logits/rejected": 1.3484258651733398,
+      "logps/chosen": -148.17161560058594,
+      "logps/rejected": -205.789306640625,
+      "loss": 0.583,
+      "pred_label": 930.4749755859375,
+      "rewards/accuracies": 0.35624998807907104,
+      "rewards/chosen": -0.7591380476951599,
+      "rewards/margins": 0.4158584177494049,
+      "rewards/rejected": -1.1749964952468872,
       "step": 450,
-      "use_label": 0.0
+      "use_label": 7215.52490234375
     },
     {
       "epoch": 0.96,
-      "grad_norm": 1.8359375,
+      "grad_norm": 2.875,
       "learning_rate": 1.93478202307823e-08,
-      "logits/chosen": 1.1906068325042725,
-      "logits/rejected": 1.2149587869644165,
-      "logps/chosen": -83.74864196777344,
-      "logps/rejected": -130.91348266601562,
-      "loss": 0.6154,
-      "pred_label": 0.0,
-      "rewards/accuracies": 0.3375000059604645,
-      "rewards/chosen": -0.3762877583503723,
-      "rewards/margins": 0.2993956208229065,
-      "rewards/rejected": -0.6756833791732788,
+      "logits/chosen": 1.4640157222747803,
+      "logits/rejected": 1.4903802871704102,
+      "logps/chosen": -96.6323471069336,
+      "logps/rejected": -150.8868865966797,
+      "loss": 0.5814,
+      "pred_label": 961.25,
+      "rewards/accuracies": 0.32499998807907104,
+      "rewards/chosen": -0.5051247477531433,
+      "rewards/margins": 0.3702928125858307,
+      "rewards/rejected": -0.8754175901412964,
       "step": 460,
-      "use_label": 0.0
+      "use_label": 7344.75
     },
     {
       "epoch": 0.98,
-      "grad_norm": 2.375,
+      "grad_norm": 2.75,
       "learning_rate": 3.283947088983663e-09,
-      "logits/chosen": 1.1844379901885986,
-      "logits/rejected": 0.9474547505378723,
-      "logps/chosen": -113.1079330444336,
-      "logps/rejected": -141.49147033691406,
-      "loss": 0.6213,
-      "pred_label": 0.0,
-      "rewards/accuracies": 0.3125,
-      "rewards/chosen": -0.4577876627445221,
-      "rewards/margins": 0.26655709743499756,
-      "rewards/rejected": -0.7243447303771973,
+      "logits/chosen": 1.464422345161438,
+      "logits/rejected": 1.2297132015228271,
+      "logps/chosen": -130.30838012695312,
+      "logps/rejected": -166.67605590820312,
+      "loss": 0.5822,
+      "pred_label": 982.8499755859375,
+      "rewards/accuracies": 0.30000001192092896,
+      "rewards/chosen": -0.6297920346260071,
+      "rewards/margins": 0.34639838337898254,
+      "rewards/rejected": -0.9761903882026672,
       "step": 470,
-      "use_label": 0.0
+      "use_label": 7483.14990234375
     },
     {
       "epoch": 1.0,
       "step": 477,
       "total_flos": 0.0,
-      "train_loss": 0.6357159084743924,
-      "train_runtime": 9601.7268,
-      "train_samples_per_second": 6.367,
-      "train_steps_per_second": 0.05
+      "train_loss": 0.6110695428068533,
+      "train_runtime": 9999.3279,
+      "train_samples_per_second": 6.114,
+      "train_steps_per_second": 0.048
     }
   ],
   "logging_steps": 10,