diff --git "a/trainer_state.json" "b/trainer_state.json"
new file mode 100644--- /dev/null
+++ "b/trainer_state.json"
@@ -0,0 +1,19313 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 2.0,
+  "eval_steps": 100,
+  "global_step": 11608,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.00017229496898690558,
+      "grad_norm": 8.949012284889877,
+      "learning_rate": 4.306632213608957e-11,
+      "logits/chosen": -3.5275135040283203,
+      "logits/rejected": -3.49973726272583,
+      "logps/chosen": -1.2767510414123535,
+      "logps/rejected": -1.5448579788208008,
+      "loss": 1.0376,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.2767510414123535,
+      "rewards/margins": 0.26810693740844727,
+      "rewards/rejected": -1.5448579788208008,
+      "step": 1
+    },
+    {
+      "epoch": 0.0017229496898690559,
+      "grad_norm": 10.271383221266612,
+      "learning_rate": 4.306632213608958e-10,
+      "logits/chosen": -3.564272880554199,
+      "logits/rejected": -3.5475454330444336,
+      "logps/chosen": -1.3870049715042114,
+      "logps/rejected": -1.5200228691101074,
+      "loss": 1.1191,
+      "rewards/accuracies": 0.6041666865348816,
+      "rewards/chosen": -1.3870049715042114,
+      "rewards/margins": 0.13301779329776764,
+      "rewards/rejected": -1.5200228691101074,
+      "step": 10
+    },
+    {
+      "epoch": 0.0034458993797381117,
+      "grad_norm": 8.807724032404167,
+      "learning_rate": 8.613264427217916e-10,
+      "logits/chosen": -3.6084179878234863,
+      "logits/rejected": -3.6022820472717285,
+      "logps/chosen": -1.4676027297973633,
+      "logps/rejected": -1.54346764087677,
+      "loss": 1.1699,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -1.4676027297973633,
+      "rewards/margins": 0.07586488872766495,
+      "rewards/rejected": -1.54346764087677,
+      "step": 20
+    },
+    {
+      "epoch": 0.005168849069607168,
+      "grad_norm": 9.383758685427086,
+      "learning_rate": 1.2919896640826872e-09,
+      "logits/chosen": -3.5949344635009766,
+      "logits/rejected": -3.580770969390869,
+      "logps/chosen": -1.482218623161316,
+      "logps/rejected": -1.6947721242904663,
+      "loss": 1.0786,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.482218623161316,
+      "rewards/margins": 0.2125532627105713,
+      "rewards/rejected": -1.6947721242904663,
+      "step": 30
+    },
+    {
+      "epoch": 0.006891798759476223,
+      "grad_norm": 9.32720295342479,
+      "learning_rate": 1.7226528854435832e-09,
+      "logits/chosen": -3.5910496711730957,
+      "logits/rejected": -3.5735466480255127,
+      "logps/chosen": -1.4905458688735962,
+      "logps/rejected": -1.553086519241333,
+      "loss": 1.1717,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -1.4905458688735962,
+      "rewards/margins": 0.06254057586193085,
+      "rewards/rejected": -1.553086519241333,
+      "step": 40
+    },
+    {
+      "epoch": 0.00861474844934528,
+      "grad_norm": 10.337830840347708,
+      "learning_rate": 2.153316106804479e-09,
+      "logits/chosen": -3.5898654460906982,
+      "logits/rejected": -3.579383134841919,
+      "logps/chosen": -1.4443176984786987,
+      "logps/rejected": -1.5602178573608398,
+      "loss": 1.1421,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -1.4443176984786987,
+      "rewards/margins": 0.11590013653039932,
+      "rewards/rejected": -1.5602178573608398,
+      "step": 50
+    },
+    {
+      "epoch": 0.010337698139214336,
+      "grad_norm": 8.571140144177852,
+      "learning_rate": 2.5839793281653743e-09,
+      "logits/chosen": -3.6229915618896484,
+      "logits/rejected": -3.6051769256591797,
+      "logps/chosen": -1.4595787525177002,
+      "logps/rejected": -1.5307068824768066,
+      "loss": 1.1594,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -1.4595787525177002,
+      "rewards/margins": 0.07112816721200943,
+      "rewards/rejected": -1.5307068824768066,
+      "step": 60
+    },
+    {
+      "epoch": 0.012060647829083391,
+      "grad_norm": 9.643107858881745,
+      "learning_rate": 3.01464254952627e-09,
+      "logits/chosen": -3.56121826171875,
+      "logits/rejected": -3.548574924468994,
+      "logps/chosen": -1.4370390176773071,
+      "logps/rejected": -1.5738071203231812,
+      "loss": 1.1189,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.4370390176773071,
+      "rewards/margins": 0.13676835596561432,
+      "rewards/rejected": -1.5738071203231812,
+      "step": 70
+    },
+    {
+      "epoch": 0.013783597518952447,
+      "grad_norm": 7.941916967558139,
+      "learning_rate": 3.4453057708871665e-09,
+      "logits/chosen": -3.6386570930480957,
+      "logits/rejected": -3.6183300018310547,
+      "logps/chosen": -1.4778589010238647,
+      "logps/rejected": -1.522172212600708,
+      "loss": 1.1756,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -1.4778589010238647,
+      "rewards/margins": 0.04431319981813431,
+      "rewards/rejected": -1.522172212600708,
+      "step": 80
+    },
+    {
+      "epoch": 0.015506547208821502,
+      "grad_norm": 9.652972898473822,
+      "learning_rate": 3.8759689922480615e-09,
+      "logits/chosen": -3.5194010734558105,
+      "logits/rejected": -3.5088322162628174,
+      "logps/chosen": -1.449768304824829,
+      "logps/rejected": -1.5658804178237915,
+      "loss": 1.1416,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -1.449768304824829,
+      "rewards/margins": 0.11611209809780121,
+      "rewards/rejected": -1.5658804178237915,
+      "step": 90
+    },
+    {
+      "epoch": 0.01722949689869056,
+      "grad_norm": 10.893119387170158,
+      "learning_rate": 4.306632213608958e-09,
+      "logits/chosen": -3.6415717601776123,
+      "logits/rejected": -3.606274127960205,
+      "logps/chosen": -1.4258615970611572,
+      "logps/rejected": -1.6059787273406982,
+      "loss": 1.1055,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.4258615970611572,
+      "rewards/margins": 0.18011704087257385,
+      "rewards/rejected": -1.6059787273406982,
+      "step": 100
+    },
+    {
+      "epoch": 0.01722949689869056,
+      "eval_logits/chosen": -3.636707067489624,
+      "eval_logits/rejected": -3.6325948238372803,
+      "eval_logps/chosen": -1.4808173179626465,
+      "eval_logps/rejected": -1.6083329916000366,
+      "eval_loss": 1.117266058921814,
+      "eval_rewards/accuracies": 0.5954925417900085,
+      "eval_rewards/chosen": -1.4808173179626465,
+      "eval_rewards/margins": 0.1275155246257782,
+      "eval_rewards/rejected": -1.6083329916000366,
+      "eval_runtime": 155.4413,
+      "eval_samples_per_second": 27.689,
+      "eval_steps_per_second": 3.461,
+      "step": 100
+    },
+    {
+      "epoch": 0.018952446588559616,
+      "grad_norm": 9.516819900256879,
+      "learning_rate": 4.737295434969853e-09,
+      "logits/chosen": -3.610856294631958,
+      "logits/rejected": -3.5970611572265625,
+      "logps/chosen": -1.463639497756958,
+      "logps/rejected": -1.537428617477417,
+      "loss": 1.1591,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.463639497756958,
+      "rewards/margins": 0.07378916442394257,
+      "rewards/rejected": -1.537428617477417,
+      "step": 110
+    },
+    {
+      "epoch": 0.02067539627842867,
+      "grad_norm": 8.693214992115893,
+      "learning_rate": 5.167958656330749e-09,
+      "logits/chosen": -3.575594663619995,
+      "logits/rejected": -3.566729784011841,
+      "logps/chosen": -1.4178352355957031,
+      "logps/rejected": -1.626700758934021,
+      "loss": 1.0798,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.4178352355957031,
+      "rewards/margins": 0.20886549353599548,
+      "rewards/rejected": -1.626700758934021,
+      "step": 120
+    },
+    {
+      "epoch": 0.022398345968297727,
+      "grad_norm": 7.793960960717675,
+      "learning_rate": 5.598621877691645e-09,
+      "logits/chosen": -3.59881591796875,
+      "logits/rejected": -3.5898208618164062,
+      "logps/chosen": -1.458284854888916,
+      "logps/rejected": -1.5261564254760742,
+      "loss": 1.1603,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -1.458284854888916,
+      "rewards/margins": 0.06787154078483582,
+      "rewards/rejected": -1.5261564254760742,
+      "step": 130
+    },
+    {
+      "epoch": 0.024121295658166782,
+      "grad_norm": 10.680029086939655,
+      "learning_rate": 6.02928509905254e-09,
+      "logits/chosen": -3.6059634685516357,
+      "logits/rejected": -3.5932743549346924,
+      "logps/chosen": -1.4701769351959229,
+      "logps/rejected": -1.6092592477798462,
+      "loss": 1.1242,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.4701769351959229,
+      "rewards/margins": 0.1390821635723114,
+      "rewards/rejected": -1.6092592477798462,
+      "step": 140
+    },
+    {
+      "epoch": 0.025844245348035838,
+      "grad_norm": 8.398917910742579,
+      "learning_rate": 6.459948320413436e-09,
+      "logits/chosen": -3.539930820465088,
+      "logits/rejected": -3.529597520828247,
+      "logps/chosen": -1.462100863456726,
+      "logps/rejected": -1.5747325420379639,
+      "loss": 1.1311,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.462100863456726,
+      "rewards/margins": 0.11263159662485123,
+      "rewards/rejected": -1.5747325420379639,
+      "step": 150
+    },
+    {
+      "epoch": 0.027567195037904894,
+      "grad_norm": 8.738882216260961,
+      "learning_rate": 6.890611541774333e-09,
+      "logits/chosen": -3.572937488555908,
+      "logits/rejected": -3.5609943866729736,
+      "logps/chosen": -1.4516582489013672,
+      "logps/rejected": -1.586464285850525,
+      "loss": 1.1165,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.4516582489013672,
+      "rewards/margins": 0.13480611145496368,
+      "rewards/rejected": -1.586464285850525,
+      "step": 160
+    },
+    {
+      "epoch": 0.02929014472777395,
+      "grad_norm": 9.150895607833515,
+      "learning_rate": 7.321274763135228e-09,
+      "logits/chosen": -3.5841903686523438,
+      "logits/rejected": -3.573251724243164,
+      "logps/chosen": -1.4862561225891113,
+      "logps/rejected": -1.5192008018493652,
+      "loss": 1.1918,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -1.4862561225891113,
+      "rewards/margins": 0.03294467180967331,
+      "rewards/rejected": -1.5192008018493652,
+      "step": 170
+    },
+    {
+      "epoch": 0.031013094417643005,
+      "grad_norm": 10.762997709537508,
+      "learning_rate": 7.751937984496123e-09,
+      "logits/chosen": -3.574800491333008,
+      "logits/rejected": -3.563720703125,
+      "logps/chosen": -1.6037276983261108,
+      "logps/rejected": -1.749906301498413,
+      "loss": 1.1572,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -1.6037276983261108,
+      "rewards/margins": 0.1461784541606903,
+      "rewards/rejected": -1.749906301498413,
+      "step": 180
+    },
+    {
+      "epoch": 0.03273604410751206,
+      "grad_norm": 9.588291178042303,
+      "learning_rate": 8.18260120585702e-09,
+      "logits/chosen": -3.6010947227478027,
+      "logits/rejected": -3.5747103691101074,
+      "logps/chosen": -1.5290495157241821,
+      "logps/rejected": -1.649243712425232,
+      "loss": 1.1425,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.5290495157241821,
+      "rewards/margins": 0.12019409239292145,
+      "rewards/rejected": -1.649243712425232,
+      "step": 190
+    },
+    {
+      "epoch": 0.03445899379738112,
+      "grad_norm": 8.033047408729338,
+      "learning_rate": 8.613264427217916e-09,
+      "logits/chosen": -3.568943738937378,
+      "logits/rejected": -3.559791088104248,
+      "logps/chosen": -1.5066125392913818,
+      "logps/rejected": -1.663662314414978,
+      "loss": 1.1204,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.5066125392913818,
+      "rewards/margins": 0.15704968571662903,
+      "rewards/rejected": -1.663662314414978,
+      "step": 200
+    },
+    {
+      "epoch": 0.03445899379738112,
+      "eval_logits/chosen": -3.6430861949920654,
+      "eval_logits/rejected": -3.639011859893799,
+      "eval_logps/chosen": -1.4808188676834106,
+      "eval_logps/rejected": -1.608304738998413,
+      "eval_loss": 1.1172897815704346,
+      "eval_rewards/accuracies": 0.5947955250740051,
+      "eval_rewards/chosen": -1.4808188676834106,
+      "eval_rewards/margins": 0.12748593091964722,
+      "eval_rewards/rejected": -1.608304738998413,
+      "eval_runtime": 155.4798,
+      "eval_samples_per_second": 27.682,
+      "eval_steps_per_second": 3.46,
+      "step": 200
+    },
+    {
+      "epoch": 0.03618194348725017,
+      "grad_norm": 8.661508078790767,
+      "learning_rate": 9.043927648578812e-09,
+      "logits/chosen": -3.5436654090881348,
+      "logits/rejected": -3.5380516052246094,
+      "logps/chosen": -1.4893193244934082,
+      "logps/rejected": -1.690930962562561,
+      "loss": 1.0779,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.4893193244934082,
+      "rewards/margins": 0.2016117125749588,
+      "rewards/rejected": -1.690930962562561,
+      "step": 210
+    },
+    {
+      "epoch": 0.03790489317711923,
+      "grad_norm": 9.492269329889085,
+      "learning_rate": 9.474590869939706e-09,
+      "logits/chosen": -3.566500186920166,
+      "logits/rejected": -3.5469253063201904,
+      "logps/chosen": -1.4146173000335693,
+      "logps/rejected": -1.5534894466400146,
+      "loss": 1.1167,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.4146173000335693,
+      "rewards/margins": 0.1388724148273468,
+      "rewards/rejected": -1.5534894466400146,
+      "step": 220
+    },
+    {
+      "epoch": 0.03962784286698828,
+      "grad_norm": 8.286211126536468,
+      "learning_rate": 9.905254091300603e-09,
+      "logits/chosen": -3.567793369293213,
+      "logits/rejected": -3.5564846992492676,
+      "logps/chosen": -1.4027880430221558,
+      "logps/rejected": -1.5592424869537354,
+      "loss": 1.109,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -1.4027880430221558,
+      "rewards/margins": 0.15645429491996765,
+      "rewards/rejected": -1.5592424869537354,
+      "step": 230
+    },
+    {
+      "epoch": 0.04135079255685734,
+      "grad_norm": 7.269968276568492,
+      "learning_rate": 1.0335917312661497e-08,
+      "logits/chosen": -3.5572662353515625,
+      "logits/rejected": -3.532341718673706,
+      "logps/chosen": -1.4713029861450195,
+      "logps/rejected": -1.5996345281600952,
+      "loss": 1.1293,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.4713029861450195,
+      "rewards/margins": 0.12833161652088165,
+      "rewards/rejected": -1.5996345281600952,
+      "step": 240
+    },
+    {
+      "epoch": 0.043073742246726394,
+      "grad_norm": 9.404350753071236,
+      "learning_rate": 1.0766580534022395e-08,
+      "logits/chosen": -3.604480743408203,
+      "logits/rejected": -3.5943679809570312,
+      "logps/chosen": -1.445573329925537,
+      "logps/rejected": -1.5041121244430542,
+      "loss": 1.1653,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -1.445573329925537,
+      "rewards/margins": 0.05853888392448425,
+      "rewards/rejected": -1.5041121244430542,
+      "step": 250
+    },
+    {
+      "epoch": 0.044796691936595454,
+      "grad_norm": 9.261925491005883,
+      "learning_rate": 1.119724375538329e-08,
+      "logits/chosen": -3.594891309738159,
+      "logits/rejected": -3.587692975997925,
+      "logps/chosen": -1.472755789756775,
+      "logps/rejected": -1.6015924215316772,
+      "loss": 1.1222,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.472755789756775,
+      "rewards/margins": 0.1288367062807083,
+      "rewards/rejected": -1.6015924215316772,
+      "step": 260
+    },
+    {
+      "epoch": 0.046519641626464506,
+      "grad_norm": 10.334646467597846,
+      "learning_rate": 1.1627906976744186e-08,
+      "logits/chosen": -3.5516746044158936,
+      "logits/rejected": -3.540189743041992,
+      "logps/chosen": -1.5366517305374146,
+      "logps/rejected": -1.6552066802978516,
+      "loss": 1.1333,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.5366517305374146,
+      "rewards/margins": 0.11855493485927582,
+      "rewards/rejected": -1.6552066802978516,
+      "step": 270
+    },
+    {
+      "epoch": 0.048242591316333565,
+      "grad_norm": 9.258572354059487,
+      "learning_rate": 1.205857019810508e-08,
+      "logits/chosen": -3.608914852142334,
+      "logits/rejected": -3.587123155593872,
+      "logps/chosen": -1.479244589805603,
+      "logps/rejected": -1.5783261060714722,
+      "loss": 1.1504,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -1.479244589805603,
+      "rewards/margins": 0.09908171743154526,
+      "rewards/rejected": -1.5783261060714722,
+      "step": 280
+    },
+    {
+      "epoch": 0.04996554100620262,
+      "grad_norm": 8.116370905407544,
+      "learning_rate": 1.2489233419465977e-08,
+      "logits/chosen": -3.553650379180908,
+      "logits/rejected": -3.545550584793091,
+      "logps/chosen": -1.492806315422058,
+      "logps/rejected": -1.5809742212295532,
+      "loss": 1.1492,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -1.492806315422058,
+      "rewards/margins": 0.08816774189472198,
+      "rewards/rejected": -1.5809742212295532,
+      "step": 290
+    },
+    {
+      "epoch": 0.051688490696071676,
+      "grad_norm": 8.331034353765961,
+      "learning_rate": 1.2919896640826872e-08,
+      "logits/chosen": -3.5365421772003174,
+      "logits/rejected": -3.528935194015503,
+      "logps/chosen": -1.4044264554977417,
+      "logps/rejected": -1.594384789466858,
+      "loss": 1.0875,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.4044264554977417,
+      "rewards/margins": 0.1899583339691162,
+      "rewards/rejected": -1.594384789466858,
+      "step": 300
+    },
+    {
+      "epoch": 0.051688490696071676,
+      "eval_logits/chosen": -3.633157730102539,
+      "eval_logits/rejected": -3.6290299892425537,
+      "eval_logps/chosen": -1.4810305833816528,
+      "eval_logps/rejected": -1.6083561182022095,
+      "eval_loss": 1.117384672164917,
+      "eval_rewards/accuracies": 0.5954925417900085,
+      "eval_rewards/chosen": -1.4810305833816528,
+      "eval_rewards/margins": 0.12732549011707306,
+      "eval_rewards/rejected": -1.6083561182022095,
+      "eval_runtime": 155.4833,
+      "eval_samples_per_second": 27.681,
+      "eval_steps_per_second": 3.46,
+      "step": 300
+    },
+    {
+      "epoch": 0.05341144038594073,
+      "grad_norm": 11.616099027271112,
+      "learning_rate": 1.3350559862187768e-08,
+      "logits/chosen": -3.5767135620117188,
+      "logits/rejected": -3.5719380378723145,
+      "logps/chosen": -1.5296727418899536,
+      "logps/rejected": -1.600365400314331,
+      "loss": 1.1657,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.5296727418899536,
+      "rewards/margins": 0.07069291919469833,
+      "rewards/rejected": -1.600365400314331,
+      "step": 310
+    },
+    {
+      "epoch": 0.05513439007580979,
+      "grad_norm": 17.613129571399952,
+      "learning_rate": 1.3781223083548666e-08,
+      "logits/chosen": -3.5488758087158203,
+      "logits/rejected": -3.533463716506958,
+      "logps/chosen": -1.4758307933807373,
+      "logps/rejected": -1.4959453344345093,
+      "loss": 1.1963,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -1.4758307933807373,
+      "rewards/margins": 0.02011440508067608,
+      "rewards/rejected": -1.4959453344345093,
+      "step": 320
+    },
+    {
+      "epoch": 0.05685733976567884,
+      "grad_norm": 9.227933085415176,
+      "learning_rate": 1.4211886304909559e-08,
+      "logits/chosen": -3.5872511863708496,
+      "logits/rejected": -3.573986530303955,
+      "logps/chosen": -1.4811230897903442,
+      "logps/rejected": -1.5493927001953125,
+      "loss": 1.1671,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -1.4811230897903442,
+      "rewards/margins": 0.06826958805322647,
+      "rewards/rejected": -1.5493927001953125,
+      "step": 330
+    },
+    {
+      "epoch": 0.0585802894555479,
+      "grad_norm": 8.958599303148628,
+      "learning_rate": 1.4642549526270457e-08,
+      "logits/chosen": -3.5354301929473877,
+      "logits/rejected": -3.5220768451690674,
+      "logps/chosen": -1.4512560367584229,
+      "logps/rejected": -1.5167008638381958,
+      "loss": 1.1665,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.4512560367584229,
+      "rewards/margins": 0.06544484198093414,
+      "rewards/rejected": -1.5167008638381958,
+      "step": 340
+    },
+    {
+      "epoch": 0.06030323914541695,
+      "grad_norm": 8.585435040725525,
+      "learning_rate": 1.507321274763135e-08,
+      "logits/chosen": -3.5081679821014404,
+      "logits/rejected": -3.485311508178711,
+      "logps/chosen": -1.5003323554992676,
+      "logps/rejected": -1.6760448217391968,
+      "loss": 1.1092,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.5003323554992676,
+      "rewards/margins": 0.1757124662399292,
+      "rewards/rejected": -1.6760448217391968,
+      "step": 350
+    },
+    {
+      "epoch": 0.06202618883528601,
+      "grad_norm": 9.615196495392787,
+      "learning_rate": 1.5503875968992246e-08,
+      "logits/chosen": -3.619009494781494,
+      "logits/rejected": -3.606295347213745,
+      "logps/chosen": -1.4811457395553589,
+      "logps/rejected": -1.562282681465149,
+      "loss": 1.1532,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.4811457395553589,
+      "rewards/margins": 0.08113692700862885,
+      "rewards/rejected": -1.562282681465149,
+      "step": 360
+    },
+    {
+      "epoch": 0.06374913852515507,
+      "grad_norm": 9.513808789083624,
+      "learning_rate": 1.5934539190353144e-08,
+      "logits/chosen": -3.601954698562622,
+      "logits/rejected": -3.5856614112854004,
+      "logps/chosen": -1.4202988147735596,
+      "logps/rejected": -1.5544952154159546,
+      "loss": 1.1197,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.4202988147735596,
+      "rewards/margins": 0.1341966688632965,
+      "rewards/rejected": -1.5544952154159546,
+      "step": 370
+    },
+    {
+      "epoch": 0.06547208821502412,
+      "grad_norm": 7.765232654492045,
+      "learning_rate": 1.636520241171404e-08,
+      "logits/chosen": -3.663115978240967,
+      "logits/rejected": -3.6488184928894043,
+      "logps/chosen": -1.4556467533111572,
+      "logps/rejected": -1.5667085647583008,
+      "loss": 1.1358,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.4556467533111572,
+      "rewards/margins": 0.111061692237854,
+      "rewards/rejected": -1.5667085647583008,
+      "step": 380
+    },
+    {
+      "epoch": 0.06719503790489317,
+      "grad_norm": 9.049479010749161,
+      "learning_rate": 1.6795865633074936e-08,
+      "logits/chosen": -3.6008517742156982,
+      "logits/rejected": -3.585549831390381,
+      "logps/chosen": -1.4624325037002563,
+      "logps/rejected": -1.6187137365341187,
+      "loss": 1.1048,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.4624325037002563,
+      "rewards/margins": 0.15628132224082947,
+      "rewards/rejected": -1.6187137365341187,
+      "step": 390
+    },
+    {
+      "epoch": 0.06891798759476224,
+      "grad_norm": 10.547979378381829,
+      "learning_rate": 1.722652885443583e-08,
+      "logits/chosen": -3.575611114501953,
+      "logits/rejected": -3.5652852058410645,
+      "logps/chosen": -1.4778941869735718,
+      "logps/rejected": -1.6164754629135132,
+      "loss": 1.1119,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.4778941869735718,
+      "rewards/margins": 0.1385813057422638,
+      "rewards/rejected": -1.6164754629135132,
+      "step": 400
+    },
+    {
+      "epoch": 0.06891798759476224,
+      "eval_logits/chosen": -3.660348415374756,
+      "eval_logits/rejected": -3.656350612640381,
+      "eval_logps/chosen": -1.4808119535446167,
+      "eval_logps/rejected": -1.6081782579421997,
+      "eval_loss": 1.117311716079712,
+      "eval_rewards/accuracies": 0.5943308472633362,
+      "eval_rewards/chosen": -1.4808119535446167,
+      "eval_rewards/margins": 0.12736624479293823,
+      "eval_rewards/rejected": -1.6081782579421997,
+      "eval_runtime": 155.6016,
+      "eval_samples_per_second": 27.66,
+      "eval_steps_per_second": 3.458,
+      "step": 400
+    },
+    {
+      "epoch": 0.07064093728463129,
+      "grad_norm": 8.38452109106696,
+      "learning_rate": 1.7657192075796726e-08,
+      "logits/chosen": -3.5848166942596436,
+      "logits/rejected": -3.5825557708740234,
+      "logps/chosen": -1.4075849056243896,
+      "logps/rejected": -1.556728720664978,
+      "loss": 1.1109,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.4075849056243896,
+      "rewards/margins": 0.14914394915103912,
+      "rewards/rejected": -1.556728720664978,
+      "step": 410
+    },
+    {
+      "epoch": 0.07236388697450034,
+      "grad_norm": 8.015195542079804,
+      "learning_rate": 1.8087855297157624e-08,
+      "logits/chosen": -3.579657793045044,
+      "logits/rejected": -3.5748729705810547,
+      "logps/chosen": -1.4747427701950073,
+      "logps/rejected": -1.5332664251327515,
+      "loss": 1.1704,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -1.4747427701950073,
+      "rewards/margins": 0.05852360278367996,
+      "rewards/rejected": -1.5332664251327515,
+      "step": 420
+    },
+    {
+      "epoch": 0.0740868366643694,
+      "grad_norm": 9.503724429232618,
+      "learning_rate": 1.8518518518518518e-08,
+      "logits/chosen": -3.5929627418518066,
+      "logits/rejected": -3.5842068195343018,
+      "logps/chosen": -1.4139988422393799,
+      "logps/rejected": -1.5562350749969482,
+      "loss": 1.1175,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -1.4139988422393799,
+      "rewards/margins": 0.14223656058311462,
+      "rewards/rejected": -1.5562350749969482,
+      "step": 430
+    },
+    {
+      "epoch": 0.07580978635423846,
+      "grad_norm": 7.615632114423127,
+      "learning_rate": 1.8949181739879413e-08,
+      "logits/chosen": -3.609557628631592,
+      "logits/rejected": -3.5880050659179688,
+      "logps/chosen": -1.4261208772659302,
+      "logps/rejected": -1.6449368000030518,
+      "loss": 1.0713,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.4261208772659302,
+      "rewards/margins": 0.21881604194641113,
+      "rewards/rejected": -1.6449368000030518,
+      "step": 440
+    },
+    {
+      "epoch": 0.07753273604410751,
+      "grad_norm": 8.491886356072635,
+      "learning_rate": 1.937984496124031e-08,
+      "logits/chosen": -3.565767288208008,
+      "logits/rejected": -3.5463497638702393,
+      "logps/chosen": -1.4551998376846313,
+      "logps/rejected": -1.6312446594238281,
+      "loss": 1.1014,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.4551998376846313,
+      "rewards/margins": 0.1760447472333908,
+      "rewards/rejected": -1.6312446594238281,
+      "step": 450
+    },
+    {
+      "epoch": 0.07925568573397657,
+      "grad_norm": 8.408991069212364,
+      "learning_rate": 1.9810508182601205e-08,
+      "logits/chosen": -3.5486748218536377,
+      "logits/rejected": -3.5363166332244873,
+      "logps/chosen": -1.4897191524505615,
+      "logps/rejected": -1.52727210521698,
+      "loss": 1.1975,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -1.4897191524505615,
+      "rewards/margins": 0.03755301982164383,
+      "rewards/rejected": -1.52727210521698,
+      "step": 460
+    },
+    {
+      "epoch": 0.08097863542384562,
+      "grad_norm": 9.79118024915952,
+      "learning_rate": 2.02411714039621e-08,
+      "logits/chosen": -3.5543625354766846,
+      "logits/rejected": -3.5348598957061768,
+      "logps/chosen": -1.4300401210784912,
+      "logps/rejected": -1.565441370010376,
+      "loss": 1.1121,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.4300401210784912,
+      "rewards/margins": 0.13540129363536835,
+      "rewards/rejected": -1.565441370010376,
+      "step": 470
+    },
+    {
+      "epoch": 0.08270158511371468,
+      "grad_norm": 9.8634683958523,
+      "learning_rate": 2.0671834625322995e-08,
+      "logits/chosen": -3.553302764892578,
+      "logits/rejected": -3.5521228313446045,
+      "logps/chosen": -1.4453966617584229,
+      "logps/rejected": -1.6318104267120361,
+      "loss": 1.0998,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.4453966617584229,
+      "rewards/margins": 0.1864137351512909,
+      "rewards/rejected": -1.6318104267120361,
+      "step": 480
+    },
+    {
+      "epoch": 0.08442453480358374,
+      "grad_norm": 9.695324260579032,
+      "learning_rate": 2.1102497846683892e-08,
+      "logits/chosen": -3.499178409576416,
+      "logits/rejected": -3.468339443206787,
+      "logps/chosen": -1.521965742111206,
+      "logps/rejected": -1.623814582824707,
+      "loss": 1.1469,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -1.521965742111206,
+      "rewards/margins": 0.10184874385595322,
+      "rewards/rejected": -1.623814582824707,
+      "step": 490
+    },
+    {
+      "epoch": 0.08614748449345279,
+      "grad_norm": 8.208028943432536,
+      "learning_rate": 2.153316106804479e-08,
+      "logits/chosen": -3.548506259918213,
+      "logits/rejected": -3.5298848152160645,
+      "logps/chosen": -1.444461464881897,
+      "logps/rejected": -1.542236566543579,
+      "loss": 1.1439,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -1.444461464881897,
+      "rewards/margins": 0.0977751687169075,
+      "rewards/rejected": -1.542236566543579,
+      "step": 500
+    },
+    {
+      "epoch": 0.08614748449345279,
+      "eval_logits/chosen": -3.6475989818573,
+      "eval_logits/rejected": -3.643543004989624,
+      "eval_logps/chosen": -1.480989694595337,
+      "eval_logps/rejected": -1.6082936525344849,
+      "eval_loss": 1.1173921823501587,
+      "eval_rewards/accuracies": 0.5954925417900085,
+      "eval_rewards/chosen": -1.480989694595337,
+      "eval_rewards/margins": 0.12730395793914795,
+      "eval_rewards/rejected": -1.6082936525344849,
+      "eval_runtime": 155.4282,
+      "eval_samples_per_second": 27.691,
+      "eval_steps_per_second": 3.461,
+      "step": 500
+    },
+    {
+      "epoch": 0.08787043418332184,
+      "grad_norm": 8.6951697451053,
+      "learning_rate": 2.1963824289405682e-08,
+      "logits/chosen": -3.5439248085021973,
+      "logits/rejected": -3.5296573638916016,
+      "logps/chosen": -1.4840186834335327,
+      "logps/rejected": -1.5022141933441162,
+      "loss": 1.1946,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -1.4840186834335327,
+      "rewards/margins": 0.018195567652583122,
+      "rewards/rejected": -1.5022141933441162,
+      "step": 510
+    },
+    {
+      "epoch": 0.08959338387319091,
+      "grad_norm": 8.139298351143864,
+      "learning_rate": 2.239448751076658e-08,
+      "logits/chosen": -3.5641303062438965,
+      "logits/rejected": -3.548149824142456,
+      "logps/chosen": -1.5145920515060425,
+      "logps/rejected": -1.5433037281036377,
+      "loss": 1.1943,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": -1.5145920515060425,
+      "rewards/margins": 0.028711611405014992,
+      "rewards/rejected": -1.5433037281036377,
+      "step": 520
+    },
+    {
+      "epoch": 0.09131633356305996,
+      "grad_norm": 9.379603455271445,
+      "learning_rate": 2.2825150732127478e-08,
+      "logits/chosen": -3.5645813941955566,
+      "logits/rejected": -3.540370225906372,
+      "logps/chosen": -1.404667615890503,
+      "logps/rejected": -1.5424883365631104,
+      "loss": 1.1254,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.404667615890503,
+      "rewards/margins": 0.13782072067260742,
+      "rewards/rejected": -1.5424883365631104,
+      "step": 530
+    },
+    {
+      "epoch": 0.09303928325292901,
+      "grad_norm": 9.091453141760738,
+      "learning_rate": 2.3255813953488372e-08,
+      "logits/chosen": -3.5529913902282715,
+      "logits/rejected": -3.544365406036377,
+      "logps/chosen": -1.4848788976669312,
+      "logps/rejected": -1.6299892663955688,
+      "loss": 1.116,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.4848788976669312,
+      "rewards/margins": 0.1451103389263153,
+      "rewards/rejected": -1.6299892663955688,
+      "step": 540
+    },
+    {
+      "epoch": 0.09476223294279806,
+      "grad_norm": 8.836887915523528,
+      "learning_rate": 2.3686477174849267e-08,
+      "logits/chosen": -3.5901477336883545,
+      "logits/rejected": -3.5803847312927246,
+      "logps/chosen": -1.5022857189178467,
+      "logps/rejected": -1.5603463649749756,
+      "loss": 1.178,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -1.5022857189178467,
+      "rewards/margins": 0.05806078761816025,
+      "rewards/rejected": -1.5603463649749756,
+      "step": 550
+    },
+    {
+      "epoch": 0.09648518263266713,
+      "grad_norm": 8.862736068702574,
+      "learning_rate": 2.411714039621016e-08,
+      "logits/chosen": -3.576690196990967,
+      "logits/rejected": -3.574507474899292,
+      "logps/chosen": -1.480489730834961,
+      "logps/rejected": -1.6553888320922852,
+      "loss": 1.1071,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.480489730834961,
+      "rewards/margins": 0.17489919066429138,
+      "rewards/rejected": -1.6553888320922852,
+      "step": 560
+    },
+    {
+      "epoch": 0.09820813232253618,
+      "grad_norm": 9.52216355827329,
+      "learning_rate": 2.454780361757106e-08,
+      "logits/chosen": -3.5671496391296387,
+      "logits/rejected": -3.5618653297424316,
+      "logps/chosen": -1.4675098657608032,
+      "logps/rejected": -1.6020644903182983,
+      "loss": 1.1134,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.4675098657608032,
+      "rewards/margins": 0.13455474376678467,
+      "rewards/rejected": -1.6020644903182983,
+      "step": 570
+    },
+    {
+      "epoch": 0.09993108201240523,
+      "grad_norm": 8.478745178151112,
+      "learning_rate": 2.4978466838931954e-08,
+      "logits/chosen": -3.5610499382019043,
+      "logits/rejected": -3.5579872131347656,
+      "logps/chosen": -1.408745527267456,
+      "logps/rejected": -1.5310090780258179,
+      "loss": 1.1238,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -1.408745527267456,
+      "rewards/margins": 0.12226365506649017,
+      "rewards/rejected": -1.5310090780258179,
+      "step": 580
+    },
+    {
+      "epoch": 0.1016540317022743,
+      "grad_norm": 11.76336891380736,
+      "learning_rate": 2.540913006029285e-08,
+      "logits/chosen": -3.567946195602417,
+      "logits/rejected": -3.5542798042297363,
+      "logps/chosen": -1.478448748588562,
+      "logps/rejected": -1.5543912649154663,
+      "loss": 1.166,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -1.478448748588562,
+      "rewards/margins": 0.07594244182109833,
+      "rewards/rejected": -1.5543912649154663,
+      "step": 590
+    },
+    {
+      "epoch": 0.10337698139214335,
+      "grad_norm": 7.872217939686321,
+      "learning_rate": 2.5839793281653743e-08,
+      "logits/chosen": -3.547175645828247,
+      "logits/rejected": -3.533773899078369,
+      "logps/chosen": -1.3981530666351318,
+      "logps/rejected": -1.6326103210449219,
+      "loss": 1.0556,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.3981530666351318,
+      "rewards/margins": 0.23445717990398407,
+      "rewards/rejected": -1.6326103210449219,
+      "step": 600
+    },
+    {
+      "epoch": 0.10337698139214335,
+      "eval_logits/chosen": -3.6427876949310303,
+      "eval_logits/rejected": -3.6387081146240234,
+      "eval_logps/chosen": -1.4810994863510132,
+      "eval_logps/rejected": -1.6081925630569458,
+      "eval_loss": 1.1175459623336792,
+      "eval_rewards/accuracies": 0.5940985083580017,
+      "eval_rewards/chosen": -1.4810994863510132,
+      "eval_rewards/margins": 0.12709295749664307,
+      "eval_rewards/rejected": -1.6081925630569458,
+      "eval_runtime": 155.5789,
+      "eval_samples_per_second": 27.664,
+      "eval_steps_per_second": 3.458,
+      "step": 600
+    },
+    {
+      "epoch": 0.1050999310820124,
+      "grad_norm": 8.151538148849252,
+      "learning_rate": 2.6270456503014644e-08,
+      "logits/chosen": -3.535094738006592,
+      "logits/rejected": -3.532217025756836,
+      "logps/chosen": -1.455443263053894,
+      "logps/rejected": -1.5468182563781738,
+      "loss": 1.1494,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.455443263053894,
+      "rewards/margins": 0.0913749486207962,
+      "rewards/rejected": -1.5468182563781738,
+      "step": 610
+    },
+    {
+      "epoch": 0.10682288077188146,
+      "grad_norm": 9.406642718606557,
+      "learning_rate": 2.6701119724375536e-08,
+      "logits/chosen": -3.632193088531494,
+      "logits/rejected": -3.6169028282165527,
+      "logps/chosen": -1.5005719661712646,
+      "logps/rejected": -1.5948309898376465,
+      "loss": 1.1428,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.5005719661712646,
+      "rewards/margins": 0.09425897151231766,
+      "rewards/rejected": -1.5948309898376465,
+      "step": 620
+    },
+    {
+      "epoch": 0.10854583046175052,
+      "grad_norm": 8.959948310976952,
+      "learning_rate": 2.713178294573643e-08,
+      "logits/chosen": -3.6151442527770996,
+      "logits/rejected": -3.601228713989258,
+      "logps/chosen": -1.5043303966522217,
+      "logps/rejected": -1.5372164249420166,
+      "loss": 1.1816,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -1.5043303966522217,
+      "rewards/margins": 0.03288597986102104,
+      "rewards/rejected": -1.5372164249420166,
+      "step": 630
+    },
+    {
+      "epoch": 0.11026878015161957,
+      "grad_norm": 8.786883936458857,
+      "learning_rate": 2.756244616709733e-08,
+      "logits/chosen": -3.5960402488708496,
+      "logits/rejected": -3.5882256031036377,
+      "logps/chosen": -1.487899899482727,
+      "logps/rejected": -1.5676920413970947,
+      "loss": 1.1594,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -1.487899899482727,
+      "rewards/margins": 0.07979222387075424,
+      "rewards/rejected": -1.5676920413970947,
+      "step": 640
+    },
+    {
+      "epoch": 0.11199172984148863,
+      "grad_norm": 8.322842279311297,
+      "learning_rate": 2.7993109388458226e-08,
+      "logits/chosen": -3.6126532554626465,
+      "logits/rejected": -3.612745761871338,
+      "logps/chosen": -1.4698340892791748,
+      "logps/rejected": -1.6098400354385376,
+      "loss": 1.12,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.4698340892791748,
+      "rewards/margins": 0.14000579714775085,
+      "rewards/rejected": -1.6098400354385376,
+      "step": 650
+    },
+    {
+      "epoch": 0.11371467953135768,
+      "grad_norm": 8.778189097685207,
+      "learning_rate": 2.8423772609819118e-08,
+      "logits/chosen": -3.5375778675079346,
+      "logits/rejected": -3.53373384475708,
+      "logps/chosen": -1.4714914560317993,
+      "logps/rejected": -1.5646536350250244,
+      "loss": 1.1698,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -1.4714914560317993,
+      "rewards/margins": 0.09316208958625793,
+      "rewards/rejected": -1.5646536350250244,
+      "step": 660
+    },
+    {
+      "epoch": 0.11543762922122675,
+      "grad_norm": 8.556682727131038,
+      "learning_rate": 2.885443583118002e-08,
+      "logits/chosen": -3.551692247390747,
+      "logits/rejected": -3.5493876934051514,
+      "logps/chosen": -1.4510167837142944,
+      "logps/rejected": -1.5841106176376343,
+      "loss": 1.1328,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.4510167837142944,
+      "rewards/margins": 0.13309377431869507,
+      "rewards/rejected": -1.5841106176376343,
+      "step": 670
+    },
+    {
+      "epoch": 0.1171605789110958,
+      "grad_norm": 11.242559723080765,
+      "learning_rate": 2.9285099052540913e-08,
+      "logits/chosen": -3.5195937156677246,
+      "logits/rejected": -3.5070579051971436,
+      "logps/chosen": -1.5040366649627686,
+      "logps/rejected": -1.570325255393982,
+      "loss": 1.1665,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -1.5040366649627686,
+      "rewards/margins": 0.0662887841463089,
+      "rewards/rejected": -1.570325255393982,
+      "step": 680
+    },
+    {
+      "epoch": 0.11888352860096485,
+      "grad_norm": 9.144305818897282,
+      "learning_rate": 2.9715762273901808e-08,
+      "logits/chosen": -3.6142075061798096,
+      "logits/rejected": -3.5968334674835205,
+      "logps/chosen": -1.5187121629714966,
+      "logps/rejected": -1.5186699628829956,
+      "loss": 1.2116,
+      "rewards/accuracies": 0.4937500059604645,
+      "rewards/chosen": -1.5187121629714966,
+      "rewards/margins": -4.215352237224579e-05,
+      "rewards/rejected": -1.5186699628829956,
+      "step": 690
+    },
+    {
+      "epoch": 0.1206064782908339,
+      "grad_norm": 10.082237419827258,
+      "learning_rate": 3.01464254952627e-08,
+      "logits/chosen": -3.5868735313415527,
+      "logits/rejected": -3.5708167552948,
+      "logps/chosen": -1.4618594646453857,
+      "logps/rejected": -1.5504543781280518,
+      "loss": 1.1686,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -1.4618594646453857,
+      "rewards/margins": 0.08859489858150482,
+      "rewards/rejected": -1.5504543781280518,
+      "step": 700
+    },
+    {
+      "epoch": 0.1206064782908339,
+      "eval_logits/chosen": -3.653622627258301,
+      "eval_logits/rejected": -3.649606943130493,
+      "eval_logps/chosen": -1.4806456565856934,
+      "eval_logps/rejected": -1.6083095073699951,
+      "eval_loss": 1.117153286933899,
+      "eval_rewards/accuracies": 0.5945631861686707,
+      "eval_rewards/chosen": -1.4806456565856934,
+      "eval_rewards/margins": 0.1276639699935913,
+      "eval_rewards/rejected": -1.6083095073699951,
+      "eval_runtime": 155.4322,
+      "eval_samples_per_second": 27.691,
+      "eval_steps_per_second": 3.461,
+      "step": 700
+    },
+    {
+      "epoch": 0.12232942798070297,
+      "grad_norm": 8.27480148053149,
+      "learning_rate": 3.05770887166236e-08,
+      "logits/chosen": -3.5719287395477295,
+      "logits/rejected": -3.553983211517334,
+      "logps/chosen": -1.4324902296066284,
+      "logps/rejected": -1.6564991474151611,
+      "loss": 1.0614,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.4324902296066284,
+      "rewards/margins": 0.22400856018066406,
+      "rewards/rejected": -1.6564991474151611,
+      "step": 710
+    },
+    {
+      "epoch": 0.12405237767057202,
+      "grad_norm": 10.334273535259799,
+      "learning_rate": 3.100775193798449e-08,
+      "logits/chosen": -3.55265736579895,
+      "logits/rejected": -3.5503265857696533,
+      "logps/chosen": -1.5228334665298462,
+      "logps/rejected": -1.6074587106704712,
+      "loss": 1.1529,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.5228334665298462,
+      "rewards/margins": 0.08462528139352798,
+      "rewards/rejected": -1.6074587106704712,
+      "step": 720
+    },
+    {
+      "epoch": 0.12577532736044109,
+      "grad_norm": 9.306969000434556,
+      "learning_rate": 3.143841515934539e-08,
+      "logits/chosen": -3.6202404499053955,
+      "logits/rejected": -3.606217861175537,
+      "logps/chosen": -1.4912225008010864,
+      "logps/rejected": -1.5361526012420654,
+      "loss": 1.1806,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -1.4912225008010864,
+      "rewards/margins": 0.044930197298526764,
+      "rewards/rejected": -1.5361526012420654,
+      "step": 730
+    },
+    {
+      "epoch": 0.12749827705031014,
+      "grad_norm": 8.806022169521476,
+      "learning_rate": 3.186907838070629e-08,
+      "logits/chosen": -3.548826217651367,
+      "logits/rejected": -3.534964084625244,
+      "logps/chosen": -1.437606692314148,
+      "logps/rejected": -1.56428062915802,
+      "loss": 1.1303,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.437606692314148,
+      "rewards/margins": 0.12667372822761536,
+      "rewards/rejected": -1.56428062915802,
+      "step": 740
+    },
+    {
+      "epoch": 0.1292212267401792,
+      "grad_norm": 8.406170932299446,
+      "learning_rate": 3.229974160206718e-08,
+      "logits/chosen": -3.658874988555908,
+      "logits/rejected": -3.640561580657959,
+      "logps/chosen": -1.4259698390960693,
+      "logps/rejected": -1.55933678150177,
+      "loss": 1.112,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.4259698390960693,
+      "rewards/margins": 0.13336701691150665,
+      "rewards/rejected": -1.55933678150177,
+      "step": 750
+    },
+    {
+      "epoch": 0.13094417643004824,
+      "grad_norm": 9.114304594277051,
+      "learning_rate": 3.273040482342808e-08,
+      "logits/chosen": -3.558692216873169,
+      "logits/rejected": -3.53771710395813,
+      "logps/chosen": -1.4344779253005981,
+      "logps/rejected": -1.5714222192764282,
+      "loss": 1.1269,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -1.4344779253005981,
+      "rewards/margins": 0.1369442641735077,
+      "rewards/rejected": -1.5714222192764282,
+      "step": 760
+    },
+    {
+      "epoch": 0.1326671261199173,
+      "grad_norm": 8.69776410531619,
+      "learning_rate": 3.3161068044788975e-08,
+      "logits/chosen": -3.592759370803833,
+      "logits/rejected": -3.5785911083221436,
+      "logps/chosen": -1.4633291959762573,
+      "logps/rejected": -1.5984219312667847,
+      "loss": 1.1189,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.4633291959762573,
+      "rewards/margins": 0.13509267568588257,
+      "rewards/rejected": -1.5984219312667847,
+      "step": 770
+    },
+    {
+      "epoch": 0.13439007580978635,
+      "grad_norm": 9.336388549522557,
+      "learning_rate": 3.359173126614987e-08,
+      "logits/chosen": -3.592184543609619,
+      "logits/rejected": -3.5739738941192627,
+      "logps/chosen": -1.4392435550689697,
+      "logps/rejected": -1.580036997795105,
+      "loss": 1.1119,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.4392435550689697,
+      "rewards/margins": 0.14079339802265167,
+      "rewards/rejected": -1.580036997795105,
+      "step": 780
+    },
+    {
+      "epoch": 0.1361130254996554,
+      "grad_norm": 8.748321979397211,
+      "learning_rate": 3.4022394487510764e-08,
+      "logits/chosen": -3.598248243331909,
+      "logits/rejected": -3.580406904220581,
+      "logps/chosen": -1.4105949401855469,
+      "logps/rejected": -1.6200138330459595,
+      "loss": 1.0786,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.4105949401855469,
+      "rewards/margins": 0.2094188928604126,
+      "rewards/rejected": -1.6200138330459595,
+      "step": 790
+    },
+    {
+      "epoch": 0.13783597518952448,
+      "grad_norm": 10.307424589666256,
+      "learning_rate": 3.445305770887166e-08,
+      "logits/chosen": -3.5298805236816406,
+      "logits/rejected": -3.5147228240966797,
+      "logps/chosen": -1.4580105543136597,
+      "logps/rejected": -1.6410547494888306,
+      "loss": 1.1046,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.4580105543136597,
+      "rewards/margins": 0.1830442100763321,
+      "rewards/rejected": -1.6410547494888306,
+      "step": 800
+    },
+    {
+      "epoch": 0.13783597518952448,
+      "eval_logits/chosen": -3.6500885486602783,
+      "eval_logits/rejected": -3.6460535526275635,
+      "eval_logps/chosen": -1.4806969165802002,
+      "eval_logps/rejected": -1.6083589792251587,
+      "eval_loss": 1.1171507835388184,
+      "eval_rewards/accuracies": 0.5940985083580017,
+      "eval_rewards/chosen": -1.4806969165802002,
+      "eval_rewards/margins": 0.12766201794147491,
+      "eval_rewards/rejected": -1.6083589792251587,
+      "eval_runtime": 155.4182,
+      "eval_samples_per_second": 27.693,
+      "eval_steps_per_second": 3.462,
+      "step": 800
+    },
+    {
+      "epoch": 0.13955892487939353,
+      "grad_norm": 9.263075983283521,
+      "learning_rate": 3.488372093023256e-08,
+      "logits/chosen": -3.5799612998962402,
+      "logits/rejected": -3.563739776611328,
+      "logps/chosen": -1.4750475883483887,
+      "logps/rejected": -1.6324554681777954,
+      "loss": 1.1159,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.4750475883483887,
+      "rewards/margins": 0.15740787982940674,
+      "rewards/rejected": -1.6324554681777954,
+      "step": 810
+    },
+    {
+      "epoch": 0.14128187456926258,
+      "grad_norm": 8.225989699149078,
+      "learning_rate": 3.531438415159345e-08,
+      "logits/chosen": -3.6043028831481934,
+      "logits/rejected": -3.5885276794433594,
+      "logps/chosen": -1.4663821458816528,
+      "logps/rejected": -1.6286513805389404,
+      "loss": 1.1217,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -1.4663821458816528,
+      "rewards/margins": 0.1622694432735443,
+      "rewards/rejected": -1.6286513805389404,
+      "step": 820
+    },
+    {
+      "epoch": 0.14300482425913164,
+      "grad_norm": 9.325121504885157,
+      "learning_rate": 3.574504737295434e-08,
+      "logits/chosen": -3.553321361541748,
+      "logits/rejected": -3.5461878776550293,
+      "logps/chosen": -1.554410696029663,
+      "logps/rejected": -1.6077333688735962,
+      "loss": 1.1869,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -1.554410696029663,
+      "rewards/margins": 0.053322624415159225,
+      "rewards/rejected": -1.6077333688735962,
+      "step": 830
+    },
+    {
+      "epoch": 0.1447277739490007,
+      "grad_norm": 9.996154756470812,
+      "learning_rate": 3.617571059431525e-08,
+      "logits/chosen": -3.6170735359191895,
+      "logits/rejected": -3.60082745552063,
+      "logps/chosen": -1.4302030801773071,
+      "logps/rejected": -1.4935206174850464,
+      "loss": 1.1701,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -1.4302030801773071,
+      "rewards/margins": 0.06331750005483627,
+      "rewards/rejected": -1.4935206174850464,
+      "step": 840
+    },
+    {
+      "epoch": 0.14645072363886974,
+      "grad_norm": 11.132481895691344,
+      "learning_rate": 3.660637381567614e-08,
+      "logits/chosen": -3.540802478790283,
+      "logits/rejected": -3.5368094444274902,
+      "logps/chosen": -1.437247395515442,
+      "logps/rejected": -1.568040132522583,
+      "loss": 1.1224,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.437247395515442,
+      "rewards/margins": 0.13079282641410828,
+      "rewards/rejected": -1.568040132522583,
+      "step": 850
+    },
+    {
+      "epoch": 0.1481736733287388,
+      "grad_norm": 11.449330963885847,
+      "learning_rate": 3.7037037037037036e-08,
+      "logits/chosen": -3.5587494373321533,
+      "logits/rejected": -3.5470738410949707,
+      "logps/chosen": -1.5097761154174805,
+      "logps/rejected": -1.6295726299285889,
+      "loss": 1.1396,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.5097761154174805,
+      "rewards/margins": 0.11979658901691437,
+      "rewards/rejected": -1.6295726299285889,
+      "step": 860
+    },
+    {
+      "epoch": 0.14989662301860784,
+      "grad_norm": 8.650233388890967,
+      "learning_rate": 3.7467700258397934e-08,
+      "logits/chosen": -3.6047980785369873,
+      "logits/rejected": -3.6010043621063232,
+      "logps/chosen": -1.4577372074127197,
+      "logps/rejected": -1.5727463960647583,
+      "loss": 1.1433,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.4577372074127197,
+      "rewards/margins": 0.11500944942235947,
+      "rewards/rejected": -1.5727463960647583,
+      "step": 870
+    },
+    {
+      "epoch": 0.15161957270847692,
+      "grad_norm": 8.07227460957563,
+      "learning_rate": 3.7898363479758826e-08,
+      "logits/chosen": -3.5543975830078125,
+      "logits/rejected": -3.541515827178955,
+      "logps/chosen": -1.413339614868164,
+      "logps/rejected": -1.625514268875122,
+      "loss": 1.1005,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.413339614868164,
+      "rewards/margins": 0.2121744155883789,
+      "rewards/rejected": -1.625514268875122,
+      "step": 880
+    },
+    {
+      "epoch": 0.15334252239834598,
+      "grad_norm": 9.661968082471915,
+      "learning_rate": 3.8329026701119724e-08,
+      "logits/chosen": -3.5664615631103516,
+      "logits/rejected": -3.544980525970459,
+      "logps/chosen": -1.4622489213943481,
+      "logps/rejected": -1.6259300708770752,
+      "loss": 1.1081,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.4622489213943481,
+      "rewards/margins": 0.16368110477924347,
+      "rewards/rejected": -1.6259300708770752,
+      "step": 890
+    },
+    {
+      "epoch": 0.15506547208821503,
+      "grad_norm": 8.699611455953535,
+      "learning_rate": 3.875968992248062e-08,
+      "logits/chosen": -3.5807602405548096,
+      "logits/rejected": -3.5738091468811035,
+      "logps/chosen": -1.51506769657135,
+      "logps/rejected": -1.5654428005218506,
+      "loss": 1.1817,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.51506769657135,
+      "rewards/margins": 0.050375234335660934,
+      "rewards/rejected": -1.5654428005218506,
+      "step": 900
+    },
+    {
+      "epoch": 0.15506547208821503,
+      "eval_logits/chosen": -3.635263681411743,
+      "eval_logits/rejected": -3.6311538219451904,
+      "eval_logps/chosen": -1.480484127998352,
+      "eval_logps/rejected": -1.608588695526123,
+      "eval_loss": 1.116858720779419,
+      "eval_rewards/accuracies": 0.5950278639793396,
+      "eval_rewards/chosen": -1.480484127998352,
+      "eval_rewards/margins": 0.12810443341732025,
+      "eval_rewards/rejected": -1.608588695526123,
+      "eval_runtime": 155.4394,
+      "eval_samples_per_second": 27.689,
+      "eval_steps_per_second": 3.461,
+      "step": 900
+    },
+    {
+      "epoch": 0.15678842177808408,
+      "grad_norm": 7.803415086175332,
+      "learning_rate": 3.919035314384151e-08,
+      "logits/chosen": -3.571023464202881,
+      "logits/rejected": -3.5641732215881348,
+      "logps/chosen": -1.3982170820236206,
+      "logps/rejected": -1.5130685567855835,
+      "loss": 1.1401,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.3982170820236206,
+      "rewards/margins": 0.11485134065151215,
+      "rewards/rejected": -1.5130685567855835,
+      "step": 910
+    },
+    {
+      "epoch": 0.15851137146795313,
+      "grad_norm": 9.3387442545329,
+      "learning_rate": 3.962101636520241e-08,
+      "logits/chosen": -3.592866897583008,
+      "logits/rejected": -3.5678820610046387,
+      "logps/chosen": -1.463932752609253,
+      "logps/rejected": -1.5852473974227905,
+      "loss": 1.129,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.463932752609253,
+      "rewards/margins": 0.12131450325250626,
+      "rewards/rejected": -1.5852473974227905,
+      "step": 920
+    },
+    {
+      "epoch": 0.16023432115782218,
+      "grad_norm": 9.189711298512881,
+      "learning_rate": 4.005167958656331e-08,
+      "logits/chosen": -3.6345748901367188,
+      "logits/rejected": -3.624006748199463,
+      "logps/chosen": -1.4560471773147583,
+      "logps/rejected": -1.6305192708969116,
+      "loss": 1.099,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.4560471773147583,
+      "rewards/margins": 0.17447233200073242,
+      "rewards/rejected": -1.6305192708969116,
+      "step": 930
+    },
+    {
+      "epoch": 0.16195727084769124,
+      "grad_norm": 7.666097197579203,
+      "learning_rate": 4.04823428079242e-08,
+      "logits/chosen": -3.617074489593506,
+      "logits/rejected": -3.593635082244873,
+      "logps/chosen": -1.5049219131469727,
+      "logps/rejected": -1.6181217432022095,
+      "loss": 1.1403,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.5049219131469727,
+      "rewards/margins": 0.11319967359304428,
+      "rewards/rejected": -1.6181217432022095,
+      "step": 940
+    },
+    {
+      "epoch": 0.16368022053756032,
+      "grad_norm": 8.30720790045963,
+      "learning_rate": 4.09130060292851e-08,
+      "logits/chosen": -3.48144268989563,
+      "logits/rejected": -3.47333025932312,
+      "logps/chosen": -1.4661669731140137,
+      "logps/rejected": -1.6263736486434937,
+      "loss": 1.1095,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.4661669731140137,
+      "rewards/margins": 0.16020676493644714,
+      "rewards/rejected": -1.6263736486434937,
+      "step": 950
+    },
+    {
+      "epoch": 0.16540317022742937,
+      "grad_norm": 11.291793070122239,
+      "learning_rate": 4.134366925064599e-08,
+      "logits/chosen": -3.4642608165740967,
+      "logits/rejected": -3.466218948364258,
+      "logps/chosen": -1.4512989521026611,
+      "logps/rejected": -1.571076512336731,
+      "loss": 1.1385,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.4512989521026611,
+      "rewards/margins": 0.11977739632129669,
+      "rewards/rejected": -1.571076512336731,
+      "step": 960
+    },
+    {
+      "epoch": 0.16712611991729842,
+      "grad_norm": 9.889258025375133,
+      "learning_rate": 4.177433247200689e-08,
+      "logits/chosen": -3.573826551437378,
+      "logits/rejected": -3.5549404621124268,
+      "logps/chosen": -1.5537605285644531,
+      "logps/rejected": -1.5632851123809814,
+      "loss": 1.2065,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -1.5537605285644531,
+      "rewards/margins": 0.009524703025817871,
+      "rewards/rejected": -1.5632851123809814,
+      "step": 970
+    },
+    {
+      "epoch": 0.16884906960716747,
+      "grad_norm": 10.627609443069394,
+      "learning_rate": 4.2204995693367785e-08,
+      "logits/chosen": -3.6357028484344482,
+      "logits/rejected": -3.625994920730591,
+      "logps/chosen": -1.4830236434936523,
+      "logps/rejected": -1.5726128816604614,
+      "loss": 1.1494,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.4830236434936523,
+      "rewards/margins": 0.08958908170461655,
+      "rewards/rejected": -1.5726128816604614,
+      "step": 980
+    },
+    {
+      "epoch": 0.17057201929703653,
+      "grad_norm": 9.201830613796968,
+      "learning_rate": 4.2635658914728676e-08,
+      "logits/chosen": -3.55415415763855,
+      "logits/rejected": -3.5401320457458496,
+      "logps/chosen": -1.432652235031128,
+      "logps/rejected": -1.5611107349395752,
+      "loss": 1.1338,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -1.432652235031128,
+      "rewards/margins": 0.12845852971076965,
+      "rewards/rejected": -1.5611107349395752,
+      "step": 990
+    },
+    {
+      "epoch": 0.17229496898690558,
+      "grad_norm": 8.92774079146362,
+      "learning_rate": 4.306632213608958e-08,
+      "logits/chosen": -3.5305137634277344,
+      "logits/rejected": -3.5163979530334473,
+      "logps/chosen": -1.5089213848114014,
+      "logps/rejected": -1.5436208248138428,
+      "loss": 1.1844,
+      "rewards/accuracies": 0.4937500059604645,
+      "rewards/chosen": -1.5089213848114014,
+      "rewards/margins": 0.03469962626695633,
+      "rewards/rejected": -1.5436208248138428,
+      "step": 1000
+    },
+    {
+      "epoch": 0.17229496898690558,
+      "eval_logits/chosen": -3.6573781967163086,
+      "eval_logits/rejected": -3.653388500213623,
+      "eval_logps/chosen": -1.4805710315704346,
+      "eval_logps/rejected": -1.608561635017395,
+      "eval_loss": 1.116957187652588,
+      "eval_rewards/accuracies": 0.5966542959213257,
+      "eval_rewards/chosen": -1.4805710315704346,
+      "eval_rewards/margins": 0.12799039483070374,
+      "eval_rewards/rejected": -1.608561635017395,
+      "eval_runtime": 155.6367,
+      "eval_samples_per_second": 27.654,
+      "eval_steps_per_second": 3.457,
+      "step": 1000
+    },
+    {
+      "epoch": 0.17401791867677463,
+      "grad_norm": 9.816126196379532,
+      "learning_rate": 4.349698535745047e-08,
+      "logits/chosen": -3.484614133834839,
+      "logits/rejected": -3.491356372833252,
+      "logps/chosen": -1.536503553390503,
+      "logps/rejected": -1.6023414134979248,
+      "loss": 1.1708,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.536503553390503,
+      "rewards/margins": 0.06583789736032486,
+      "rewards/rejected": -1.6023414134979248,
+      "step": 1010
+    },
+    {
+      "epoch": 0.17574086836664368,
+      "grad_norm": 7.766407360809908,
+      "learning_rate": 4.3927648578811363e-08,
+      "logits/chosen": -3.604102611541748,
+      "logits/rejected": -3.58256196975708,
+      "logps/chosen": -1.4883391857147217,
+      "logps/rejected": -1.6217081546783447,
+      "loss": 1.1227,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.4883391857147217,
+      "rewards/margins": 0.1333690732717514,
+      "rewards/rejected": -1.6217081546783447,
+      "step": 1020
+    },
+    {
+      "epoch": 0.17746381805651276,
+      "grad_norm": 9.346732463115233,
+      "learning_rate": 4.435831180017227e-08,
+      "logits/chosen": -3.626645565032959,
+      "logits/rejected": -3.6112418174743652,
+      "logps/chosen": -1.4350141286849976,
+      "logps/rejected": -1.5454413890838623,
+      "loss": 1.1401,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -1.4350141286849976,
+      "rewards/margins": 0.11042718589305878,
+      "rewards/rejected": -1.5454413890838623,
+      "step": 1030
+    },
+    {
+      "epoch": 0.17918676774638181,
+      "grad_norm": 12.839990005519343,
+      "learning_rate": 4.478897502153316e-08,
+      "logits/chosen": -3.5756173133850098,
+      "logits/rejected": -3.5693302154541016,
+      "logps/chosen": -1.531140685081482,
+      "logps/rejected": -1.6391258239746094,
+      "loss": 1.1409,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.531140685081482,
+      "rewards/margins": 0.1079854741692543,
+      "rewards/rejected": -1.6391258239746094,
+      "step": 1040
+    },
+    {
+      "epoch": 0.18090971743625087,
+      "grad_norm": 9.987334387358754,
+      "learning_rate": 4.521963824289405e-08,
+      "logits/chosen": -3.5315067768096924,
+      "logits/rejected": -3.5118134021759033,
+      "logps/chosen": -1.500441074371338,
+      "logps/rejected": -1.5241626501083374,
+      "loss": 1.1954,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.500441074371338,
+      "rewards/margins": 0.023721490055322647,
+      "rewards/rejected": -1.5241626501083374,
+      "step": 1050
+    },
+    {
+      "epoch": 0.18263266712611992,
+      "grad_norm": 11.13858613421704,
+      "learning_rate": 4.5650301464254955e-08,
+      "logits/chosen": -3.535428285598755,
+      "logits/rejected": -3.525745391845703,
+      "logps/chosen": -1.5127344131469727,
+      "logps/rejected": -1.5752580165863037,
+      "loss": 1.1613,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -1.5127344131469727,
+      "rewards/margins": 0.06252352148294449,
+      "rewards/rejected": -1.5752580165863037,
+      "step": 1060
+    },
+    {
+      "epoch": 0.18435561681598897,
+      "grad_norm": 9.053159669109746,
+      "learning_rate": 4.6080964685615846e-08,
+      "logits/chosen": -3.625441312789917,
+      "logits/rejected": -3.6078553199768066,
+      "logps/chosen": -1.5064266920089722,
+      "logps/rejected": -1.6778600215911865,
+      "loss": 1.1256,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -1.5064266920089722,
+      "rewards/margins": 0.17143329977989197,
+      "rewards/rejected": -1.6778600215911865,
+      "step": 1070
+    },
+    {
+      "epoch": 0.18607856650585802,
+      "grad_norm": 8.348457477317858,
+      "learning_rate": 4.6511627906976744e-08,
+      "logits/chosen": -3.6062190532684326,
+      "logits/rejected": -3.5932726860046387,
+      "logps/chosen": -1.501305341720581,
+      "logps/rejected": -1.5489680767059326,
+      "loss": 1.1889,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -1.501305341720581,
+      "rewards/margins": 0.04766266793012619,
+      "rewards/rejected": -1.5489680767059326,
+      "step": 1080
+    },
+    {
+      "epoch": 0.18780151619572708,
+      "grad_norm": 9.325192320243692,
+      "learning_rate": 4.6942291128337636e-08,
+      "logits/chosen": -3.5297489166259766,
+      "logits/rejected": -3.523674726486206,
+      "logps/chosen": -1.4604606628417969,
+      "logps/rejected": -1.584928274154663,
+      "loss": 1.1336,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -1.4604606628417969,
+      "rewards/margins": 0.12446783483028412,
+      "rewards/rejected": -1.584928274154663,
+      "step": 1090
+    },
+    {
+      "epoch": 0.18952446588559613,
+      "grad_norm": 9.308469791409037,
+      "learning_rate": 4.7372954349698534e-08,
+      "logits/chosen": -3.572493314743042,
+      "logits/rejected": -3.5693702697753906,
+      "logps/chosen": -1.4663598537445068,
+      "logps/rejected": -1.6092230081558228,
+      "loss": 1.1161,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.4663598537445068,
+      "rewards/margins": 0.14286303520202637,
+      "rewards/rejected": -1.6092230081558228,
+      "step": 1100
+    },
+    {
+      "epoch": 0.18952446588559613,
+      "eval_logits/chosen": -3.6369152069091797,
+      "eval_logits/rejected": -3.632840633392334,
+      "eval_logps/chosen": -1.4804579019546509,
+      "eval_logps/rejected": -1.6086174249649048,
+      "eval_loss": 1.1168462038040161,
+      "eval_rewards/accuracies": 0.5961896181106567,
+      "eval_rewards/chosen": -1.4804579019546509,
+      "eval_rewards/margins": 0.1281598061323166,
+      "eval_rewards/rejected": -1.6086174249649048,
+      "eval_runtime": 155.4619,
+      "eval_samples_per_second": 27.685,
+      "eval_steps_per_second": 3.461,
+      "step": 1100
+    },
+    {
+      "epoch": 0.1912474155754652,
+      "grad_norm": 11.574797044580277,
+      "learning_rate": 4.780361757105943e-08,
+      "logits/chosen": -3.5725035667419434,
+      "logits/rejected": -3.5802314281463623,
+      "logps/chosen": -1.5203903913497925,
+      "logps/rejected": -1.6037156581878662,
+      "loss": 1.1609,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.5203903913497925,
+      "rewards/margins": 0.0833253264427185,
+      "rewards/rejected": -1.6037156581878662,
+      "step": 1110
+    },
+    {
+      "epoch": 0.19297036526533426,
+      "grad_norm": 8.759868774687968,
+      "learning_rate": 4.823428079242032e-08,
+      "logits/chosen": -3.595228672027588,
+      "logits/rejected": -3.582097291946411,
+      "logps/chosen": -1.4783647060394287,
+      "logps/rejected": -1.614039659500122,
+      "loss": 1.1303,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.4783647060394287,
+      "rewards/margins": 0.1356751173734665,
+      "rewards/rejected": -1.614039659500122,
+      "step": 1120
+    },
+    {
+      "epoch": 0.1946933149552033,
+      "grad_norm": 10.976461387116005,
+      "learning_rate": 4.866494401378122e-08,
+      "logits/chosen": -3.633756637573242,
+      "logits/rejected": -3.6182034015655518,
+      "logps/chosen": -1.4554688930511475,
+      "logps/rejected": -1.6649646759033203,
+      "loss": 1.0754,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.4554688930511475,
+      "rewards/margins": 0.20949570834636688,
+      "rewards/rejected": -1.6649646759033203,
+      "step": 1130
+    },
+    {
+      "epoch": 0.19641626464507236,
+      "grad_norm": 8.662587529817367,
+      "learning_rate": 4.909560723514212e-08,
+      "logits/chosen": -3.57020902633667,
+      "logits/rejected": -3.548369884490967,
+      "logps/chosen": -1.509648084640503,
+      "logps/rejected": -1.643873929977417,
+      "loss": 1.1348,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.509648084640503,
+      "rewards/margins": 0.1342259794473648,
+      "rewards/rejected": -1.643873929977417,
+      "step": 1140
+    },
+    {
+      "epoch": 0.19813921433494142,
+      "grad_norm": 9.351215015456358,
+      "learning_rate": 4.952627045650301e-08,
+      "logits/chosen": -3.5312302112579346,
+      "logits/rejected": -3.518414258956909,
+      "logps/chosen": -1.45508873462677,
+      "logps/rejected": -1.6163194179534912,
+      "loss": 1.1051,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.45508873462677,
+      "rewards/margins": 0.16123102605342865,
+      "rewards/rejected": -1.6163194179534912,
+      "step": 1150
+    },
+    {
+      "epoch": 0.19986216402481047,
+      "grad_norm": 9.08881804247513,
+      "learning_rate": 4.995693367786391e-08,
+      "logits/chosen": -3.598334550857544,
+      "logits/rejected": -3.593904972076416,
+      "logps/chosen": -1.4725420475006104,
+      "logps/rejected": -1.535893201828003,
+      "loss": 1.1711,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.4725420475006104,
+      "rewards/margins": 0.06335121393203735,
+      "rewards/rejected": -1.535893201828003,
+      "step": 1160
+    },
+    {
+      "epoch": 0.20158511371467952,
+      "grad_norm": 8.625493872228944,
+      "learning_rate": 4.9999908438832287e-08,
+      "logits/chosen": -3.5094153881073,
+      "logits/rejected": -3.5007617473602295,
+      "logps/chosen": -1.4695771932601929,
+      "logps/rejected": -1.660736083984375,
+      "loss": 1.0915,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.4695771932601929,
+      "rewards/margins": 0.1911587417125702,
+      "rewards/rejected": -1.660736083984375,
+      "step": 1170
+    },
+    {
+      "epoch": 0.2033080634045486,
+      "grad_norm": 9.750028401776353,
+      "learning_rate": 4.999959193195308e-08,
+      "logits/chosen": -3.490715503692627,
+      "logits/rejected": -3.473031520843506,
+      "logps/chosen": -1.4576960802078247,
+      "logps/rejected": -1.6318676471710205,
+      "loss": 1.1006,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.4576960802078247,
+      "rewards/margins": 0.1741715371608734,
+      "rewards/rejected": -1.6318676471710205,
+      "step": 1180
+    },
+    {
+      "epoch": 0.20503101309441765,
+      "grad_norm": 9.815610622724664,
+      "learning_rate": 4.9999049351839105e-08,
+      "logits/chosen": -3.6181583404541016,
+      "logits/rejected": -3.597895860671997,
+      "logps/chosen": -1.5018386840820312,
+      "logps/rejected": -1.5320472717285156,
+      "loss": 1.2007,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -1.5018386840820312,
+      "rewards/margins": 0.0302087664604187,
+      "rewards/rejected": -1.5320472717285156,
+      "step": 1190
+    },
+    {
+      "epoch": 0.2067539627842867,
+      "grad_norm": 9.045364021800864,
+      "learning_rate": 4.9998280703396977e-08,
+      "logits/chosen": -3.5234375,
+      "logits/rejected": -3.5134167671203613,
+      "logps/chosen": -1.4296133518218994,
+      "logps/rejected": -1.5477523803710938,
+      "loss": 1.1305,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.4296133518218994,
+      "rewards/margins": 0.11813901364803314,
+      "rewards/rejected": -1.5477523803710938,
+      "step": 1200
+    },
+    {
+      "epoch": 0.2067539627842867,
+      "eval_logits/chosen": -3.630875825881958,
+      "eval_logits/rejected": -3.6267852783203125,
+      "eval_logps/chosen": -1.4801783561706543,
+      "eval_logps/rejected": -1.608912706375122,
+      "eval_loss": 1.116463303565979,
+      "eval_rewards/accuracies": 0.595724880695343,
+      "eval_rewards/chosen": -1.4801783561706543,
+      "eval_rewards/margins": 0.12873412668704987,
+      "eval_rewards/rejected": -1.608912706375122,
+      "eval_runtime": 155.4281,
+      "eval_samples_per_second": 27.691,
+      "eval_steps_per_second": 3.461,
+      "step": 1200
+    },
+    {
+      "epoch": 0.20847691247415576,
+      "grad_norm": 9.99982536137658,
+      "learning_rate": 4.9997285993577624e-08,
+      "logits/chosen": -3.5676262378692627,
+      "logits/rejected": -3.5466322898864746,
+      "logps/chosen": -1.4554429054260254,
+      "logps/rejected": -1.6698334217071533,
+      "loss": 1.0785,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.4554429054260254,
+      "rewards/margins": 0.2143905907869339,
+      "rewards/rejected": -1.6698334217071533,
+      "step": 1210
+    },
+    {
+      "epoch": 0.2101998621640248,
+      "grad_norm": 8.439838407545702,
+      "learning_rate": 4.999606523137628e-08,
+      "logits/chosen": -3.5851426124572754,
+      "logits/rejected": -3.5696380138397217,
+      "logps/chosen": -1.4669678211212158,
+      "logps/rejected": -1.5776492357254028,
+      "loss": 1.1408,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.4669678211212158,
+      "rewards/margins": 0.11068136990070343,
+      "rewards/rejected": -1.5776492357254028,
+      "step": 1220
+    },
+    {
+      "epoch": 0.21192281185389386,
+      "grad_norm": 10.391221075618285,
+      "learning_rate": 4.99946184278324e-08,
+      "logits/chosen": -3.6194560527801514,
+      "logits/rejected": -3.5941288471221924,
+      "logps/chosen": -1.4786808490753174,
+      "logps/rejected": -1.6246073246002197,
+      "loss": 1.1153,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.4786808490753174,
+      "rewards/margins": 0.14592646062374115,
+      "rewards/rejected": -1.6246073246002197,
+      "step": 1230
+    },
+    {
+      "epoch": 0.2136457615437629,
+      "grad_norm": 8.93149196891545,
+      "learning_rate": 4.9992945596029545e-08,
+      "logits/chosen": -3.541337490081787,
+      "logits/rejected": -3.5301690101623535,
+      "logps/chosen": -1.4520667791366577,
+      "logps/rejected": -1.5694843530654907,
+      "loss": 1.1375,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.4520667791366577,
+      "rewards/margins": 0.11741757392883301,
+      "rewards/rejected": -1.5694843530654907,
+      "step": 1240
+    },
+    {
+      "epoch": 0.21536871123363197,
+      "grad_norm": 10.864272147476829,
+      "learning_rate": 4.999104675109525e-08,
+      "logits/chosen": -3.6182236671447754,
+      "logits/rejected": -3.5989608764648438,
+      "logps/chosen": -1.4510705471038818,
+      "logps/rejected": -1.5661760568618774,
+      "loss": 1.138,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -1.4510705471038818,
+      "rewards/margins": 0.11510531604290009,
+      "rewards/rejected": -1.5661760568618774,
+      "step": 1250
+    },
+    {
+      "epoch": 0.21709166092350105,
+      "grad_norm": 9.785504873165793,
+      "learning_rate": 4.998892191020092e-08,
+      "logits/chosen": -3.5264439582824707,
+      "logits/rejected": -3.5058510303497314,
+      "logps/chosen": -1.4345242977142334,
+      "logps/rejected": -1.6021684408187866,
+      "loss": 1.11,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.4345242977142334,
+      "rewards/margins": 0.1676442176103592,
+      "rewards/rejected": -1.6021684408187866,
+      "step": 1260
+    },
+    {
+      "epoch": 0.2188146106133701,
+      "grad_norm": 9.898014952364466,
+      "learning_rate": 4.9986571092561664e-08,
+      "logits/chosen": -3.567234516143799,
+      "logits/rejected": -3.562983274459839,
+      "logps/chosen": -1.4404027462005615,
+      "logps/rejected": -1.6028454303741455,
+      "loss": 1.1184,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -1.4404027462005615,
+      "rewards/margins": 0.1624426543712616,
+      "rewards/rejected": -1.6028454303741455,
+      "step": 1270
+    },
+    {
+      "epoch": 0.22053756030323915,
+      "grad_norm": 8.609133546818134,
+      "learning_rate": 4.9983994319436093e-08,
+      "logits/chosen": -3.6200618743896484,
+      "logits/rejected": -3.6227810382843018,
+      "logps/chosen": -1.4395456314086914,
+      "logps/rejected": -1.6711490154266357,
+      "loss": 1.0774,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.4395456314086914,
+      "rewards/margins": 0.2316035032272339,
+      "rewards/rejected": -1.6711490154266357,
+      "step": 1280
+    },
+    {
+      "epoch": 0.2222605099931082,
+      "grad_norm": 8.396907249342807,
+      "learning_rate": 4.998119161412618e-08,
+      "logits/chosen": -3.5514895915985107,
+      "logits/rejected": -3.532341480255127,
+      "logps/chosen": -1.4516041278839111,
+      "logps/rejected": -1.5142688751220703,
+      "loss": 1.1625,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.4516041278839111,
+      "rewards/margins": 0.06266452372074127,
+      "rewards/rejected": -1.5142688751220703,
+      "step": 1290
+    },
+    {
+      "epoch": 0.22398345968297725,
+      "grad_norm": 8.752067706029534,
+      "learning_rate": 4.997816300197699e-08,
+      "logits/chosen": -3.595925807952881,
+      "logits/rejected": -3.5876095294952393,
+      "logps/chosen": -1.4629942178726196,
+      "logps/rejected": -1.6329936981201172,
+      "loss": 1.0977,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.4629942178726196,
+      "rewards/margins": 0.16999953985214233,
+      "rewards/rejected": -1.6329936981201172,
+      "step": 1300
+    },
+    {
+      "epoch": 0.22398345968297725,
+      "eval_logits/chosen": -3.6355748176574707,
+      "eval_logits/rejected": -3.6315109729766846,
+      "eval_logps/chosen": -1.480148196220398,
+      "eval_logps/rejected": -1.6092027425765991,
+      "eval_loss": 1.1163121461868286,
+      "eval_rewards/accuracies": 0.5966542959213257,
+      "eval_rewards/chosen": -1.480148196220398,
+      "eval_rewards/margins": 0.12905465066432953,
+      "eval_rewards/rejected": -1.6092027425765991,
+      "eval_runtime": 155.4995,
+      "eval_samples_per_second": 27.679,
+      "eval_steps_per_second": 3.46,
+      "step": 1300
+    },
+    {
+      "epoch": 0.2257064093728463,
+      "grad_norm": 9.71554271151232,
+      "learning_rate": 4.99749085103765e-08,
+      "logits/chosen": -3.5950355529785156,
+      "logits/rejected": -3.5750935077667236,
+      "logps/chosen": -1.4615724086761475,
+      "logps/rejected": -1.5291401147842407,
+      "loss": 1.1628,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.4615724086761475,
+      "rewards/margins": 0.06756796687841415,
+      "rewards/rejected": -1.5291401147842407,
+      "step": 1310
+    },
+    {
+      "epoch": 0.22742935906271536,
+      "grad_norm": 7.564603449494134,
+      "learning_rate": 4.9971428168755336e-08,
+      "logits/chosen": -3.572880983352661,
+      "logits/rejected": -3.5578951835632324,
+      "logps/chosen": -1.456709861755371,
+      "logps/rejected": -1.5381460189819336,
+      "loss": 1.1543,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.456709861755371,
+      "rewards/margins": 0.081435926258564,
+      "rewards/rejected": -1.5381460189819336,
+      "step": 1320
+    },
+    {
+      "epoch": 0.22915230875258444,
+      "grad_norm": 10.037028947254694,
+      "learning_rate": 4.9967722008586484e-08,
+      "logits/chosen": -3.622544527053833,
+      "logits/rejected": -3.6054511070251465,
+      "logps/chosen": -1.4836233854293823,
+      "logps/rejected": -1.578221321105957,
+      "loss": 1.1758,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.4836233854293823,
+      "rewards/margins": 0.0945979505777359,
+      "rewards/rejected": -1.578221321105957,
+      "step": 1330
+    },
+    {
+      "epoch": 0.2308752584424535,
+      "grad_norm": 9.178526620075157,
+      "learning_rate": 4.996379006338504e-08,
+      "logits/chosen": -3.533280849456787,
+      "logits/rejected": -3.520592212677002,
+      "logps/chosen": -1.4766193628311157,
+      "logps/rejected": -1.553138017654419,
+      "loss": 1.1574,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.4766193628311157,
+      "rewards/margins": 0.07651875168085098,
+      "rewards/rejected": -1.553138017654419,
+      "step": 1340
+    },
+    {
+      "epoch": 0.23259820813232254,
+      "grad_norm": 8.297229084302739,
+      "learning_rate": 4.995963236870789e-08,
+      "logits/chosen": -3.5702507495880127,
+      "logits/rejected": -3.5570387840270996,
+      "logps/chosen": -1.4747014045715332,
+      "logps/rejected": -1.5849618911743164,
+      "loss": 1.143,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.4747014045715332,
+      "rewards/margins": 0.11026038229465485,
+      "rewards/rejected": -1.5849618911743164,
+      "step": 1350
+    },
+    {
+      "epoch": 0.2343211578221916,
+      "grad_norm": 11.371842204693957,
+      "learning_rate": 4.995524896215339e-08,
+      "logits/chosen": -3.5446228981018066,
+      "logits/rejected": -3.5382752418518066,
+      "logps/chosen": -1.5246260166168213,
+      "logps/rejected": -1.643259048461914,
+      "loss": 1.132,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -1.5246260166168213,
+      "rewards/margins": 0.11863304674625397,
+      "rewards/rejected": -1.643259048461914,
+      "step": 1360
+    },
+    {
+      "epoch": 0.23604410751206065,
+      "grad_norm": 8.985460615542202,
+      "learning_rate": 4.9950639883361015e-08,
+      "logits/chosen": -3.614001750946045,
+      "logits/rejected": -3.6019866466522217,
+      "logps/chosen": -1.4788681268692017,
+      "logps/rejected": -1.6330759525299072,
+      "loss": 1.1088,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.4788681268692017,
+      "rewards/margins": 0.15420787036418915,
+      "rewards/rejected": -1.6330759525299072,
+      "step": 1370
+    },
+    {
+      "epoch": 0.2377670572019297,
+      "grad_norm": 8.49451275953699,
+      "learning_rate": 4.9945805174011024e-08,
+      "logits/chosen": -3.537050724029541,
+      "logits/rejected": -3.5240225791931152,
+      "logps/chosen": -1.4441369771957397,
+      "logps/rejected": -1.5583983659744263,
+      "loss": 1.1332,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -1.4441369771957397,
+      "rewards/margins": 0.11426124721765518,
+      "rewards/rejected": -1.5583983659744263,
+      "step": 1380
+    },
+    {
+      "epoch": 0.23949000689179875,
+      "grad_norm": 8.410500504338398,
+      "learning_rate": 4.994074487782406e-08,
+      "logits/chosen": -3.638920307159424,
+      "logits/rejected": -3.623122453689575,
+      "logps/chosen": -1.5021255016326904,
+      "logps/rejected": -1.59280526638031,
+      "loss": 1.1479,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.5021255016326904,
+      "rewards/margins": 0.09067968279123306,
+      "rewards/rejected": -1.59280526638031,
+      "step": 1390
+    },
+    {
+      "epoch": 0.2412129565816678,
+      "grad_norm": 9.079149565877778,
+      "learning_rate": 4.9935459040560776e-08,
+      "logits/chosen": -3.5273163318634033,
+      "logits/rejected": -3.5112547874450684,
+      "logps/chosen": -1.5132635831832886,
+      "logps/rejected": -1.6382343769073486,
+      "loss": 1.1432,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -1.5132635831832886,
+      "rewards/margins": 0.12497074902057648,
+      "rewards/rejected": -1.6382343769073486,
+      "step": 1400
+    },
+    {
+      "epoch": 0.2412129565816678,
+      "eval_logits/chosen": -3.6429061889648438,
+      "eval_logits/rejected": -3.6388914585113525,
+      "eval_logps/chosen": -1.480460524559021,
+      "eval_logps/rejected": -1.6099385023117065,
+      "eval_loss": 1.1160637140274048,
+      "eval_rewards/accuracies": 0.5959572196006775,
+      "eval_rewards/chosen": -1.480460524559021,
+      "eval_rewards/margins": 0.12947790324687958,
+      "eval_rewards/rejected": -1.6099385023117065,
+      "eval_runtime": 155.3996,
+      "eval_samples_per_second": 27.696,
+      "eval_steps_per_second": 3.462,
+      "step": 1400
+    },
+    {
+      "epoch": 0.24293590627153688,
+      "grad_norm": 8.607818594055319,
+      "learning_rate": 4.9929947710021415e-08,
+      "logits/chosen": -3.5663979053497314,
+      "logits/rejected": -3.556598663330078,
+      "logps/chosen": -1.5133002996444702,
+      "logps/rejected": -1.5943195819854736,
+      "loss": 1.1531,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -1.5133002996444702,
+      "rewards/margins": 0.08101936429738998,
+      "rewards/rejected": -1.5943195819854736,
+      "step": 1410
+    },
+    {
+      "epoch": 0.24465885596140594,
+      "grad_norm": 7.748122477253017,
+      "learning_rate": 4.992421093604534e-08,
+      "logits/chosen": -3.5142712593078613,
+      "logits/rejected": -3.5164332389831543,
+      "logps/chosen": -1.4515101909637451,
+      "logps/rejected": -1.5454955101013184,
+      "loss": 1.1531,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.4515101909637451,
+      "rewards/margins": 0.09398544579744339,
+      "rewards/rejected": -1.5454955101013184,
+      "step": 1420
+    },
+    {
+      "epoch": 0.246381805651275,
+      "grad_norm": 9.583071559391538,
+      "learning_rate": 4.9918248770510664e-08,
+      "logits/chosen": -3.59727144241333,
+      "logits/rejected": -3.5919559001922607,
+      "logps/chosen": -1.522887110710144,
+      "logps/rejected": -1.6652920246124268,
+      "loss": 1.1238,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.522887110710144,
+      "rewards/margins": 0.14240476489067078,
+      "rewards/rejected": -1.6652920246124268,
+      "step": 1430
+    },
+    {
+      "epoch": 0.24810475534114404,
+      "grad_norm": 9.544488729612658,
+      "learning_rate": 4.9912061267333696e-08,
+      "logits/chosen": -3.5452492237091064,
+      "logits/rejected": -3.526981830596924,
+      "logps/chosen": -1.4126319885253906,
+      "logps/rejected": -1.549820065498352,
+      "loss": 1.1178,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.4126319885253906,
+      "rewards/margins": 0.137188121676445,
+      "rewards/rejected": -1.549820065498352,
+      "step": 1440
+    },
+    {
+      "epoch": 0.2498277050310131,
+      "grad_norm": 10.304017147286483,
+      "learning_rate": 4.99056484824685e-08,
+      "logits/chosen": -3.559828519821167,
+      "logits/rejected": -3.5391852855682373,
+      "logps/chosen": -1.4362605810165405,
+      "logps/rejected": -1.6020727157592773,
+      "loss": 1.1013,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.4362605810165405,
+      "rewards/margins": 0.16581206023693085,
+      "rewards/rejected": -1.6020727157592773,
+      "step": 1450
+    },
+    {
+      "epoch": 0.25155065472088217,
+      "grad_norm": 11.370059760445338,
+      "learning_rate": 4.98990104739064e-08,
+      "logits/chosen": -3.548842191696167,
+      "logits/rejected": -3.53136944770813,
+      "logps/chosen": -1.447721242904663,
+      "logps/rejected": -1.6532838344573975,
+      "loss": 1.0915,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.447721242904663,
+      "rewards/margins": 0.2055627405643463,
+      "rewards/rejected": -1.6532838344573975,
+      "step": 1460
+    },
+    {
+      "epoch": 0.2532736044107512,
+      "grad_norm": 10.341614111814025,
+      "learning_rate": 4.98921473016754e-08,
+      "logits/chosen": -3.6516966819763184,
+      "logits/rejected": -3.632154941558838,
+      "logps/chosen": -1.5445959568023682,
+      "logps/rejected": -1.7492244243621826,
+      "loss": 1.1158,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -1.5445959568023682,
+      "rewards/margins": 0.20462842285633087,
+      "rewards/rejected": -1.7492244243621826,
+      "step": 1470
+    },
+    {
+      "epoch": 0.2549965541006203,
+      "grad_norm": 8.442577296088945,
+      "learning_rate": 4.9885059027839705e-08,
+      "logits/chosen": -3.6376259326934814,
+      "logits/rejected": -3.6164984703063965,
+      "logps/chosen": -1.394921064376831,
+      "logps/rejected": -1.5803489685058594,
+      "loss": 1.0818,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.394921064376831,
+      "rewards/margins": 0.18542788922786713,
+      "rewards/rejected": -1.5803489685058594,
+      "step": 1480
+    },
+    {
+      "epoch": 0.2567195037904893,
+      "grad_norm": 9.531468517626623,
+      "learning_rate": 4.987774571649912e-08,
+      "logits/chosen": -3.586893081665039,
+      "logits/rejected": -3.572434902191162,
+      "logps/chosen": -1.5460751056671143,
+      "logps/rejected": -1.616434097290039,
+      "loss": 1.162,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.5460751056671143,
+      "rewards/margins": 0.07035890966653824,
+      "rewards/rejected": -1.616434097290039,
+      "step": 1490
+    },
+    {
+      "epoch": 0.2584424534803584,
+      "grad_norm": 7.620366137037931,
+      "learning_rate": 4.987020743378848e-08,
+      "logits/chosen": -3.547447919845581,
+      "logits/rejected": -3.546436309814453,
+      "logps/chosen": -1.43503737449646,
+      "logps/rejected": -1.5390199422836304,
+      "loss": 1.1427,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.43503737449646,
+      "rewards/margins": 0.10398247092962265,
+      "rewards/rejected": -1.5390199422836304,
+      "step": 1500
+    },
+    {
+      "epoch": 0.2584424534803584,
+      "eval_logits/chosen": -3.6263914108276367,
+      "eval_logits/rejected": -3.6223111152648926,
+      "eval_logps/chosen": -1.4808921813964844,
+      "eval_logps/rejected": -1.6105868816375732,
+      "eval_loss": 1.1159926652908325,
+      "eval_rewards/accuracies": 0.5954925417900085,
+      "eval_rewards/chosen": -1.4808921813964844,
+      "eval_rewards/margins": 0.12969458103179932,
+      "eval_rewards/rejected": -1.6105868816375732,
+      "eval_runtime": 155.6443,
+      "eval_samples_per_second": 27.653,
+      "eval_steps_per_second": 3.457,
+      "step": 1500
+    },
+    {
+      "epoch": 0.2601654031702274,
+      "grad_norm": 9.100950251723738,
+      "learning_rate": 4.9862444247877054e-08,
+      "logits/chosen": -3.4849891662597656,
+      "logits/rejected": -3.464808702468872,
+      "logps/chosen": -1.5767319202423096,
+      "logps/rejected": -1.6484384536743164,
+      "loss": 1.1739,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -1.5767319202423096,
+      "rewards/margins": 0.07170670479536057,
+      "rewards/rejected": -1.6484384536743164,
+      "step": 1510
+    },
+    {
+      "epoch": 0.2618883528600965,
+      "grad_norm": 9.401744068257223,
+      "learning_rate": 4.985445622896794e-08,
+      "logits/chosen": -3.564509868621826,
+      "logits/rejected": -3.5598950386047363,
+      "logps/chosen": -1.5152881145477295,
+      "logps/rejected": -1.5692977905273438,
+      "loss": 1.1781,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -1.5152881145477295,
+      "rewards/margins": 0.054009683430194855,
+      "rewards/rejected": -1.5692977905273438,
+      "step": 1520
+    },
+    {
+      "epoch": 0.26361130254996556,
+      "grad_norm": 10.323003220568687,
+      "learning_rate": 4.98462434492974e-08,
+      "logits/chosen": -3.5007259845733643,
+      "logits/rejected": -3.489821195602417,
+      "logps/chosen": -1.4229803085327148,
+      "logps/rejected": -1.5248156785964966,
+      "loss": 1.1507,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -1.4229803085327148,
+      "rewards/margins": 0.10183525085449219,
+      "rewards/rejected": -1.5248156785964966,
+      "step": 1530
+    },
+    {
+      "epoch": 0.2653342522398346,
+      "grad_norm": 11.622237173329605,
+      "learning_rate": 4.983780598313423e-08,
+      "logits/chosen": -3.5772109031677246,
+      "logits/rejected": -3.5598843097686768,
+      "logps/chosen": -1.5237246751785278,
+      "logps/rejected": -1.5880690813064575,
+      "loss": 1.1655,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -1.5237246751785278,
+      "rewards/margins": 0.06434442102909088,
+      "rewards/rejected": -1.5880690813064575,
+      "step": 1540
+    },
+    {
+      "epoch": 0.26705720192970367,
+      "grad_norm": 7.773843072518502,
+      "learning_rate": 4.982914390677909e-08,
+      "logits/chosen": -3.53175687789917,
+      "logits/rejected": -3.519580125808716,
+      "logps/chosen": -1.534958004951477,
+      "logps/rejected": -1.6059799194335938,
+      "loss": 1.1606,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -1.534958004951477,
+      "rewards/margins": 0.07102178037166595,
+      "rewards/rejected": -1.6059799194335938,
+      "step": 1550
+    },
+    {
+      "epoch": 0.2687801516195727,
+      "grad_norm": 9.408111522536476,
+      "learning_rate": 4.982025729856381e-08,
+      "logits/chosen": -3.5352489948272705,
+      "logits/rejected": -3.526162624359131,
+      "logps/chosen": -1.4781947135925293,
+      "logps/rejected": -1.5787986516952515,
+      "loss": 1.1439,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -1.4781947135925293,
+      "rewards/margins": 0.1006040945649147,
+      "rewards/rejected": -1.5787986516952515,
+      "step": 1560
+    },
+    {
+      "epoch": 0.2705031013094418,
+      "grad_norm": 8.247669788711871,
+      "learning_rate": 4.981114623885067e-08,
+      "logits/chosen": -3.5758960247039795,
+      "logits/rejected": -3.5762381553649902,
+      "logps/chosen": -1.4333528280258179,
+      "logps/rejected": -1.6404941082000732,
+      "loss": 1.0887,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.4333528280258179,
+      "rewards/margins": 0.20714128017425537,
+      "rewards/rejected": -1.6404941082000732,
+      "step": 1570
+    },
+    {
+      "epoch": 0.2722260509993108,
+      "grad_norm": 9.028781536203159,
+      "learning_rate": 4.980181081003167e-08,
+      "logits/chosen": -3.532313585281372,
+      "logits/rejected": -3.526517868041992,
+      "logps/chosen": -1.512021780014038,
+      "logps/rejected": -1.6078389883041382,
+      "loss": 1.1497,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.512021780014038,
+      "rewards/margins": 0.09581725299358368,
+      "rewards/rejected": -1.6078389883041382,
+      "step": 1580
+    },
+    {
+      "epoch": 0.2739490006891799,
+      "grad_norm": 8.61020330048939,
+      "learning_rate": 4.9792251096527826e-08,
+      "logits/chosen": -3.555513858795166,
+      "logits/rejected": -3.5484211444854736,
+      "logps/chosen": -1.4531891345977783,
+      "logps/rejected": -1.5783554315567017,
+      "loss": 1.1292,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.4531891345977783,
+      "rewards/margins": 0.1251663863658905,
+      "rewards/rejected": -1.5783554315567017,
+      "step": 1590
+    },
+    {
+      "epoch": 0.27567195037904896,
+      "grad_norm": 9.816994132667192,
+      "learning_rate": 4.978246718478836e-08,
+      "logits/chosen": -3.5611846446990967,
+      "logits/rejected": -3.537736415863037,
+      "logps/chosen": -1.4463131427764893,
+      "logps/rejected": -1.6123243570327759,
+      "loss": 1.1065,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.4463131427764893,
+      "rewards/margins": 0.16601115465164185,
+      "rewards/rejected": -1.6123243570327759,
+      "step": 1600
+    },
+    {
+      "epoch": 0.27567195037904896,
+      "eval_logits/chosen": -3.636399507522583,
+      "eval_logits/rejected": -3.632383346557617,
+      "eval_logps/chosen": -1.4808340072631836,
+      "eval_logps/rejected": -1.6113137006759644,
+      "eval_loss": 1.1154773235321045,
+      "eval_rewards/accuracies": 0.5938661694526672,
+      "eval_rewards/chosen": -1.4808340072631836,
+      "eval_rewards/margins": 0.13047975301742554,
+      "eval_rewards/rejected": -1.6113137006759644,
+      "eval_runtime": 155.4423,
+      "eval_samples_per_second": 27.689,
+      "eval_steps_per_second": 3.461,
+      "step": 1600
+    },
+    {
+      "epoch": 0.277394900068918,
+      "grad_norm": 8.771797057320093,
+      "learning_rate": 4.9772459163289934e-08,
+      "logits/chosen": -3.5799899101257324,
+      "logits/rejected": -3.565293073654175,
+      "logps/chosen": -1.544592261314392,
+      "logps/rejected": -1.7185720205307007,
+      "loss": 1.115,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.544592261314392,
+      "rewards/margins": 0.17397978901863098,
+      "rewards/rejected": -1.7185720205307007,
+      "step": 1610
+    },
+    {
+      "epoch": 0.27911784975878706,
+      "grad_norm": 9.057796806553597,
+      "learning_rate": 4.976222712253587e-08,
+      "logits/chosen": -3.5400052070617676,
+      "logits/rejected": -3.5265719890594482,
+      "logps/chosen": -1.427344799041748,
+      "logps/rejected": -1.62514328956604,
+      "loss": 1.088,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.427344799041748,
+      "rewards/margins": 0.197798490524292,
+      "rewards/rejected": -1.62514328956604,
+      "step": 1620
+    },
+    {
+      "epoch": 0.2808407994486561,
+      "grad_norm": 8.540234292401845,
+      "learning_rate": 4.9751771155055295e-08,
+      "logits/chosen": -3.5896599292755127,
+      "logits/rejected": -3.5730438232421875,
+      "logps/chosen": -1.5255366563796997,
+      "logps/rejected": -1.6556854248046875,
+      "loss": 1.1401,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -1.5255366563796997,
+      "rewards/margins": 0.13014879822731018,
+      "rewards/rejected": -1.6556854248046875,
+      "step": 1630
+    },
+    {
+      "epoch": 0.28256374913852517,
+      "grad_norm": 8.962692724323208,
+      "learning_rate": 4.974109135540232e-08,
+      "logits/chosen": -3.594813823699951,
+      "logits/rejected": -3.5737595558166504,
+      "logps/chosen": -1.5236104726791382,
+      "logps/rejected": -1.6805284023284912,
+      "loss": 1.1381,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -1.5236104726791382,
+      "rewards/margins": 0.15691788494586945,
+      "rewards/rejected": -1.6805284023284912,
+      "step": 1640
+    },
+    {
+      "epoch": 0.2842866988283942,
+      "grad_norm": 9.217946857810777,
+      "learning_rate": 4.97301878201552e-08,
+      "logits/chosen": -3.5736281871795654,
+      "logits/rejected": -3.5576674938201904,
+      "logps/chosen": -1.537434697151184,
+      "logps/rejected": -1.6023845672607422,
+      "loss": 1.1749,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -1.537434697151184,
+      "rewards/margins": 0.0649498850107193,
+      "rewards/rejected": -1.6023845672607422,
+      "step": 1650
+    },
+    {
+      "epoch": 0.28600964851826327,
+      "grad_norm": 9.640226377271745,
+      "learning_rate": 4.971906064791544e-08,
+      "logits/chosen": -3.607038974761963,
+      "logits/rejected": -3.5787124633789062,
+      "logps/chosen": -1.4626582860946655,
+      "logps/rejected": -1.661026954650879,
+      "loss": 1.0819,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.4626582860946655,
+      "rewards/margins": 0.1983688771724701,
+      "rewards/rejected": -1.661026954650879,
+      "step": 1660
+    },
+    {
+      "epoch": 0.2877325982081323,
+      "grad_norm": 8.674779184243462,
+      "learning_rate": 4.970770993930693e-08,
+      "logits/chosen": -3.6016101837158203,
+      "logits/rejected": -3.5898163318634033,
+      "logps/chosen": -1.4576081037521362,
+      "logps/rejected": -1.6540238857269287,
+      "loss": 1.1024,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.4576081037521362,
+      "rewards/margins": 0.19641588628292084,
+      "rewards/rejected": -1.6540238857269287,
+      "step": 1670
+    },
+    {
+      "epoch": 0.2894555478980014,
+      "grad_norm": 9.570898480094508,
+      "learning_rate": 4.969613579697499e-08,
+      "logits/chosen": -3.571596145629883,
+      "logits/rejected": -3.556870698928833,
+      "logps/chosen": -1.5613571405410767,
+      "logps/rejected": -1.7099859714508057,
+      "loss": 1.1221,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.5613571405410767,
+      "rewards/margins": 0.14862892031669617,
+      "rewards/rejected": -1.7099859714508057,
+      "step": 1680
+    },
+    {
+      "epoch": 0.29117849758787046,
+      "grad_norm": 7.78658460533033,
+      "learning_rate": 4.968433832558549e-08,
+      "logits/chosen": -3.555373430252075,
+      "logits/rejected": -3.545031785964966,
+      "logps/chosen": -1.542994499206543,
+      "logps/rejected": -1.6029647588729858,
+      "loss": 1.1773,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -1.542994499206543,
+      "rewards/margins": 0.0599704384803772,
+      "rewards/rejected": -1.6029647588729858,
+      "step": 1690
+    },
+    {
+      "epoch": 0.2929014472777395,
+      "grad_norm": 10.562982508445227,
+      "learning_rate": 4.967231763182385e-08,
+      "logits/chosen": -3.4932034015655518,
+      "logits/rejected": -3.497729539871216,
+      "logps/chosen": -1.4713702201843262,
+      "logps/rejected": -1.6032283306121826,
+      "loss": 1.1183,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.4713702201843262,
+      "rewards/margins": 0.1318581998348236,
+      "rewards/rejected": -1.6032283306121826,
+      "step": 1700
+    },
+    {
+      "epoch": 0.2929014472777395,
+      "eval_logits/chosen": -3.6216518878936768,
+      "eval_logits/rejected": -3.617584228515625,
+      "eval_logps/chosen": -1.482006311416626,
+      "eval_logps/rejected": -1.6128884553909302,
+      "eval_loss": 1.1152552366256714,
+      "eval_rewards/accuracies": 0.5961896181106567,
+      "eval_rewards/chosen": -1.482006311416626,
+      "eval_rewards/margins": 0.130882129073143,
+      "eval_rewards/rejected": -1.6128884553909302,
+      "eval_runtime": 155.7415,
+      "eval_samples_per_second": 27.636,
+      "eval_steps_per_second": 3.454,
+      "step": 1700
+    },
+    {
+      "epoch": 0.29462439696760856,
+      "grad_norm": 8.371535097865223,
+      "learning_rate": 4.966007382439414e-08,
+      "logits/chosen": -3.577683925628662,
+      "logits/rejected": -3.554837465286255,
+      "logps/chosen": -1.5099198818206787,
+      "logps/rejected": -1.6713581085205078,
+      "loss": 1.1154,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.5099198818206787,
+      "rewards/margins": 0.1614382266998291,
+      "rewards/rejected": -1.6713581085205078,
+      "step": 1710
+    },
+    {
+      "epoch": 0.2963473466574776,
+      "grad_norm": 9.083374834956018,
+      "learning_rate": 4.964760701401807e-08,
+      "logits/chosen": -3.5936055183410645,
+      "logits/rejected": -3.5782253742218018,
+      "logps/chosen": -1.5432225465774536,
+      "logps/rejected": -1.5661277770996094,
+      "loss": 1.2017,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -1.5432225465774536,
+      "rewards/margins": 0.022905509918928146,
+      "rewards/rejected": -1.5661277770996094,
+      "step": 1720
+    },
+    {
+      "epoch": 0.29807029634734666,
+      "grad_norm": 8.81621940685166,
+      "learning_rate": 4.963491731343395e-08,
+      "logits/chosen": -3.574502944946289,
+      "logits/rejected": -3.5635809898376465,
+      "logps/chosen": -1.5191514492034912,
+      "logps/rejected": -1.5796552896499634,
+      "loss": 1.1756,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -1.5191514492034912,
+      "rewards/margins": 0.060503702610731125,
+      "rewards/rejected": -1.5796552896499634,
+      "step": 1730
+    },
+    {
+      "epoch": 0.2997932460372157,
+      "grad_norm": 7.656526501944126,
+      "learning_rate": 4.9622004837395725e-08,
+      "logits/chosen": -3.546746015548706,
+      "logits/rejected": -3.540705442428589,
+      "logps/chosen": -1.4870233535766602,
+      "logps/rejected": -1.6325324773788452,
+      "loss": 1.1251,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.4870233535766602,
+      "rewards/margins": 0.14550921320915222,
+      "rewards/rejected": -1.6325324773788452,
+      "step": 1740
+    },
+    {
+      "epoch": 0.30151619572708477,
+      "grad_norm": 8.838049530493036,
+      "learning_rate": 4.9608869702671903e-08,
+      "logits/chosen": -3.5892186164855957,
+      "logits/rejected": -3.580435276031494,
+      "logps/chosen": -1.4661469459533691,
+      "logps/rejected": -1.5750914812088013,
+      "loss": 1.1371,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -1.4661469459533691,
+      "rewards/margins": 0.10894447565078735,
+      "rewards/rejected": -1.5750914812088013,
+      "step": 1750
+    },
+    {
+      "epoch": 0.30323914541695385,
+      "grad_norm": 8.08425801835279,
+      "learning_rate": 4.9595512028044526e-08,
+      "logits/chosen": -3.572713851928711,
+      "logits/rejected": -3.551302433013916,
+      "logps/chosen": -1.481914758682251,
+      "logps/rejected": -1.6230970621109009,
+      "loss": 1.1213,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.481914758682251,
+      "rewards/margins": 0.1411823183298111,
+      "rewards/rejected": -1.6230970621109009,
+      "step": 1760
+    },
+    {
+      "epoch": 0.3049620951068229,
+      "grad_norm": 7.158214824637413,
+      "learning_rate": 4.958193193430807e-08,
+      "logits/chosen": -3.588897228240967,
+      "logits/rejected": -3.5688843727111816,
+      "logps/chosen": -1.52801513671875,
+      "logps/rejected": -1.6208654642105103,
+      "loss": 1.1468,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.52801513671875,
+      "rewards/margins": 0.09285024553537369,
+      "rewards/rejected": -1.6208654642105103,
+      "step": 1770
+    },
+    {
+      "epoch": 0.30668504479669195,
+      "grad_norm": 9.062261342870794,
+      "learning_rate": 4.956812954426837e-08,
+      "logits/chosen": -3.5264506340026855,
+      "logits/rejected": -3.528055191040039,
+      "logps/chosen": -1.5251517295837402,
+      "logps/rejected": -1.6560192108154297,
+      "loss": 1.1234,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.5251517295837402,
+      "rewards/margins": 0.13086728751659393,
+      "rewards/rejected": -1.6560192108154297,
+      "step": 1780
+    },
+    {
+      "epoch": 0.308407994486561,
+      "grad_norm": 8.172425625001665,
+      "learning_rate": 4.9554104982741504e-08,
+      "logits/chosen": -3.6364598274230957,
+      "logits/rejected": -3.6227550506591797,
+      "logps/chosen": -1.429160714149475,
+      "logps/rejected": -1.590442419052124,
+      "loss": 1.1057,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.429160714149475,
+      "rewards/margins": 0.16128182411193848,
+      "rewards/rejected": -1.590442419052124,
+      "step": 1790
+    },
+    {
+      "epoch": 0.31013094417643006,
+      "grad_norm": 8.705863618645926,
+      "learning_rate": 4.953985837655266e-08,
+      "logits/chosen": -3.5992684364318848,
+      "logits/rejected": -3.5862250328063965,
+      "logps/chosen": -1.4731065034866333,
+      "logps/rejected": -1.653342843055725,
+      "loss": 1.0866,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.4731065034866333,
+      "rewards/margins": 0.1802363395690918,
+      "rewards/rejected": -1.653342843055725,
+      "step": 1800
+    },
+    {
+      "epoch": 0.31013094417643006,
+      "eval_logits/chosen": -3.624330759048462,
+      "eval_logits/rejected": -3.620286703109741,
+      "eval_logps/chosen": -1.4822847843170166,
+      "eval_logps/rejected": -1.6138156652450562,
+      "eval_loss": 1.1148669719696045,
+      "eval_rewards/accuracies": 0.5971189737319946,
+      "eval_rewards/chosen": -1.4822847843170166,
+      "eval_rewards/margins": 0.1315307319164276,
+      "eval_rewards/rejected": -1.6138156652450562,
+      "eval_runtime": 155.509,
+      "eval_samples_per_second": 27.677,
+      "eval_steps_per_second": 3.46,
+      "step": 1800
+    },
+    {
+      "epoch": 0.3118538938662991,
+      "grad_norm": 10.343739313334776,
+      "learning_rate": 4.952538985453499e-08,
+      "logits/chosen": -3.6106178760528564,
+      "logits/rejected": -3.5929102897644043,
+      "logps/chosen": -1.4777882099151611,
+      "logps/rejected": -1.534605622291565,
+      "loss": 1.1805,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -1.4777882099151611,
+      "rewards/margins": 0.05681762844324112,
+      "rewards/rejected": -1.534605622291565,
+      "step": 1810
+    },
+    {
+      "epoch": 0.31357684355616816,
+      "grad_norm": 9.535271421192423,
+      "learning_rate": 4.9510699547528456e-08,
+      "logits/chosen": -3.5892577171325684,
+      "logits/rejected": -3.567037582397461,
+      "logps/chosen": -1.5061160326004028,
+      "logps/rejected": -1.5864845514297485,
+      "loss": 1.158,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.5061160326004028,
+      "rewards/margins": 0.08036859333515167,
+      "rewards/rejected": -1.5864845514297485,
+      "step": 1820
+    },
+    {
+      "epoch": 0.31529979324603724,
+      "grad_norm": 8.364974831839044,
+      "learning_rate": 4.949578758837864e-08,
+      "logits/chosen": -3.519291639328003,
+      "logits/rejected": -3.509608030319214,
+      "logps/chosen": -1.5382378101348877,
+      "logps/rejected": -1.6768662929534912,
+      "loss": 1.1255,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.5382378101348877,
+      "rewards/margins": 0.13862848281860352,
+      "rewards/rejected": -1.6768662929534912,
+      "step": 1830
+    },
+    {
+      "epoch": 0.31702274293590627,
+      "grad_norm": 9.37091060021702,
+      "learning_rate": 4.948065411193554e-08,
+      "logits/chosen": -3.662789821624756,
+      "logits/rejected": -3.660649538040161,
+      "logps/chosen": -1.5870919227600098,
+      "logps/rejected": -1.715182900428772,
+      "loss": 1.1621,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.5870919227600098,
+      "rewards/margins": 0.12809090316295624,
+      "rewards/rejected": -1.715182900428772,
+      "step": 1840
+    },
+    {
+      "epoch": 0.31874569262577535,
+      "grad_norm": 9.624819364282102,
+      "learning_rate": 4.946529925505233e-08,
+      "logits/chosen": -3.572620391845703,
+      "logits/rejected": -3.572563886642456,
+      "logps/chosen": -1.4941456317901611,
+      "logps/rejected": -1.5951688289642334,
+      "loss": 1.1491,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.4941456317901611,
+      "rewards/margins": 0.10102321952581406,
+      "rewards/rejected": -1.5951688289642334,
+      "step": 1850
+    },
+    {
+      "epoch": 0.32046864231564437,
+      "grad_norm": 11.651393854892156,
+      "learning_rate": 4.9449723156584175e-08,
+      "logits/chosen": -3.5314345359802246,
+      "logits/rejected": -3.5093250274658203,
+      "logps/chosen": -1.4843990802764893,
+      "logps/rejected": -1.6998584270477295,
+      "loss": 1.0778,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.4843990802764893,
+      "rewards/margins": 0.21545949578285217,
+      "rewards/rejected": -1.6998584270477295,
+      "step": 1860
+    },
+    {
+      "epoch": 0.32219159200551345,
+      "grad_norm": 10.460532235172016,
+      "learning_rate": 4.943392595738694e-08,
+      "logits/chosen": -3.5690855979919434,
+      "logits/rejected": -3.5554332733154297,
+      "logps/chosen": -1.5141363143920898,
+      "logps/rejected": -1.6883583068847656,
+      "loss": 1.1046,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -1.5141363143920898,
+      "rewards/margins": 0.17422200739383698,
+      "rewards/rejected": -1.6883583068847656,
+      "step": 1870
+    },
+    {
+      "epoch": 0.3239145416953825,
+      "grad_norm": 9.18442867917496,
+      "learning_rate": 4.9417907800315904e-08,
+      "logits/chosen": -3.5879623889923096,
+      "logits/rejected": -3.5641307830810547,
+      "logps/chosen": -1.399189829826355,
+      "logps/rejected": -1.6410026550292969,
+      "loss": 1.0702,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.399189829826355,
+      "rewards/margins": 0.24181267619132996,
+      "rewards/rejected": -1.6410026550292969,
+      "step": 1880
+    },
+    {
+      "epoch": 0.32563749138525155,
+      "grad_norm": 12.193682310065121,
+      "learning_rate": 4.94016688302245e-08,
+      "logits/chosen": -3.606241226196289,
+      "logits/rejected": -3.601780652999878,
+      "logps/chosen": -1.4991823434829712,
+      "logps/rejected": -1.6474968194961548,
+      "loss": 1.1265,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -1.4991823434829712,
+      "rewards/margins": 0.14831440150737762,
+      "rewards/rejected": -1.6474968194961548,
+      "step": 1890
+    },
+    {
+      "epoch": 0.32736044107512063,
+      "grad_norm": 9.242696712971243,
+      "learning_rate": 4.9385209193962974e-08,
+      "logits/chosen": -3.5562233924865723,
+      "logits/rejected": -3.5363094806671143,
+      "logps/chosen": -1.52472722530365,
+      "logps/rejected": -1.589041829109192,
+      "loss": 1.1745,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -1.52472722530365,
+      "rewards/margins": 0.06431449949741364,
+      "rewards/rejected": -1.589041829109192,
+      "step": 1900
+    },
+    {
+      "epoch": 0.32736044107512063,
+      "eval_logits/chosen": -3.6254584789276123,
+      "eval_logits/rejected": -3.6214439868927,
+      "eval_logps/chosen": -1.4834810495376587,
+      "eval_logps/rejected": -1.6155078411102295,
+      "eval_loss": 1.114662528038025,
+      "eval_rewards/accuracies": 0.6001393795013428,
+      "eval_rewards/chosen": -1.4834810495376587,
+      "eval_rewards/margins": 0.13202698528766632,
+      "eval_rewards/rejected": -1.6155078411102295,
+      "eval_runtime": 155.6746,
+      "eval_samples_per_second": 27.647,
+      "eval_steps_per_second": 3.456,
+      "step": 1900
+    },
+    {
+      "epoch": 0.32908339076498966,
+      "grad_norm": 7.6883253203658475,
+      "learning_rate": 4.93685290403771e-08,
+      "logits/chosen": -3.4900894165039062,
+      "logits/rejected": -3.4698398113250732,
+      "logps/chosen": -1.4763076305389404,
+      "logps/rejected": -1.6699857711791992,
+      "loss": 1.0791,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.4763076305389404,
+      "rewards/margins": 0.19367823004722595,
+      "rewards/rejected": -1.6699857711791992,
+      "step": 1910
+    },
+    {
+      "epoch": 0.33080634045485874,
+      "grad_norm": 9.158601357875666,
+      "learning_rate": 4.9351628520306774e-08,
+      "logits/chosen": -3.5717647075653076,
+      "logits/rejected": -3.5562915802001953,
+      "logps/chosen": -1.5173413753509521,
+      "logps/rejected": -1.679216742515564,
+      "loss": 1.1208,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.5173413753509521,
+      "rewards/margins": 0.1618753969669342,
+      "rewards/rejected": -1.679216742515564,
+      "step": 1920
+    },
+    {
+      "epoch": 0.33252929014472776,
+      "grad_norm": 9.38759338296592,
+      "learning_rate": 4.933450778658472e-08,
+      "logits/chosen": -3.537283420562744,
+      "logits/rejected": -3.5144882202148438,
+      "logps/chosen": -1.4468146562576294,
+      "logps/rejected": -1.758718490600586,
+      "loss": 1.0497,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.4468146562576294,
+      "rewards/margins": 0.31190386414527893,
+      "rewards/rejected": -1.758718490600586,
+      "step": 1930
+    },
+    {
+      "epoch": 0.33425223983459684,
+      "grad_norm": 8.124495258949395,
+      "learning_rate": 4.9317166994035036e-08,
+      "logits/chosen": -3.5561130046844482,
+      "logits/rejected": -3.5463173389434814,
+      "logps/chosen": -1.4890791177749634,
+      "logps/rejected": -1.6363897323608398,
+      "loss": 1.1174,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.4890791177749634,
+      "rewards/margins": 0.14731065928936005,
+      "rewards/rejected": -1.6363897323608398,
+      "step": 1940
+    },
+    {
+      "epoch": 0.33597518952446587,
+      "grad_norm": 9.618423399533341,
+      "learning_rate": 4.929960629947185e-08,
+      "logits/chosen": -3.5452873706817627,
+      "logits/rejected": -3.543883800506592,
+      "logps/chosen": -1.519731044769287,
+      "logps/rejected": -1.6294746398925781,
+      "loss": 1.1414,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -1.519731044769287,
+      "rewards/margins": 0.10974369943141937,
+      "rewards/rejected": -1.6294746398925781,
+      "step": 1950
+    },
+    {
+      "epoch": 0.33769813921433495,
+      "grad_norm": 9.526996304072375,
+      "learning_rate": 4.928182586169787e-08,
+      "logits/chosen": -3.5529685020446777,
+      "logits/rejected": -3.5428760051727295,
+      "logps/chosen": -1.5380550622940063,
+      "logps/rejected": -1.7287752628326416,
+      "loss": 1.1176,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.5380550622940063,
+      "rewards/margins": 0.1907200664281845,
+      "rewards/rejected": -1.7287752628326416,
+      "step": 1960
+    },
+    {
+      "epoch": 0.33942108890420397,
+      "grad_norm": 9.050754719660823,
+      "learning_rate": 4.926382584150298e-08,
+      "logits/chosen": -3.5908329486846924,
+      "logits/rejected": -3.5721678733825684,
+      "logps/chosen": -1.522635579109192,
+      "logps/rejected": -1.6528599262237549,
+      "loss": 1.1424,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -1.522635579109192,
+      "rewards/margins": 0.13022422790527344,
+      "rewards/rejected": -1.6528599262237549,
+      "step": 1970
+    },
+    {
+      "epoch": 0.34114403859407305,
+      "grad_norm": 8.782565408940735,
+      "learning_rate": 4.924560640166273e-08,
+      "logits/chosen": -3.5299715995788574,
+      "logits/rejected": -3.5261306762695312,
+      "logps/chosen": -1.5514591932296753,
+      "logps/rejected": -1.6045089960098267,
+      "loss": 1.1719,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -1.5514591932296753,
+      "rewards/margins": 0.053049731999635696,
+      "rewards/rejected": -1.6045089960098267,
+      "step": 1980
+    },
+    {
+      "epoch": 0.34286698828394213,
+      "grad_norm": 8.278223867056402,
+      "learning_rate": 4.922716770693692e-08,
+      "logits/chosen": -3.6056265830993652,
+      "logits/rejected": -3.586510419845581,
+      "logps/chosen": -1.4696927070617676,
+      "logps/rejected": -1.612221121788025,
+      "loss": 1.1216,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.4696927070617676,
+      "rewards/margins": 0.1425286829471588,
+      "rewards/rejected": -1.612221121788025,
+      "step": 1990
+    },
+    {
+      "epoch": 0.34458993797381116,
+      "grad_norm": 8.718744030985253,
+      "learning_rate": 4.920850992406809e-08,
+      "logits/chosen": -3.56001353263855,
+      "logits/rejected": -3.5631675720214844,
+      "logps/chosen": -1.532480239868164,
+      "logps/rejected": -1.7376270294189453,
+      "loss": 1.1004,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.532480239868164,
+      "rewards/margins": 0.20514675974845886,
+      "rewards/rejected": -1.7376270294189453,
+      "step": 2000
+    },
+    {
+      "epoch": 0.34458993797381116,
+      "eval_logits/chosen": -3.6254193782806396,
+      "eval_logits/rejected": -3.6214253902435303,
+      "eval_logps/chosen": -1.484678864479065,
+      "eval_logps/rejected": -1.6174685955047607,
+      "eval_loss": 1.1141842603683472,
+      "eval_rewards/accuracies": 0.6003717184066772,
+      "eval_rewards/chosen": -1.484678864479065,
+      "eval_rewards/margins": 0.1327897310256958,
+      "eval_rewards/rejected": -1.6174685955047607,
+      "eval_runtime": 155.4033,
+      "eval_samples_per_second": 27.696,
+      "eval_steps_per_second": 3.462,
+      "step": 2000
+    },
+    {
+      "epoch": 0.34631288766368024,
+      "grad_norm": 8.580618835861072,
+      "learning_rate": 4.918963322178001e-08,
+      "logits/chosen": -3.557722568511963,
+      "logits/rejected": -3.5414180755615234,
+      "logps/chosen": -1.530787706375122,
+      "logps/rejected": -1.6451923847198486,
+      "loss": 1.1383,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -1.530787706375122,
+      "rewards/margins": 0.11440470069646835,
+      "rewards/rejected": -1.6451923847198486,
+      "step": 2010
+    },
+    {
+      "epoch": 0.34803583735354926,
+      "grad_norm": 9.281213479730413,
+      "learning_rate": 4.917053777077616e-08,
+      "logits/chosen": -3.5595901012420654,
+      "logits/rejected": -3.5449485778808594,
+      "logps/chosen": -1.4598172903060913,
+      "logps/rejected": -1.7434160709381104,
+      "loss": 1.051,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.4598172903060913,
+      "rewards/margins": 0.2835986018180847,
+      "rewards/rejected": -1.7434160709381104,
+      "step": 2020
+    },
+    {
+      "epoch": 0.34975878704341834,
+      "grad_norm": 10.137321768951493,
+      "learning_rate": 4.915122374373815e-08,
+      "logits/chosen": -3.604844570159912,
+      "logits/rejected": -3.5928657054901123,
+      "logps/chosen": -1.5535376071929932,
+      "logps/rejected": -1.6784002780914307,
+      "loss": 1.1338,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.5535376071929932,
+      "rewards/margins": 0.12486279010772705,
+      "rewards/rejected": -1.6784002780914307,
+      "step": 2030
+    },
+    {
+      "epoch": 0.35148173673328736,
+      "grad_norm": 8.299949098323657,
+      "learning_rate": 4.9131691315324224e-08,
+      "logits/chosen": -3.5199809074401855,
+      "logits/rejected": -3.510056257247925,
+      "logps/chosen": -1.4815999269485474,
+      "logps/rejected": -1.670644760131836,
+      "loss": 1.0899,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.4815999269485474,
+      "rewards/margins": 0.18904481828212738,
+      "rewards/rejected": -1.670644760131836,
+      "step": 2040
+    },
+    {
+      "epoch": 0.35320468642315644,
+      "grad_norm": 8.899025153824818,
+      "learning_rate": 4.911194066216765e-08,
+      "logits/chosen": -3.6048805713653564,
+      "logits/rejected": -3.5915024280548096,
+      "logps/chosen": -1.4568990468978882,
+      "logps/rejected": -1.6984679698944092,
+      "loss": 1.0619,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.4568990468978882,
+      "rewards/margins": 0.24156884849071503,
+      "rewards/rejected": -1.6984679698944092,
+      "step": 2050
+    },
+    {
+      "epoch": 0.3549276361130255,
+      "grad_norm": 8.1013444726292,
+      "learning_rate": 4.909197196287509e-08,
+      "logits/chosen": -3.5872206687927246,
+      "logits/rejected": -3.5640861988067627,
+      "logps/chosen": -1.4317524433135986,
+      "logps/rejected": -1.5729509592056274,
+      "loss": 1.1092,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.4317524433135986,
+      "rewards/margins": 0.14119867980480194,
+      "rewards/rejected": -1.5729509592056274,
+      "step": 2060
+    },
+    {
+      "epoch": 0.35665058580289455,
+      "grad_norm": 10.485044051715956,
+      "learning_rate": 4.907178539802503e-08,
+      "logits/chosen": -3.5922820568084717,
+      "logits/rejected": -3.578859329223633,
+      "logps/chosen": -1.5264912843704224,
+      "logps/rejected": -1.6675529479980469,
+      "loss": 1.1319,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.5264912843704224,
+      "rewards/margins": 0.14106181263923645,
+      "rewards/rejected": -1.6675529479980469,
+      "step": 2070
+    },
+    {
+      "epoch": 0.35837353549276363,
+      "grad_norm": 7.871796258966659,
+      "learning_rate": 4.9051381150166136e-08,
+      "logits/chosen": -3.5675346851348877,
+      "logits/rejected": -3.550717830657959,
+      "logps/chosen": -1.5141642093658447,
+      "logps/rejected": -1.642620325088501,
+      "loss": 1.1327,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.5141642093658447,
+      "rewards/margins": 0.1284559816122055,
+      "rewards/rejected": -1.642620325088501,
+      "step": 2080
+    },
+    {
+      "epoch": 0.36009648518263265,
+      "grad_norm": 9.649436842293689,
+      "learning_rate": 4.903075940381559e-08,
+      "logits/chosen": -3.5802865028381348,
+      "logits/rejected": -3.578158140182495,
+      "logps/chosen": -1.5138075351715088,
+      "logps/rejected": -1.5619828701019287,
+      "loss": 1.1797,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -1.5138075351715088,
+      "rewards/margins": 0.048175424337387085,
+      "rewards/rejected": -1.5619828701019287,
+      "step": 2090
+    },
+    {
+      "epoch": 0.36181943487250173,
+      "grad_norm": 10.418394348832615,
+      "learning_rate": 4.900992034545743e-08,
+      "logits/chosen": -3.538865327835083,
+      "logits/rejected": -3.5269134044647217,
+      "logps/chosen": -1.5585479736328125,
+      "logps/rejected": -1.6414644718170166,
+      "loss": 1.1671,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.5585479736328125,
+      "rewards/margins": 0.08291643112897873,
+      "rewards/rejected": -1.6414644718170166,
+      "step": 2100
+    },
+    {
+      "epoch": 0.36181943487250173,
+      "eval_logits/chosen": -3.6233248710632324,
+      "eval_logits/rejected": -3.6193366050720215,
+      "eval_logps/chosen": -1.486056923866272,
+      "eval_logps/rejected": -1.6193512678146362,
+      "eval_loss": 1.1138821840286255,
+      "eval_rewards/accuracies": 0.6033921837806702,
+      "eval_rewards/chosen": -1.486056923866272,
+      "eval_rewards/margins": 0.1332944631576538,
+      "eval_rewards/rejected": -1.6193512678146362,
+      "eval_runtime": 155.5246,
+      "eval_samples_per_second": 27.674,
+      "eval_steps_per_second": 3.459,
+      "step": 2100
+    },
+    {
+      "epoch": 0.36354238456237076,
+      "grad_norm": 8.39247463525481,
+      "learning_rate": 4.898886416354088e-08,
+      "logits/chosen": -3.5929112434387207,
+      "logits/rejected": -3.5961251258850098,
+      "logps/chosen": -1.4845209121704102,
+      "logps/rejected": -1.6717685461044312,
+      "loss": 1.0963,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.4845209121704102,
+      "rewards/margins": 0.18724775314331055,
+      "rewards/rejected": -1.6717685461044312,
+      "step": 2110
+    },
+    {
+      "epoch": 0.36526533425223984,
+      "grad_norm": 8.662143224240031,
+      "learning_rate": 4.896759104847859e-08,
+      "logits/chosen": -3.4902141094207764,
+      "logits/rejected": -3.4730422496795654,
+      "logps/chosen": -1.4358088970184326,
+      "logps/rejected": -1.610970139503479,
+      "loss": 1.1087,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -1.4358088970184326,
+      "rewards/margins": 0.17516134679317474,
+      "rewards/rejected": -1.610970139503479,
+      "step": 2120
+    },
+    {
+      "epoch": 0.3669882839421089,
+      "grad_norm": 9.99221159960688,
+      "learning_rate": 4.8946101192644994e-08,
+      "logits/chosen": -3.509303569793701,
+      "logits/rejected": -3.4994564056396484,
+      "logps/chosen": -1.5596240758895874,
+      "logps/rejected": -1.667401909828186,
+      "loss": 1.1488,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -1.5596240758895874,
+      "rewards/margins": 0.10777787864208221,
+      "rewards/rejected": -1.667401909828186,
+      "step": 2130
+    },
+    {
+      "epoch": 0.36871123363197794,
+      "grad_norm": 9.325521523632396,
+      "learning_rate": 4.8924394790374505e-08,
+      "logits/chosen": -3.5076375007629395,
+      "logits/rejected": -3.503007173538208,
+      "logps/chosen": -1.5417354106903076,
+      "logps/rejected": -1.7959156036376953,
+      "loss": 1.0732,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.5417354106903076,
+      "rewards/margins": 0.25418025255203247,
+      "rewards/rejected": -1.7959156036376953,
+      "step": 2140
+    },
+    {
+      "epoch": 0.370434183321847,
+      "grad_norm": 9.243184720674815,
+      "learning_rate": 4.8902472037959796e-08,
+      "logits/chosen": -3.5364577770233154,
+      "logits/rejected": -3.5098605155944824,
+      "logps/chosen": -1.4313781261444092,
+      "logps/rejected": -1.5510797500610352,
+      "loss": 1.1277,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.4313781261444092,
+      "rewards/margins": 0.11970136314630508,
+      "rewards/rejected": -1.5510797500610352,
+      "step": 2150
+    },
+    {
+      "epoch": 0.37215713301171605,
+      "grad_norm": 10.060065377889348,
+      "learning_rate": 4.888033313365001e-08,
+      "logits/chosen": -3.520111083984375,
+      "logits/rejected": -3.5104897022247314,
+      "logps/chosen": -1.5878918170928955,
+      "logps/rejected": -1.5785599946975708,
+      "loss": 1.2303,
+      "rewards/accuracies": 0.48124998807907104,
+      "rewards/chosen": -1.5878918170928955,
+      "rewards/margins": -0.009331794455647469,
+      "rewards/rejected": -1.5785599946975708,
+      "step": 2160
+    },
+    {
+      "epoch": 0.3738800827015851,
+      "grad_norm": 10.267468692865302,
+      "learning_rate": 4.885797827764895e-08,
+      "logits/chosen": -3.579012393951416,
+      "logits/rejected": -3.569380521774292,
+      "logps/chosen": -1.5738365650177002,
+      "logps/rejected": -1.6351600885391235,
+      "loss": 1.1741,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -1.5738365650177002,
+      "rewards/margins": 0.06132357567548752,
+      "rewards/rejected": -1.6351600885391235,
+      "step": 2170
+    },
+    {
+      "epoch": 0.37560303239145415,
+      "grad_norm": 9.487603882500826,
+      "learning_rate": 4.88354076721133e-08,
+      "logits/chosen": -3.620032787322998,
+      "logits/rejected": -3.5970187187194824,
+      "logps/chosen": -1.5918430089950562,
+      "logps/rejected": -1.7353103160858154,
+      "loss": 1.1387,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -1.5918430089950562,
+      "rewards/margins": 0.14346706867218018,
+      "rewards/rejected": -1.7353103160858154,
+      "step": 2180
+    },
+    {
+      "epoch": 0.37732598208132323,
+      "grad_norm": 9.711671155964604,
+      "learning_rate": 4.88126215211508e-08,
+      "logits/chosen": -3.66611909866333,
+      "logits/rejected": -3.6625583171844482,
+      "logps/chosen": -1.527026891708374,
+      "logps/rejected": -1.655339002609253,
+      "loss": 1.1307,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -1.527026891708374,
+      "rewards/margins": 0.1283121407032013,
+      "rewards/rejected": -1.655339002609253,
+      "step": 2190
+    },
+    {
+      "epoch": 0.37904893177119225,
+      "grad_norm": 8.579536895102626,
+      "learning_rate": 4.878962003081835e-08,
+      "logits/chosen": -3.5521798133850098,
+      "logits/rejected": -3.537160873413086,
+      "logps/chosen": -1.4488705396652222,
+      "logps/rejected": -1.6617122888565063,
+      "loss": 1.0747,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.4488705396652222,
+      "rewards/margins": 0.21284165978431702,
+      "rewards/rejected": -1.6617122888565063,
+      "step": 2200
+    },
+    {
+      "epoch": 0.37904893177119225,
+      "eval_logits/chosen": -3.599992036819458,
+      "eval_logits/rejected": -3.5959177017211914,
+      "eval_logps/chosen": -1.4870764017105103,
+      "eval_logps/rejected": -1.6211183071136475,
+      "eval_loss": 1.113487958908081,
+      "eval_rewards/accuracies": 0.6019981503486633,
+      "eval_rewards/chosen": -1.4870764017105103,
+      "eval_rewards/margins": 0.13404183089733124,
+      "eval_rewards/rejected": -1.6211183071136475,
+      "eval_runtime": 155.42,
+      "eval_samples_per_second": 27.693,
+      "eval_steps_per_second": 3.462,
+      "step": 2200
+    },
+    {
+      "epoch": 0.38077188146106133,
+      "grad_norm": 8.591891903176634,
+      "learning_rate": 4.87664034091202e-08,
+      "logits/chosen": -3.5828323364257812,
+      "logits/rejected": -3.571755886077881,
+      "logps/chosen": -1.48390531539917,
+      "logps/rejected": -1.617065668106079,
+      "loss": 1.1221,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.48390531539917,
+      "rewards/margins": 0.13316050171852112,
+      "rewards/rejected": -1.617065668106079,
+      "step": 2210
+    },
+    {
+      "epoch": 0.3824948311509304,
+      "grad_norm": 10.420336869506817,
+      "learning_rate": 4.8742971866006064e-08,
+      "logits/chosen": -3.4786105155944824,
+      "logits/rejected": -3.474719285964966,
+      "logps/chosen": -1.5107654333114624,
+      "logps/rejected": -1.6830610036849976,
+      "loss": 1.108,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -1.5107654333114624,
+      "rewards/margins": 0.17229552567005157,
+      "rewards/rejected": -1.6830610036849976,
+      "step": 2220
+    },
+    {
+      "epoch": 0.38421778084079944,
+      "grad_norm": 9.505362527541953,
+      "learning_rate": 4.8719325613369177e-08,
+      "logits/chosen": -3.5765209197998047,
+      "logits/rejected": -3.5583434104919434,
+      "logps/chosen": -1.5185043811798096,
+      "logps/rejected": -1.6855275630950928,
+      "loss": 1.0998,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.5185043811798096,
+      "rewards/margins": 0.16702334582805634,
+      "rewards/rejected": -1.6855275630950928,
+      "step": 2230
+    },
+    {
+      "epoch": 0.3859407305306685,
+      "grad_norm": 8.506405011997174,
+      "learning_rate": 4.869546486504443e-08,
+      "logits/chosen": -3.5418124198913574,
+      "logits/rejected": -3.52287220954895,
+      "logps/chosen": -1.558027744293213,
+      "logps/rejected": -1.7133967876434326,
+      "loss": 1.1122,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.558027744293213,
+      "rewards/margins": 0.15536926686763763,
+      "rewards/rejected": -1.7133967876434326,
+      "step": 2240
+    },
+    {
+      "epoch": 0.38766368022053754,
+      "grad_norm": 10.041945678367812,
+      "learning_rate": 4.8671389836806395e-08,
+      "logits/chosen": -3.519800901412964,
+      "logits/rejected": -3.5012905597686768,
+      "logps/chosen": -1.5602412223815918,
+      "logps/rejected": -1.717995047569275,
+      "loss": 1.1061,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.5602412223815918,
+      "rewards/margins": 0.1577538251876831,
+      "rewards/rejected": -1.717995047569275,
+      "step": 2250
+    },
+    {
+      "epoch": 0.3893866299104066,
+      "grad_norm": 9.54972570394109,
+      "learning_rate": 4.864710074636742e-08,
+      "logits/chosen": -3.481370210647583,
+      "logits/rejected": -3.469743013381958,
+      "logps/chosen": -1.608527421951294,
+      "logps/rejected": -1.667845368385315,
+      "loss": 1.1756,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -1.608527421951294,
+      "rewards/margins": 0.05931812524795532,
+      "rewards/rejected": -1.667845368385315,
+      "step": 2260
+    },
+    {
+      "epoch": 0.39110957960027565,
+      "grad_norm": 10.20712172997886,
+      "learning_rate": 4.862259781337561e-08,
+      "logits/chosen": -3.4806771278381348,
+      "logits/rejected": -3.462923765182495,
+      "logps/chosen": -1.5732080936431885,
+      "logps/rejected": -1.7719624042510986,
+      "loss": 1.1005,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.5732080936431885,
+      "rewards/margins": 0.19875401258468628,
+      "rewards/rejected": -1.7719624042510986,
+      "step": 2270
+    },
+    {
+      "epoch": 0.3928325292901447,
+      "grad_norm": 10.769752367538553,
+      "learning_rate": 4.8597881259412874e-08,
+      "logits/chosen": -3.5415146350860596,
+      "logits/rejected": -3.5350022315979004,
+      "logps/chosen": -1.530076026916504,
+      "logps/rejected": -1.6715360879898071,
+      "loss": 1.1294,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.530076026916504,
+      "rewards/margins": 0.14146003127098083,
+      "rewards/rejected": -1.6715360879898071,
+      "step": 2280
+    },
+    {
+      "epoch": 0.3945554789800138,
+      "grad_norm": 8.788849296643331,
+      "learning_rate": 4.857295130799293e-08,
+      "logits/chosen": -3.46061372756958,
+      "logits/rejected": -3.4435105323791504,
+      "logps/chosen": -1.5006507635116577,
+      "logps/rejected": -1.7470293045043945,
+      "loss": 1.063,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.5006507635116577,
+      "rewards/margins": 0.24637861549854279,
+      "rewards/rejected": -1.7470293045043945,
+      "step": 2290
+    },
+    {
+      "epoch": 0.39627842866988283,
+      "grad_norm": 7.79961258578142,
+      "learning_rate": 4.8547808184559225e-08,
+      "logits/chosen": -3.5804367065429688,
+      "logits/rejected": -3.561884641647339,
+      "logps/chosen": -1.5013697147369385,
+      "logps/rejected": -1.6763207912445068,
+      "loss": 1.1048,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.5013697147369385,
+      "rewards/margins": 0.1749512404203415,
+      "rewards/rejected": -1.6763207912445068,
+      "step": 2300
+    },
+    {
+      "epoch": 0.39627842866988283,
+      "eval_logits/chosen": -3.622032642364502,
+      "eval_logits/rejected": -3.6180856227874756,
+      "eval_logps/chosen": -1.4894847869873047,
+      "eval_logps/rejected": -1.6241532564163208,
+      "eval_loss": 1.113089919090271,
+      "eval_rewards/accuracies": 0.6050186157226562,
+      "eval_rewards/chosen": -1.4894847869873047,
+      "eval_rewards/margins": 0.13466857373714447,
+      "eval_rewards/rejected": -1.6241532564163208,
+      "eval_runtime": 155.6502,
+      "eval_samples_per_second": 27.652,
+      "eval_steps_per_second": 3.456,
+      "step": 2300
+    },
+    {
+      "epoch": 0.3980013783597519,
+      "grad_norm": 10.537640392014017,
+      "learning_rate": 4.852245211648297e-08,
+      "logits/chosen": -3.523688554763794,
+      "logits/rejected": -3.514939785003662,
+      "logps/chosen": -1.517511010169983,
+      "logps/rejected": -1.6607134342193604,
+      "loss": 1.1215,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.517511010169983,
+      "rewards/margins": 0.14320224523544312,
+      "rewards/rejected": -1.6607134342193604,
+      "step": 2310
+    },
+    {
+      "epoch": 0.39972432804962094,
+      "grad_norm": 8.465004845221461,
+      "learning_rate": 4.8496883333061044e-08,
+      "logits/chosen": -3.577125072479248,
+      "logits/rejected": -3.5522549152374268,
+      "logps/chosen": -1.5531445741653442,
+      "logps/rejected": -1.664940595626831,
+      "loss": 1.1413,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.5531445741653442,
+      "rewards/margins": 0.11179597675800323,
+      "rewards/rejected": -1.664940595626831,
+      "step": 2320
+    },
+    {
+      "epoch": 0.40144727773949,
+      "grad_norm": 9.295851687216517,
+      "learning_rate": 4.8471102065513926e-08,
+      "logits/chosen": -3.526211977005005,
+      "logits/rejected": -3.5108699798583984,
+      "logps/chosen": -1.5409572124481201,
+      "logps/rejected": -1.6939599514007568,
+      "loss": 1.1281,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.5409572124481201,
+      "rewards/margins": 0.1530027836561203,
+      "rewards/rejected": -1.6939599514007568,
+      "step": 2330
+    },
+    {
+      "epoch": 0.40317022742935904,
+      "grad_norm": 8.03887459203861,
+      "learning_rate": 4.844510854698359e-08,
+      "logits/chosen": -3.555849552154541,
+      "logits/rejected": -3.5484778881073,
+      "logps/chosen": -1.5233571529388428,
+      "logps/rejected": -1.7341830730438232,
+      "loss": 1.0824,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.5233571529388428,
+      "rewards/margins": 0.2108260691165924,
+      "rewards/rejected": -1.7341830730438232,
+      "step": 2340
+    },
+    {
+      "epoch": 0.4048931771192281,
+      "grad_norm": 11.339067252220723,
+      "learning_rate": 4.841890301253144e-08,
+      "logits/chosen": -3.5472264289855957,
+      "logits/rejected": -3.5310206413269043,
+      "logps/chosen": -1.5976260900497437,
+      "logps/rejected": -1.7632135152816772,
+      "loss": 1.1336,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.5976260900497437,
+      "rewards/margins": 0.16558733582496643,
+      "rewards/rejected": -1.7632135152816772,
+      "step": 2350
+    },
+    {
+      "epoch": 0.4066161268090972,
+      "grad_norm": 9.690220492326478,
+      "learning_rate": 4.8392485699136144e-08,
+      "logits/chosen": -3.554180860519409,
+      "logits/rejected": -3.546344041824341,
+      "logps/chosen": -1.4996538162231445,
+      "logps/rejected": -1.6898291110992432,
+      "loss": 1.091,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.4996538162231445,
+      "rewards/margins": 0.1901753693819046,
+      "rewards/rejected": -1.6898291110992432,
+      "step": 2360
+    },
+    {
+      "epoch": 0.4083390764989662,
+      "grad_norm": 9.877557246730623,
+      "learning_rate": 4.836585684569148e-08,
+      "logits/chosen": -3.5623927116394043,
+      "logits/rejected": -3.5597338676452637,
+      "logps/chosen": -1.6194093227386475,
+      "logps/rejected": -1.7034022808074951,
+      "loss": 1.1596,
+      "rewards/accuracies": 0.4937500059604645,
+      "rewards/chosen": -1.6194093227386475,
+      "rewards/margins": 0.08399289101362228,
+      "rewards/rejected": -1.7034022808074951,
+      "step": 2370
+    },
+    {
+      "epoch": 0.4100620261888353,
+      "grad_norm": 8.527743135850782,
+      "learning_rate": 4.833901669300424e-08,
+      "logits/chosen": -3.503612518310547,
+      "logits/rejected": -3.487412929534912,
+      "logps/chosen": -1.528314471244812,
+      "logps/rejected": -1.6493940353393555,
+      "loss": 1.1381,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.528314471244812,
+      "rewards/margins": 0.1210794672369957,
+      "rewards/rejected": -1.6493940353393555,
+      "step": 2380
+    },
+    {
+      "epoch": 0.41178497587870433,
+      "grad_norm": 8.174971320109574,
+      "learning_rate": 4.831196548379198e-08,
+      "logits/chosen": -3.5644404888153076,
+      "logits/rejected": -3.548340320587158,
+      "logps/chosen": -1.5747714042663574,
+      "logps/rejected": -1.7084159851074219,
+      "loss": 1.1319,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.5747714042663574,
+      "rewards/margins": 0.13364450633525848,
+      "rewards/rejected": -1.7084159851074219,
+      "step": 2390
+    },
+    {
+      "epoch": 0.4135079255685734,
+      "grad_norm": 8.362284723647345,
+      "learning_rate": 4.828470346268089e-08,
+      "logits/chosen": -3.5890889167785645,
+      "logits/rejected": -3.574572801589966,
+      "logps/chosen": -1.461393117904663,
+      "logps/rejected": -1.708251714706421,
+      "loss": 1.0478,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.461393117904663,
+      "rewards/margins": 0.24685856699943542,
+      "rewards/rejected": -1.708251714706421,
+      "step": 2400
+    },
+    {
+      "epoch": 0.4135079255685734,
+      "eval_logits/chosen": -3.606722354888916,
+      "eval_logits/rejected": -3.6027328968048096,
+      "eval_logps/chosen": -1.4915759563446045,
+      "eval_logps/rejected": -1.6270511150360107,
+      "eval_loss": 1.1126437187194824,
+      "eval_rewards/accuracies": 0.6059479713439941,
+      "eval_rewards/chosen": -1.4915759563446045,
+      "eval_rewards/margins": 0.13547508418560028,
+      "eval_rewards/rejected": -1.6270511150360107,
+      "eval_runtime": 155.503,
+      "eval_samples_per_second": 27.678,
+      "eval_steps_per_second": 3.46,
+      "step": 2400
+    },
+    {
+      "epoch": 0.41523087525844243,
+      "grad_norm": 9.643372560756209,
+      "learning_rate": 4.825723087620349e-08,
+      "logits/chosen": -3.5794949531555176,
+      "logits/rejected": -3.555523633956909,
+      "logps/chosen": -1.4982608556747437,
+      "logps/rejected": -1.690784215927124,
+      "loss": 1.0958,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.4982608556747437,
+      "rewards/margins": 0.19252346456050873,
+      "rewards/rejected": -1.690784215927124,
+      "step": 2410
+    },
+    {
+      "epoch": 0.4169538249483115,
+      "grad_norm": 9.170063772751305,
+      "learning_rate": 4.822954797279652e-08,
+      "logits/chosen": -3.5759644508361816,
+      "logits/rejected": -3.5660147666931152,
+      "logps/chosen": -1.5427138805389404,
+      "logps/rejected": -1.7172151803970337,
+      "loss": 1.1037,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.5427138805389404,
+      "rewards/margins": 0.17450109124183655,
+      "rewards/rejected": -1.7172151803970337,
+      "step": 2420
+    },
+    {
+      "epoch": 0.41867677463818054,
+      "grad_norm": 8.185152096011437,
+      "learning_rate": 4.82016550027986e-08,
+      "logits/chosen": -3.5652804374694824,
+      "logits/rejected": -3.551367998123169,
+      "logps/chosen": -1.5138072967529297,
+      "logps/rejected": -1.7130963802337646,
+      "loss": 1.1126,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -1.5138072967529297,
+      "rewards/margins": 0.19928905367851257,
+      "rewards/rejected": -1.7130963802337646,
+      "step": 2430
+    },
+    {
+      "epoch": 0.4203997243280496,
+      "grad_norm": 9.898286928447718,
+      "learning_rate": 4.817355221844801e-08,
+      "logits/chosen": -3.5509421825408936,
+      "logits/rejected": -3.54337739944458,
+      "logps/chosen": -1.5814521312713623,
+      "logps/rejected": -1.6955993175506592,
+      "loss": 1.1473,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.5814521312713623,
+      "rewards/margins": 0.11414710432291031,
+      "rewards/rejected": -1.6955993175506592,
+      "step": 2440
+    },
+    {
+      "epoch": 0.4221226740179187,
+      "grad_norm": 7.978367879816051,
+      "learning_rate": 4.814523987388038e-08,
+      "logits/chosen": -3.52856707572937,
+      "logits/rejected": -3.5154213905334473,
+      "logps/chosen": -1.5860412120819092,
+      "logps/rejected": -1.7607500553131104,
+      "loss": 1.1128,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.5860412120819092,
+      "rewards/margins": 0.17470884323120117,
+      "rewards/rejected": -1.7607500553131104,
+      "step": 2450
+    },
+    {
+      "epoch": 0.4238456237077877,
+      "grad_norm": 9.180050057867438,
+      "learning_rate": 4.811671822512644e-08,
+      "logits/chosen": -3.5011277198791504,
+      "logits/rejected": -3.4836068153381348,
+      "logps/chosen": -1.6185615062713623,
+      "logps/rejected": -1.7135651111602783,
+      "loss": 1.1467,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.6185615062713623,
+      "rewards/margins": 0.09500333666801453,
+      "rewards/rejected": -1.7135651111602783,
+      "step": 2460
+    },
+    {
+      "epoch": 0.4255685733976568,
+      "grad_norm": 8.929188673077972,
+      "learning_rate": 4.808798753010965e-08,
+      "logits/chosen": -3.611974000930786,
+      "logits/rejected": -3.5994460582733154,
+      "logps/chosen": -1.5093005895614624,
+      "logps/rejected": -1.619179368019104,
+      "loss": 1.1463,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -1.5093005895614624,
+      "rewards/margins": 0.10987883806228638,
+      "rewards/rejected": -1.619179368019104,
+      "step": 2470
+    },
+    {
+      "epoch": 0.4272915230875258,
+      "grad_norm": 9.150319050926155,
+      "learning_rate": 4.805904804864389e-08,
+      "logits/chosen": -3.5479702949523926,
+      "logits/rejected": -3.530494213104248,
+      "logps/chosen": -1.5695327520370483,
+      "logps/rejected": -1.6769025325775146,
+      "loss": 1.1447,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -1.5695327520370483,
+      "rewards/margins": 0.1073700338602066,
+      "rewards/rejected": -1.6769025325775146,
+      "step": 2480
+    },
+    {
+      "epoch": 0.4290144727773949,
+      "grad_norm": 9.29885320419482,
+      "learning_rate": 4.802990004243112e-08,
+      "logits/chosen": -3.5348925590515137,
+      "logits/rejected": -3.5301742553710938,
+      "logps/chosen": -1.5420172214508057,
+      "logps/rejected": -1.6663042306900024,
+      "loss": 1.1335,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.5420172214508057,
+      "rewards/margins": 0.12428691238164902,
+      "rewards/rejected": -1.6663042306900024,
+      "step": 2490
+    },
+    {
+      "epoch": 0.43073742246726393,
+      "grad_norm": 10.349625231719559,
+      "learning_rate": 4.800054377505901e-08,
+      "logits/chosen": -3.5921034812927246,
+      "logits/rejected": -3.578320264816284,
+      "logps/chosen": -1.589381456375122,
+      "logps/rejected": -1.7116740942001343,
+      "loss": 1.1417,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -1.589381456375122,
+      "rewards/margins": 0.12229261547327042,
+      "rewards/rejected": -1.7116740942001343,
+      "step": 2500
+    },
+    {
+      "epoch": 0.43073742246726393,
+      "eval_logits/chosen": -3.6044397354125977,
+      "eval_logits/rejected": -3.6004550457000732,
+      "eval_logps/chosen": -1.4939520359039307,
+      "eval_logps/rejected": -1.6305620670318604,
+      "eval_loss": 1.1120483875274658,
+      "eval_rewards/accuracies": 0.6064126491546631,
+      "eval_rewards/chosen": -1.4939520359039307,
+      "eval_rewards/margins": 0.13660991191864014,
+      "eval_rewards/rejected": -1.6305620670318604,
+      "eval_runtime": 155.7376,
+      "eval_samples_per_second": 27.636,
+      "eval_steps_per_second": 3.455,
+      "step": 2500
+    },
+    {
+      "epoch": 0.432460372157133,
+      "grad_norm": 9.167241228051012,
+      "learning_rate": 4.797097951199854e-08,
+      "logits/chosen": -3.49882435798645,
+      "logits/rejected": -3.4927573204040527,
+      "logps/chosen": -1.610544204711914,
+      "logps/rejected": -1.6926467418670654,
+      "loss": 1.1705,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.610544204711914,
+      "rewards/margins": 0.08210258185863495,
+      "rewards/rejected": -1.6926467418670654,
+      "step": 2510
+    },
+    {
+      "epoch": 0.4341833218470021,
+      "grad_norm": 9.225665549082622,
+      "learning_rate": 4.7941207520601625e-08,
+      "logits/chosen": -3.488086223602295,
+      "logits/rejected": -3.471892833709717,
+      "logps/chosen": -1.4622722864151,
+      "logps/rejected": -1.6004787683486938,
+      "loss": 1.1209,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.4622722864151,
+      "rewards/margins": 0.1382065713405609,
+      "rewards/rejected": -1.6004787683486938,
+      "step": 2520
+    },
+    {
+      "epoch": 0.4359062715368711,
+      "grad_norm": 9.713039778216356,
+      "learning_rate": 4.791122807009867e-08,
+      "logits/chosen": -3.538599729537964,
+      "logits/rejected": -3.536027193069458,
+      "logps/chosen": -1.5306090116500854,
+      "logps/rejected": -1.6774742603302002,
+      "loss": 1.1162,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.5306090116500854,
+      "rewards/margins": 0.14686518907546997,
+      "rewards/rejected": -1.6774742603302002,
+      "step": 2530
+    },
+    {
+      "epoch": 0.4376292212267402,
+      "grad_norm": 10.715519456357946,
+      "learning_rate": 4.7881041431596156e-08,
+      "logits/chosen": -3.5673413276672363,
+      "logits/rejected": -3.5627620220184326,
+      "logps/chosen": -1.63469660282135,
+      "logps/rejected": -1.7306814193725586,
+      "loss": 1.1515,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -1.63469660282135,
+      "rewards/margins": 0.09598477184772491,
+      "rewards/rejected": -1.7306814193725586,
+      "step": 2540
+    },
+    {
+      "epoch": 0.4393521709166092,
+      "grad_norm": 9.42880211947525,
+      "learning_rate": 4.7850647878074176e-08,
+      "logits/chosen": -3.5225281715393066,
+      "logits/rejected": -3.5047905445098877,
+      "logps/chosen": -1.5407750606536865,
+      "logps/rejected": -1.672753930091858,
+      "loss": 1.1277,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.5407750606536865,
+      "rewards/margins": 0.13197895884513855,
+      "rewards/rejected": -1.672753930091858,
+      "step": 2550
+    },
+    {
+      "epoch": 0.4410751206064783,
+      "grad_norm": 8.932950435382823,
+      "learning_rate": 4.782004768438399e-08,
+      "logits/chosen": -3.576164960861206,
+      "logits/rejected": -3.5630173683166504,
+      "logps/chosen": -1.5771715641021729,
+      "logps/rejected": -1.725873589515686,
+      "loss": 1.1309,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.5771715641021729,
+      "rewards/margins": 0.14870189130306244,
+      "rewards/rejected": -1.725873589515686,
+      "step": 2560
+    },
+    {
+      "epoch": 0.4427980702963473,
+      "grad_norm": 8.42942512101924,
+      "learning_rate": 4.7789241127245484e-08,
+      "logits/chosen": -3.543442487716675,
+      "logits/rejected": -3.5313918590545654,
+      "logps/chosen": -1.5779088735580444,
+      "logps/rejected": -1.7364460229873657,
+      "loss": 1.1343,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -1.5779088735580444,
+      "rewards/margins": 0.158536896109581,
+      "rewards/rejected": -1.7364460229873657,
+      "step": 2570
+    },
+    {
+      "epoch": 0.4445210199862164,
+      "grad_norm": 10.05555306633659,
+      "learning_rate": 4.775822848524473e-08,
+      "logits/chosen": -3.5564637184143066,
+      "logits/rejected": -3.545177936553955,
+      "logps/chosen": -1.6046082973480225,
+      "logps/rejected": -1.7665237188339233,
+      "loss": 1.1379,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.6046082973480225,
+      "rewards/margins": 0.16191527247428894,
+      "rewards/rejected": -1.7665237188339233,
+      "step": 2580
+    },
+    {
+      "epoch": 0.4462439696760855,
+      "grad_norm": 11.23778816329687,
+      "learning_rate": 4.7727010038831456e-08,
+      "logits/chosen": -3.5496420860290527,
+      "logits/rejected": -3.5278515815734863,
+      "logps/chosen": -1.601085901260376,
+      "logps/rejected": -1.765232801437378,
+      "loss": 1.1255,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.601085901260376,
+      "rewards/margins": 0.16414692997932434,
+      "rewards/rejected": -1.765232801437378,
+      "step": 2590
+    },
+    {
+      "epoch": 0.4479669193659545,
+      "grad_norm": 11.954932448221802,
+      "learning_rate": 4.769558607031646e-08,
+      "logits/chosen": -3.514209032058716,
+      "logits/rejected": -3.484711170196533,
+      "logps/chosen": -1.5878870487213135,
+      "logps/rejected": -1.7463195323944092,
+      "loss": 1.1261,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -1.5878870487213135,
+      "rewards/margins": 0.15843269228935242,
+      "rewards/rejected": -1.7463195323944092,
+      "step": 2600
+    },
+    {
+      "epoch": 0.4479669193659545,
+      "eval_logits/chosen": -3.591796636581421,
+      "eval_logits/rejected": -3.5877723693847656,
+      "eval_logps/chosen": -1.4969524145126343,
+      "eval_logps/rejected": -1.634235143661499,
+      "eval_loss": 1.111628770828247,
+      "eval_rewards/accuracies": 0.6078066825866699,
+      "eval_rewards/chosen": -1.4969524145126343,
+      "eval_rewards/margins": 0.1372828483581543,
+      "eval_rewards/rejected": -1.634235143661499,
+      "eval_runtime": 155.5009,
+      "eval_samples_per_second": 27.678,
+      "eval_steps_per_second": 3.46,
+      "step": 2600
+    },
+    {
+      "epoch": 0.4496898690558236,
+      "grad_norm": 9.079473906604631,
+      "learning_rate": 4.766395686386911e-08,
+      "logits/chosen": -3.506345272064209,
+      "logits/rejected": -3.4826302528381348,
+      "logps/chosen": -1.5740907192230225,
+      "logps/rejected": -1.7391849756240845,
+      "loss": 1.1347,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.5740907192230225,
+      "rewards/margins": 0.16509395837783813,
+      "rewards/rejected": -1.7391849756240845,
+      "step": 2610
+    },
+    {
+      "epoch": 0.4514128187456926,
+      "grad_norm": 7.834017534572625,
+      "learning_rate": 4.7632122705514764e-08,
+      "logits/chosen": -3.5797336101531982,
+      "logits/rejected": -3.5617835521698,
+      "logps/chosen": -1.5660189390182495,
+      "logps/rejected": -1.7695693969726562,
+      "loss": 1.1013,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.5660189390182495,
+      "rewards/margins": 0.20355041325092316,
+      "rewards/rejected": -1.7695693969726562,
+      "step": 2620
+    },
+    {
+      "epoch": 0.4531357684355617,
+      "grad_norm": 10.99190491262731,
+      "learning_rate": 4.760008388313216e-08,
+      "logits/chosen": -3.5022544860839844,
+      "logits/rejected": -3.482717990875244,
+      "logps/chosen": -1.5797499418258667,
+      "logps/rejected": -1.7831165790557861,
+      "loss": 1.091,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.5797499418258667,
+      "rewards/margins": 0.20336654782295227,
+      "rewards/rejected": -1.7831165790557861,
+      "step": 2630
+    },
+    {
+      "epoch": 0.4548587181254307,
+      "grad_norm": 9.878556342776122,
+      "learning_rate": 4.7567840686450835e-08,
+      "logits/chosen": -3.516876697540283,
+      "logits/rejected": -3.502979278564453,
+      "logps/chosen": -1.493472933769226,
+      "logps/rejected": -1.6501750946044922,
+      "loss": 1.1131,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.493472933769226,
+      "rewards/margins": 0.1567022055387497,
+      "rewards/rejected": -1.6501750946044922,
+      "step": 2640
+    },
+    {
+      "epoch": 0.4565816678152998,
+      "grad_norm": 8.418231076704377,
+      "learning_rate": 4.7535393407048503e-08,
+      "logits/chosen": -3.499467134475708,
+      "logits/rejected": -3.48779034614563,
+      "logps/chosen": -1.5018670558929443,
+      "logps/rejected": -1.6948683261871338,
+      "loss": 1.0982,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.5018670558929443,
+      "rewards/margins": 0.19300125539302826,
+      "rewards/rejected": -1.6948683261871338,
+      "step": 2650
+    },
+    {
+      "epoch": 0.4583046175051689,
+      "grad_norm": 8.907303540914612,
+      "learning_rate": 4.7502742338348405e-08,
+      "logits/chosen": -3.509742021560669,
+      "logits/rejected": -3.4853763580322266,
+      "logps/chosen": -1.614696741104126,
+      "logps/rejected": -1.8692476749420166,
+      "loss": 1.0703,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.614696741104126,
+      "rewards/margins": 0.25455087423324585,
+      "rewards/rejected": -1.8692476749420166,
+      "step": 2660
+    },
+    {
+      "epoch": 0.4600275671950379,
+      "grad_norm": 9.823704369127581,
+      "learning_rate": 4.746988777561668e-08,
+      "logits/chosen": -3.487753391265869,
+      "logits/rejected": -3.4710288047790527,
+      "logps/chosen": -1.5321165323257446,
+      "logps/rejected": -1.7221482992172241,
+      "loss": 1.11,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.5321165323257446,
+      "rewards/margins": 0.19003185629844666,
+      "rewards/rejected": -1.7221482992172241,
+      "step": 2670
+    },
+    {
+      "epoch": 0.461750516884907,
+      "grad_norm": 10.950191372029837,
+      "learning_rate": 4.743683001595965e-08,
+      "logits/chosen": -3.5161125659942627,
+      "logits/rejected": -3.5013115406036377,
+      "logps/chosen": -1.6431413888931274,
+      "logps/rejected": -1.7961227893829346,
+      "loss": 1.1384,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -1.6431413888931274,
+      "rewards/margins": 0.15298131108283997,
+      "rewards/rejected": -1.7961227893829346,
+      "step": 2680
+    },
+    {
+      "epoch": 0.463473466574776,
+      "grad_norm": 7.4486839920447965,
+      "learning_rate": 4.7403569358321205e-08,
+      "logits/chosen": -3.5092556476593018,
+      "logits/rejected": -3.494269847869873,
+      "logps/chosen": -1.5307735204696655,
+      "logps/rejected": -1.8265111446380615,
+      "loss": 1.0388,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.5307735204696655,
+      "rewards/margins": 0.2957378327846527,
+      "rewards/rejected": -1.8265111446380615,
+      "step": 2690
+    },
+    {
+      "epoch": 0.4651964162646451,
+      "grad_norm": 9.31668965827334,
+      "learning_rate": 4.737010610348001e-08,
+      "logits/chosen": -3.527475357055664,
+      "logits/rejected": -3.5133652687072754,
+      "logps/chosen": -1.4935544729232788,
+      "logps/rejected": -1.7019037008285522,
+      "loss": 1.0752,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.4935544729232788,
+      "rewards/margins": 0.20834927260875702,
+      "rewards/rejected": -1.7019037008285522,
+      "step": 2700
+    },
+    {
+      "epoch": 0.4651964162646451,
+      "eval_logits/chosen": -3.5942909717559814,
+      "eval_logits/rejected": -3.5903093814849854,
+      "eval_logps/chosen": -1.5008198022842407,
+      "eval_logps/rejected": -1.639434814453125,
+      "eval_loss": 1.1108793020248413,
+      "eval_rewards/accuracies": 0.6126858592033386,
+      "eval_rewards/chosen": -1.5008198022842407,
+      "eval_rewards/margins": 0.1386152058839798,
+      "eval_rewards/rejected": -1.639434814453125,
+      "eval_runtime": 155.7605,
+      "eval_samples_per_second": 27.632,
+      "eval_steps_per_second": 3.454,
+      "step": 2700
+    },
+    {
+      "epoch": 0.4669193659545141,
+      "grad_norm": 10.454706908409717,
+      "learning_rate": 4.733644055404687e-08,
+      "logits/chosen": -3.530062198638916,
+      "logits/rejected": -3.520596742630005,
+      "logps/chosen": -1.5897412300109863,
+      "logps/rejected": -1.6460819244384766,
+      "loss": 1.1837,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -1.5897412300109863,
+      "rewards/margins": 0.05634066462516785,
+      "rewards/rejected": -1.6460819244384766,
+      "step": 2710
+    },
+    {
+      "epoch": 0.4686423156443832,
+      "grad_norm": 8.782148354694232,
+      "learning_rate": 4.730257301446193e-08,
+      "logits/chosen": -3.5335426330566406,
+      "logits/rejected": -3.5338597297668457,
+      "logps/chosen": -1.5679161548614502,
+      "logps/rejected": -1.685532808303833,
+      "loss": 1.1409,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -1.5679161548614502,
+      "rewards/margins": 0.11761681735515594,
+      "rewards/rejected": -1.685532808303833,
+      "step": 2720
+    },
+    {
+      "epoch": 0.4703652653342522,
+      "grad_norm": 8.204338801774556,
+      "learning_rate": 4.726850379099198e-08,
+      "logits/chosen": -3.539498805999756,
+      "logits/rejected": -3.528214693069458,
+      "logps/chosen": -1.5829863548278809,
+      "logps/rejected": -1.7656309604644775,
+      "loss": 1.111,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.5829863548278809,
+      "rewards/margins": 0.1826445311307907,
+      "rewards/rejected": -1.7656309604644775,
+      "step": 2730
+    },
+    {
+      "epoch": 0.4720882150241213,
+      "grad_norm": 11.648072160673312,
+      "learning_rate": 4.7234233191727604e-08,
+      "logits/chosen": -3.533815860748291,
+      "logits/rejected": -3.520245313644409,
+      "logps/chosen": -1.5599262714385986,
+      "logps/rejected": -1.6004273891448975,
+      "loss": 1.194,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.5599262714385986,
+      "rewards/margins": 0.04050111770629883,
+      "rewards/rejected": -1.6004273891448975,
+      "step": 2740
+    },
+    {
+      "epoch": 0.4738111647139904,
+      "grad_norm": 8.113191878307573,
+      "learning_rate": 4.7199761526580484e-08,
+      "logits/chosen": -3.4874165058135986,
+      "logits/rejected": -3.4825472831726074,
+      "logps/chosen": -1.5572025775909424,
+      "logps/rejected": -1.7300984859466553,
+      "loss": 1.1096,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.5572025775909424,
+      "rewards/margins": 0.172896146774292,
+      "rewards/rejected": -1.7300984859466553,
+      "step": 2750
+    },
+    {
+      "epoch": 0.4755341144038594,
+      "grad_norm": 10.533366153181937,
+      "learning_rate": 4.716508910728054e-08,
+      "logits/chosen": -3.520174026489258,
+      "logits/rejected": -3.507814884185791,
+      "logps/chosen": -1.5909929275512695,
+      "logps/rejected": -1.768484354019165,
+      "loss": 1.1069,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -1.5909929275512695,
+      "rewards/margins": 0.17749133706092834,
+      "rewards/rejected": -1.768484354019165,
+      "step": 2760
+    },
+    {
+      "epoch": 0.4772570640937285,
+      "grad_norm": 9.338795487581447,
+      "learning_rate": 4.713021624737312e-08,
+      "logits/chosen": -3.5973358154296875,
+      "logits/rejected": -3.582235813140869,
+      "logps/chosen": -1.5345796346664429,
+      "logps/rejected": -1.6543534994125366,
+      "loss": 1.1326,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.5345796346664429,
+      "rewards/margins": 0.1197737455368042,
+      "rewards/rejected": -1.6543534994125366,
+      "step": 2770
+    },
+    {
+      "epoch": 0.4789800137835975,
+      "grad_norm": 11.173500991281108,
+      "learning_rate": 4.70951432622162e-08,
+      "logits/chosen": -3.493271589279175,
+      "logits/rejected": -3.475271701812744,
+      "logps/chosen": -1.5888370275497437,
+      "logps/rejected": -1.788230538368225,
+      "loss": 1.0938,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.5888370275497437,
+      "rewards/margins": 0.19939354062080383,
+      "rewards/rejected": -1.788230538368225,
+      "step": 2780
+    },
+    {
+      "epoch": 0.4807029634734666,
+      "grad_norm": 9.207293494633715,
+      "learning_rate": 4.7059870468977484e-08,
+      "logits/chosen": -3.5631237030029297,
+      "logits/rejected": -3.5430006980895996,
+      "logps/chosen": -1.475109338760376,
+      "logps/rejected": -1.6650257110595703,
+      "loss": 1.0918,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.475109338760376,
+      "rewards/margins": 0.18991632759571075,
+      "rewards/rejected": -1.6650257110595703,
+      "step": 2790
+    },
+    {
+      "epoch": 0.4824259131633356,
+      "grad_norm": 9.136998401292782,
+      "learning_rate": 4.702439818663153e-08,
+      "logits/chosen": -3.5423080921173096,
+      "logits/rejected": -3.5291247367858887,
+      "logps/chosen": -1.6715797185897827,
+      "logps/rejected": -1.7642061710357666,
+      "loss": 1.1623,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -1.6715797185897827,
+      "rewards/margins": 0.09262636303901672,
+      "rewards/rejected": -1.7642061710357666,
+      "step": 2800
+    },
+    {
+      "epoch": 0.4824259131633356,
+      "eval_logits/chosen": -3.6040496826171875,
+      "eval_logits/rejected": -3.600142240524292,
+      "eval_logps/chosen": -1.5047415494918823,
+      "eval_logps/rejected": -1.6439995765686035,
+      "eval_loss": 1.110492467880249,
+      "eval_rewards/accuracies": 0.6112918257713318,
+      "eval_rewards/chosen": -1.5047415494918823,
+      "eval_rewards/margins": 0.13925811648368835,
+      "eval_rewards/rejected": -1.6439995765686035,
+      "eval_runtime": 155.3793,
+      "eval_samples_per_second": 27.7,
+      "eval_steps_per_second": 3.462,
+      "step": 2800
+    },
+    {
+      "epoch": 0.4841488628532047,
+      "grad_norm": 9.631767601249107,
+      "learning_rate": 4.6988726735956954e-08,
+      "logits/chosen": -3.4731974601745605,
+      "logits/rejected": -3.459533214569092,
+      "logps/chosen": -1.5727657079696655,
+      "logps/rejected": -1.730197548866272,
+      "loss": 1.1052,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.5727657079696655,
+      "rewards/margins": 0.15743179619312286,
+      "rewards/rejected": -1.730197548866272,
+      "step": 2810
+    },
+    {
+      "epoch": 0.48587181254307377,
+      "grad_norm": 9.414261595105659,
+      "learning_rate": 4.69528564395334e-08,
+      "logits/chosen": -3.567905902862549,
+      "logits/rejected": -3.561178684234619,
+      "logps/chosen": -1.610030174255371,
+      "logps/rejected": -1.7266442775726318,
+      "loss": 1.1563,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -1.610030174255371,
+      "rewards/margins": 0.11661408096551895,
+      "rewards/rejected": -1.7266442775726318,
+      "step": 2820
+    },
+    {
+      "epoch": 0.4875947622329428,
+      "grad_norm": 7.599277893231088,
+      "learning_rate": 4.691678762173874e-08,
+      "logits/chosen": -3.4506072998046875,
+      "logits/rejected": -3.441523313522339,
+      "logps/chosen": -1.5950828790664673,
+      "logps/rejected": -1.7152217626571655,
+      "loss": 1.1379,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.5950828790664673,
+      "rewards/margins": 0.120138980448246,
+      "rewards/rejected": -1.7152217626571655,
+      "step": 2830
+    },
+    {
+      "epoch": 0.48931771192281187,
+      "grad_norm": 10.384438850686081,
+      "learning_rate": 4.688052060874606e-08,
+      "logits/chosen": -3.5529301166534424,
+      "logits/rejected": -3.546264171600342,
+      "logps/chosen": -1.64028000831604,
+      "logps/rejected": -1.7011635303497314,
+      "loss": 1.1793,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -1.64028000831604,
+      "rewards/margins": 0.06088363006711006,
+      "rewards/rejected": -1.7011635303497314,
+      "step": 2840
+    },
+    {
+      "epoch": 0.4910406616126809,
+      "grad_norm": 8.386005088847273,
+      "learning_rate": 4.684405572852077e-08,
+      "logits/chosen": -3.531562089920044,
+      "logits/rejected": -3.5197646617889404,
+      "logps/chosen": -1.5464965105056763,
+      "logps/rejected": -1.851794958114624,
+      "loss": 1.0343,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.5464965105056763,
+      "rewards/margins": 0.30529865622520447,
+      "rewards/rejected": -1.851794958114624,
+      "step": 2850
+    },
+    {
+      "epoch": 0.49276361130255,
+      "grad_norm": 9.562457155586396,
+      "learning_rate": 4.6807393310817575e-08,
+      "logits/chosen": -3.515758991241455,
+      "logits/rejected": -3.502673387527466,
+      "logps/chosen": -1.5639783143997192,
+      "logps/rejected": -1.680419921875,
+      "loss": 1.1416,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.5639783143997192,
+      "rewards/margins": 0.11644172668457031,
+      "rewards/rejected": -1.680419921875,
+      "step": 2860
+    },
+    {
+      "epoch": 0.494486560992419,
+      "grad_norm": 9.001282439734755,
+      "learning_rate": 4.677053368717754e-08,
+      "logits/chosen": -3.542102098464966,
+      "logits/rejected": -3.527468204498291,
+      "logps/chosen": -1.5502660274505615,
+      "logps/rejected": -1.7253986597061157,
+      "loss": 1.0994,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.5502660274505615,
+      "rewards/margins": 0.17513257265090942,
+      "rewards/rejected": -1.7253986597061157,
+      "step": 2870
+    },
+    {
+      "epoch": 0.4962095106822881,
+      "grad_norm": 11.827834093584565,
+      "learning_rate": 4.673347719092507e-08,
+      "logits/chosen": -3.5854382514953613,
+      "logits/rejected": -3.5715584754943848,
+      "logps/chosen": -1.6331123113632202,
+      "logps/rejected": -1.7330087423324585,
+      "loss": 1.1415,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.6331123113632202,
+      "rewards/margins": 0.09989641606807709,
+      "rewards/rejected": -1.7330087423324585,
+      "step": 2880
+    },
+    {
+      "epoch": 0.49793246037215716,
+      "grad_norm": 8.031944852199219,
+      "learning_rate": 4.669622415716494e-08,
+      "logits/chosen": -3.59136962890625,
+      "logits/rejected": -3.588791608810425,
+      "logps/chosen": -1.6095445156097412,
+      "logps/rejected": -1.7277612686157227,
+      "loss": 1.1475,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.6095445156097412,
+      "rewards/margins": 0.11821679770946503,
+      "rewards/rejected": -1.7277612686157227,
+      "step": 2890
+    },
+    {
+      "epoch": 0.4996554100620262,
+      "grad_norm": 10.823849788058101,
+      "learning_rate": 4.665877492277919e-08,
+      "logits/chosen": -3.5467333793640137,
+      "logits/rejected": -3.541468858718872,
+      "logps/chosen": -1.5395801067352295,
+      "logps/rejected": -1.5993454456329346,
+      "loss": 1.1744,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -1.5395801067352295,
+      "rewards/margins": 0.059765420854091644,
+      "rewards/rejected": -1.5993454456329346,
+      "step": 2900
+    },
+    {
+      "epoch": 0.4996554100620262,
+      "eval_logits/chosen": -3.5994787216186523,
+      "eval_logits/rejected": -3.595571517944336,
+      "eval_logps/chosen": -1.5102325677871704,
+      "eval_logps/rejected": -1.6505348682403564,
+      "eval_loss": 1.1099823713302612,
+      "eval_rewards/accuracies": 0.6129181981086731,
+      "eval_rewards/chosen": -1.5102325677871704,
+      "eval_rewards/margins": 0.14030234515666962,
+      "eval_rewards/rejected": -1.6505348682403564,
+      "eval_runtime": 155.3708,
+      "eval_samples_per_second": 27.701,
+      "eval_steps_per_second": 3.463,
+      "step": 2900
+    },
+    {
+      "epoch": 0.5013783597518953,
+      "grad_norm": 10.47870833874534,
+      "learning_rate": 4.6621129826424115e-08,
+      "logits/chosen": -3.5669541358947754,
+      "logits/rejected": -3.5609488487243652,
+      "logps/chosen": -1.5903078317642212,
+      "logps/rejected": -1.7240289449691772,
+      "loss": 1.1236,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.5903078317642212,
+      "rewards/margins": 0.13372120261192322,
+      "rewards/rejected": -1.7240289449691772,
+      "step": 2910
+    },
+    {
+      "epoch": 0.5031013094417643,
+      "grad_norm": 9.393523885177366,
+      "learning_rate": 4.6583289208527247e-08,
+      "logits/chosen": -3.53071665763855,
+      "logits/rejected": -3.5292491912841797,
+      "logps/chosen": -1.590759515762329,
+      "logps/rejected": -1.7681719064712524,
+      "loss": 1.1027,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.590759515762329,
+      "rewards/margins": 0.17741231620311737,
+      "rewards/rejected": -1.7681719064712524,
+      "step": 2920
+    },
+    {
+      "epoch": 0.5048242591316333,
+      "grad_norm": 9.083637642805446,
+      "learning_rate": 4.654525341128418e-08,
+      "logits/chosen": -3.5194294452667236,
+      "logits/rejected": -3.5033416748046875,
+      "logps/chosen": -1.5137364864349365,
+      "logps/rejected": -1.7660770416259766,
+      "loss": 1.0769,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.5137364864349365,
+      "rewards/margins": 0.25234049558639526,
+      "rewards/rejected": -1.7660770416259766,
+      "step": 2930
+    },
+    {
+      "epoch": 0.5065472088215024,
+      "grad_norm": 9.105246109382973,
+      "learning_rate": 4.650702277865558e-08,
+      "logits/chosen": -3.5272281169891357,
+      "logits/rejected": -3.5068085193634033,
+      "logps/chosen": -1.5123523473739624,
+      "logps/rejected": -1.782472014427185,
+      "loss": 1.0602,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.5123523473739624,
+      "rewards/margins": 0.27011972665786743,
+      "rewards/rejected": -1.782472014427185,
+      "step": 2940
+    },
+    {
+      "epoch": 0.5082701585113715,
+      "grad_norm": 8.74877181000663,
+      "learning_rate": 4.6468597656363996e-08,
+      "logits/chosen": -3.555302381515503,
+      "logits/rejected": -3.5444374084472656,
+      "logps/chosen": -1.5981149673461914,
+      "logps/rejected": -1.7523514032363892,
+      "loss": 1.1254,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.5981149673461914,
+      "rewards/margins": 0.15423637628555298,
+      "rewards/rejected": -1.7523514032363892,
+      "step": 2950
+    },
+    {
+      "epoch": 0.5099931082012406,
+      "grad_norm": 10.322257823906753,
+      "learning_rate": 4.642997839189076e-08,
+      "logits/chosen": -3.5286686420440674,
+      "logits/rejected": -3.513028383255005,
+      "logps/chosen": -1.5580331087112427,
+      "logps/rejected": -1.7452188730239868,
+      "loss": 1.1071,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.5580331087112427,
+      "rewards/margins": 0.18718580901622772,
+      "rewards/rejected": -1.7452188730239868,
+      "step": 2960
+    },
+    {
+      "epoch": 0.5117160578911096,
+      "grad_norm": 11.667295420039917,
+      "learning_rate": 4.639116533447286e-08,
+      "logits/chosen": -3.4796440601348877,
+      "logits/rejected": -3.461742401123047,
+      "logps/chosen": -1.610792875289917,
+      "logps/rejected": -1.759937047958374,
+      "loss": 1.1232,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.610792875289917,
+      "rewards/margins": 0.14914412796497345,
+      "rewards/rejected": -1.759937047958374,
+      "step": 2970
+    },
+    {
+      "epoch": 0.5134390075809786,
+      "grad_norm": 8.236954710915622,
+      "learning_rate": 4.6352158835099756e-08,
+      "logits/chosen": -3.491403102874756,
+      "logits/rejected": -3.476069211959839,
+      "logps/chosen": -1.5353374481201172,
+      "logps/rejected": -1.7204663753509521,
+      "loss": 1.0939,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.5353374481201172,
+      "rewards/margins": 0.18512912094593048,
+      "rewards/rejected": -1.7204663753509521,
+      "step": 2980
+    },
+    {
+      "epoch": 0.5151619572708477,
+      "grad_norm": 8.237678965880988,
+      "learning_rate": 4.6312959246510234e-08,
+      "logits/chosen": -3.515076160430908,
+      "logits/rejected": -3.497706651687622,
+      "logps/chosen": -1.57639741897583,
+      "logps/rejected": -1.7800220251083374,
+      "loss": 1.0833,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.57639741897583,
+      "rewards/margins": 0.20362453162670135,
+      "rewards/rejected": -1.7800220251083374,
+      "step": 2990
+    },
+    {
+      "epoch": 0.5168849069607168,
+      "grad_norm": 8.439390184220704,
+      "learning_rate": 4.627356692318919e-08,
+      "logits/chosen": -3.5203659534454346,
+      "logits/rejected": -3.516258955001831,
+      "logps/chosen": -1.5130456686019897,
+      "logps/rejected": -1.6257851123809814,
+      "loss": 1.1373,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -1.5130456686019897,
+      "rewards/margins": 0.1127394288778305,
+      "rewards/rejected": -1.6257851123809814,
+      "step": 3000
+    },
+    {
+      "epoch": 0.5168849069607168,
+      "eval_logits/chosen": -3.5859289169311523,
+      "eval_logits/rejected": -3.5819735527038574,
+      "eval_logps/chosen": -1.5155746936798096,
+      "eval_logps/rejected": -1.6569726467132568,
+      "eval_loss": 1.1094133853912354,
+      "eval_rewards/accuracies": 0.6143122911453247,
+      "eval_rewards/chosen": -1.5155746936798096,
+      "eval_rewards/margins": 0.14139802753925323,
+      "eval_rewards/rejected": -1.6569726467132568,
+      "eval_runtime": 155.4245,
+      "eval_samples_per_second": 27.692,
+      "eval_steps_per_second": 3.461,
+      "step": 3000
+    },
+    {
+      "epoch": 0.5186078566505858,
+      "grad_norm": 8.442509286895747,
+      "learning_rate": 4.6233982221364434e-08,
+      "logits/chosen": -3.543048143386841,
+      "logits/rejected": -3.5297179222106934,
+      "logps/chosen": -1.5920108556747437,
+      "logps/rejected": -1.7920939922332764,
+      "loss": 1.0908,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.5920108556747437,
+      "rewards/margins": 0.20008325576782227,
+      "rewards/rejected": -1.7920939922332764,
+      "step": 3010
+    },
+    {
+      "epoch": 0.5203308063404548,
+      "grad_norm": 8.629372755763978,
+      "learning_rate": 4.6194205499003467e-08,
+      "logits/chosen": -3.5965657234191895,
+      "logits/rejected": -3.577505588531494,
+      "logps/chosen": -1.5747871398925781,
+      "logps/rejected": -1.7411123514175415,
+      "loss": 1.1174,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.5747871398925781,
+      "rewards/margins": 0.1663251668214798,
+      "rewards/rejected": -1.7411123514175415,
+      "step": 3020
+    },
+    {
+      "epoch": 0.5220537560303239,
+      "grad_norm": 8.982698391775937,
+      "learning_rate": 4.6154237115810266e-08,
+      "logits/chosen": -3.518263339996338,
+      "logits/rejected": -3.506272792816162,
+      "logps/chosen": -1.662444829940796,
+      "logps/rejected": -1.7727781534194946,
+      "loss": 1.1379,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.662444829940796,
+      "rewards/margins": 0.11033342778682709,
+      "rewards/rejected": -1.7727781534194946,
+      "step": 3030
+    },
+    {
+      "epoch": 0.523776705720193,
+      "grad_norm": 7.730811674929311,
+      "learning_rate": 4.6114077433221995e-08,
+      "logits/chosen": -3.591066837310791,
+      "logits/rejected": -3.5877156257629395,
+      "logps/chosen": -1.5834176540374756,
+      "logps/rejected": -1.8303254842758179,
+      "loss": 1.0896,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.5834176540374756,
+      "rewards/margins": 0.24690775573253632,
+      "rewards/rejected": -1.8303254842758179,
+      "step": 3040
+    },
+    {
+      "epoch": 0.525499655410062,
+      "grad_norm": 8.307435293283143,
+      "learning_rate": 4.6073726814405746e-08,
+      "logits/chosen": -3.501377582550049,
+      "logits/rejected": -3.4929351806640625,
+      "logps/chosen": -1.5409209728240967,
+      "logps/rejected": -1.7757800817489624,
+      "loss": 1.0651,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.5409209728240967,
+      "rewards/margins": 0.23485907912254333,
+      "rewards/rejected": -1.7757800817489624,
+      "step": 3050
+    },
+    {
+      "epoch": 0.5272226050999311,
+      "grad_norm": 9.978752270278708,
+      "learning_rate": 4.603318562425528e-08,
+      "logits/chosen": -3.531449556350708,
+      "logits/rejected": -3.5214638710021973,
+      "logps/chosen": -1.6296383142471313,
+      "logps/rejected": -1.7464641332626343,
+      "loss": 1.1509,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -1.6296383142471313,
+      "rewards/margins": 0.11682607978582382,
+      "rewards/rejected": -1.7464641332626343,
+      "step": 3060
+    },
+    {
+      "epoch": 0.5289455547898001,
+      "grad_norm": 9.226240750789682,
+      "learning_rate": 4.59924542293877e-08,
+      "logits/chosen": -3.501885175704956,
+      "logits/rejected": -3.4875850677490234,
+      "logps/chosen": -1.6356384754180908,
+      "logps/rejected": -1.8084115982055664,
+      "loss": 1.1284,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -1.6356384754180908,
+      "rewards/margins": 0.17277322709560394,
+      "rewards/rejected": -1.8084115982055664,
+      "step": 3070
+    },
+    {
+      "epoch": 0.5306685044796692,
+      "grad_norm": 10.337092060119454,
+      "learning_rate": 4.5951532998140134e-08,
+      "logits/chosen": -3.4408512115478516,
+      "logits/rejected": -3.4287612438201904,
+      "logps/chosen": -1.6574437618255615,
+      "logps/rejected": -1.9280357360839844,
+      "loss": 1.0513,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.6574437618255615,
+      "rewards/margins": 0.2705918252468109,
+      "rewards/rejected": -1.9280357360839844,
+      "step": 3080
+    },
+    {
+      "epoch": 0.5323914541695383,
+      "grad_norm": 8.915373623740145,
+      "learning_rate": 4.591042230056644e-08,
+      "logits/chosen": -3.5093600749969482,
+      "logits/rejected": -3.4992928504943848,
+      "logps/chosen": -1.5691301822662354,
+      "logps/rejected": -1.8421266078948975,
+      "loss": 1.0446,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.5691301822662354,
+      "rewards/margins": 0.2729964554309845,
+      "rewards/rejected": -1.8421266078948975,
+      "step": 3090
+    },
+    {
+      "epoch": 0.5341144038594073,
+      "grad_norm": 9.623006596465551,
+      "learning_rate": 4.5869122508433834e-08,
+      "logits/chosen": -3.5342376232147217,
+      "logits/rejected": -3.5169079303741455,
+      "logps/chosen": -1.54226553440094,
+      "logps/rejected": -1.717892050743103,
+      "loss": 1.0913,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.54226553440094,
+      "rewards/margins": 0.17562679946422577,
+      "rewards/rejected": -1.717892050743103,
+      "step": 3100
+    },
+    {
+      "epoch": 0.5341144038594073,
+      "eval_logits/chosen": -3.5871589183807373,
+      "eval_logits/rejected": -3.5832390785217285,
+      "eval_logps/chosen": -1.5184277296066284,
+      "eval_logps/rejected": -1.6607660055160522,
+      "eval_loss": 1.1089062690734863,
+      "eval_rewards/accuracies": 0.6154739856719971,
+      "eval_rewards/chosen": -1.5184277296066284,
+      "eval_rewards/margins": 0.14233826100826263,
+      "eval_rewards/rejected": -1.6607660055160522,
+      "eval_runtime": 155.4387,
+      "eval_samples_per_second": 27.689,
+      "eval_steps_per_second": 3.461,
+      "step": 3100
+    },
+    {
+      "epoch": 0.5358373535492763,
+      "grad_norm": 10.050748542463259,
+      "learning_rate": 4.5827633995219486e-08,
+      "logits/chosen": -3.483391523361206,
+      "logits/rejected": -3.4890105724334717,
+      "logps/chosen": -1.6568834781646729,
+      "logps/rejected": -1.813035249710083,
+      "loss": 1.1493,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.6568834781646729,
+      "rewards/margins": 0.15615172684192657,
+      "rewards/rejected": -1.813035249710083,
+      "step": 3110
+    },
+    {
+      "epoch": 0.5375603032391454,
+      "grad_norm": 9.243586706780993,
+      "learning_rate": 4.5785957136107236e-08,
+      "logits/chosen": -3.5092475414276123,
+      "logits/rejected": -3.4947190284729004,
+      "logps/chosen": -1.680565595626831,
+      "logps/rejected": -1.8927099704742432,
+      "loss": 1.0884,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.680565595626831,
+      "rewards/margins": 0.21214430034160614,
+      "rewards/rejected": -1.8927099704742432,
+      "step": 3120
+    },
+    {
+      "epoch": 0.5392832529290145,
+      "grad_norm": 8.73938445985878,
+      "learning_rate": 4.574409230798413e-08,
+      "logits/chosen": -3.4614319801330566,
+      "logits/rejected": -3.454329013824463,
+      "logps/chosen": -1.6001850366592407,
+      "logps/rejected": -1.7785412073135376,
+      "loss": 1.106,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.6001850366592407,
+      "rewards/margins": 0.17835648357868195,
+      "rewards/rejected": -1.7785412073135376,
+      "step": 3130
+    },
+    {
+      "epoch": 0.5410062026188835,
+      "grad_norm": 9.34139293673317,
+      "learning_rate": 4.5702039889437015e-08,
+      "logits/chosen": -3.541647434234619,
+      "logits/rejected": -3.5333714485168457,
+      "logps/chosen": -1.589685320854187,
+      "logps/rejected": -1.8051960468292236,
+      "loss": 1.0924,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.589685320854187,
+      "rewards/margins": 0.21551057696342468,
+      "rewards/rejected": -1.8051960468292236,
+      "step": 3140
+    },
+    {
+      "epoch": 0.5427291523087526,
+      "grad_norm": 8.312432897779319,
+      "learning_rate": 4.565980026074917e-08,
+      "logits/chosen": -3.5285816192626953,
+      "logits/rejected": -3.5140891075134277,
+      "logps/chosen": -1.556545615196228,
+      "logps/rejected": -1.740289330482483,
+      "loss": 1.1041,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.556545615196228,
+      "rewards/margins": 0.1837436854839325,
+      "rewards/rejected": -1.740289330482483,
+      "step": 3150
+    },
+    {
+      "epoch": 0.5444521019986216,
+      "grad_norm": 8.619241922274595,
+      "learning_rate": 4.5617373803896796e-08,
+      "logits/chosen": -3.5037734508514404,
+      "logits/rejected": -3.486937999725342,
+      "logps/chosen": -1.4796823263168335,
+      "logps/rejected": -1.6995967626571655,
+      "loss": 1.0727,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.4796823263168335,
+      "rewards/margins": 0.21991446614265442,
+      "rewards/rejected": -1.6995967626571655,
+      "step": 3160
+    },
+    {
+      "epoch": 0.5461750516884907,
+      "grad_norm": 10.62876758719283,
+      "learning_rate": 4.557476090254562e-08,
+      "logits/chosen": -3.519787549972534,
+      "logits/rejected": -3.4998538494110107,
+      "logps/chosen": -1.5677258968353271,
+      "logps/rejected": -1.7106120586395264,
+      "loss": 1.1244,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.5677258968353271,
+      "rewards/margins": 0.14288616180419922,
+      "rewards/rejected": -1.7106120586395264,
+      "step": 3170
+    },
+    {
+      "epoch": 0.5478980013783598,
+      "grad_norm": 11.243237901076043,
+      "learning_rate": 4.5531961942047384e-08,
+      "logits/chosen": -3.541259765625,
+      "logits/rejected": -3.5198912620544434,
+      "logps/chosen": -1.5673919916152954,
+      "logps/rejected": -1.8028663396835327,
+      "loss": 1.0719,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.5673919916152954,
+      "rewards/margins": 0.23547430336475372,
+      "rewards/rejected": -1.8028663396835327,
+      "step": 3180
+    },
+    {
+      "epoch": 0.5496209510682288,
+      "grad_norm": 7.63886779255092,
+      "learning_rate": 4.548897730943638e-08,
+      "logits/chosen": -3.5381901264190674,
+      "logits/rejected": -3.5345962047576904,
+      "logps/chosen": -1.563126802444458,
+      "logps/rejected": -1.759702444076538,
+      "loss": 1.0881,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.563126802444458,
+      "rewards/margins": 0.19657577574253082,
+      "rewards/rejected": -1.759702444076538,
+      "step": 3190
+    },
+    {
+      "epoch": 0.5513439007580979,
+      "grad_norm": 10.205956683799204,
+      "learning_rate": 4.544580739342596e-08,
+      "logits/chosen": -3.5039379596710205,
+      "logits/rejected": -3.4966156482696533,
+      "logps/chosen": -1.6218202114105225,
+      "logps/rejected": -1.6966960430145264,
+      "loss": 1.1645,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -1.6218202114105225,
+      "rewards/margins": 0.07487578690052032,
+      "rewards/rejected": -1.6966960430145264,
+      "step": 3200
+    },
+    {
+      "epoch": 0.5513439007580979,
+      "eval_logits/chosen": -3.5734264850616455,
+      "eval_logits/rejected": -3.569460391998291,
+      "eval_logps/chosen": -1.5257829427719116,
+      "eval_logps/rejected": -1.6690964698791504,
+      "eval_loss": 1.108445167541504,
+      "eval_rewards/accuracies": 0.6089683771133423,
+      "eval_rewards/chosen": -1.5257829427719116,
+      "eval_rewards/margins": 0.14331360161304474,
+      "eval_rewards/rejected": -1.6690964698791504,
+      "eval_runtime": 155.7335,
+      "eval_samples_per_second": 27.637,
+      "eval_steps_per_second": 3.455,
+      "step": 3200
+    },
+    {
+      "epoch": 0.5530668504479669,
+      "grad_norm": 8.27058381119597,
+      "learning_rate": 4.540245258440499e-08,
+      "logits/chosen": -3.4453647136688232,
+      "logits/rejected": -3.4320156574249268,
+      "logps/chosen": -1.5692598819732666,
+      "logps/rejected": -1.7543020248413086,
+      "loss": 1.0895,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.5692598819732666,
+      "rewards/margins": 0.18504200875759125,
+      "rewards/rejected": -1.7543020248413086,
+      "step": 3210
+    },
+    {
+      "epoch": 0.554789800137836,
+      "grad_norm": 7.56945553550291,
+      "learning_rate": 4.535891327443435e-08,
+      "logits/chosen": -3.464398145675659,
+      "logits/rejected": -3.4613430500030518,
+      "logps/chosen": -1.5932343006134033,
+      "logps/rejected": -1.788837194442749,
+      "loss": 1.0829,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.5932343006134033,
+      "rewards/margins": 0.19560299813747406,
+      "rewards/rejected": -1.788837194442749,
+      "step": 3220
+    },
+    {
+      "epoch": 0.556512749827705,
+      "grad_norm": 9.33625441285629,
+      "learning_rate": 4.531518985724338e-08,
+      "logits/chosen": -3.4936187267303467,
+      "logits/rejected": -3.4843459129333496,
+      "logps/chosen": -1.5727007389068604,
+      "logps/rejected": -1.7749049663543701,
+      "loss": 1.097,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.5727007389068604,
+      "rewards/margins": 0.20220418274402618,
+      "rewards/rejected": -1.7749049663543701,
+      "step": 3230
+    },
+    {
+      "epoch": 0.5582356995175741,
+      "grad_norm": 9.778656803027204,
+      "learning_rate": 4.527128272822629e-08,
+      "logits/chosen": -3.5645718574523926,
+      "logits/rejected": -3.549710512161255,
+      "logps/chosen": -1.6608335971832275,
+      "logps/rejected": -1.8385403156280518,
+      "loss": 1.1171,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.6608335971832275,
+      "rewards/margins": 0.17770645022392273,
+      "rewards/rejected": -1.8385403156280518,
+      "step": 3240
+    },
+    {
+      "epoch": 0.5599586492074431,
+      "grad_norm": 8.615013609988209,
+      "learning_rate": 4.5227192284438634e-08,
+      "logits/chosen": -3.510822296142578,
+      "logits/rejected": -3.497312068939209,
+      "logps/chosen": -1.5432566404342651,
+      "logps/rejected": -1.735828161239624,
+      "loss": 1.0809,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.5432566404342651,
+      "rewards/margins": 0.19257155060768127,
+      "rewards/rejected": -1.735828161239624,
+      "step": 3250
+    },
+    {
+      "epoch": 0.5616815988973122,
+      "grad_norm": 9.799686803199947,
+      "learning_rate": 4.5182918924593705e-08,
+      "logits/chosen": -3.5094618797302246,
+      "logits/rejected": -3.5005931854248047,
+      "logps/chosen": -1.5862109661102295,
+      "logps/rejected": -1.7810779809951782,
+      "loss": 1.089,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.5862109661102295,
+      "rewards/margins": 0.19486697018146515,
+      "rewards/rejected": -1.7810779809951782,
+      "step": 3260
+    },
+    {
+      "epoch": 0.5634045485871813,
+      "grad_norm": 8.992251134914655,
+      "learning_rate": 4.5138463049058887e-08,
+      "logits/chosen": -3.556962490081787,
+      "logits/rejected": -3.553687572479248,
+      "logps/chosen": -1.711226224899292,
+      "logps/rejected": -1.8713598251342773,
+      "loss": 1.1269,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.711226224899292,
+      "rewards/margins": 0.16013364493846893,
+      "rewards/rejected": -1.8713598251342773,
+      "step": 3270
+    },
+    {
+      "epoch": 0.5651274982770503,
+      "grad_norm": 10.217462753074805,
+      "learning_rate": 4.5093825059852096e-08,
+      "logits/chosen": -3.5521655082702637,
+      "logits/rejected": -3.547722578048706,
+      "logps/chosen": -1.5702674388885498,
+      "logps/rejected": -1.7599836587905884,
+      "loss": 1.0962,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.5702674388885498,
+      "rewards/margins": 0.1897161304950714,
+      "rewards/rejected": -1.7599836587905884,
+      "step": 3280
+    },
+    {
+      "epoch": 0.5668504479669194,
+      "grad_norm": 9.81668486935072,
+      "learning_rate": 4.50490053606381e-08,
+      "logits/chosen": -3.5803565979003906,
+      "logits/rejected": -3.563225269317627,
+      "logps/chosen": -1.6118943691253662,
+      "logps/rejected": -1.817233681678772,
+      "loss": 1.0845,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.6118943691253662,
+      "rewards/margins": 0.2053392380475998,
+      "rewards/rejected": -1.817233681678772,
+      "step": 3290
+    },
+    {
+      "epoch": 0.5685733976567884,
+      "grad_norm": 9.710435841997638,
+      "learning_rate": 4.5004004356724893e-08,
+      "logits/chosen": -3.512361526489258,
+      "logits/rejected": -3.4992356300354004,
+      "logps/chosen": -1.6525514125823975,
+      "logps/rejected": -1.7996203899383545,
+      "loss": 1.1238,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.6525514125823975,
+      "rewards/margins": 0.14706900715827942,
+      "rewards/rejected": -1.7996203899383545,
+      "step": 3300
+    },
+    {
+      "epoch": 0.5685733976567884,
+      "eval_logits/chosen": -3.586374521255493,
+      "eval_logits/rejected": -3.5825035572052,
+      "eval_logps/chosen": -1.5336194038391113,
+      "eval_logps/rejected": -1.6783100366592407,
+      "eval_loss": 1.107811450958252,
+      "eval_rewards/accuracies": 0.6080390214920044,
+      "eval_rewards/chosen": -1.5336194038391113,
+      "eval_rewards/margins": 0.14469067752361298,
+      "eval_rewards/rejected": -1.6783100366592407,
+      "eval_runtime": 155.241,
+      "eval_samples_per_second": 27.725,
+      "eval_steps_per_second": 3.466,
+      "step": 3300
+    },
+    {
+      "epoch": 0.5702963473466575,
+      "grad_norm": 9.342963754375628,
+      "learning_rate": 4.495882245506002e-08,
+      "logits/chosen": -3.470132827758789,
+      "logits/rejected": -3.4552714824676514,
+      "logps/chosen": -1.5767006874084473,
+      "logps/rejected": -1.7837556600570679,
+      "loss": 1.0843,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.5767006874084473,
+      "rewards/margins": 0.20705482363700867,
+      "rewards/rejected": -1.7837556600570679,
+      "step": 3310
+    },
+    {
+      "epoch": 0.5720192970365265,
+      "grad_norm": 11.02401162277804,
+      "learning_rate": 4.4913460064226896e-08,
+      "logits/chosen": -3.5203983783721924,
+      "logits/rejected": -3.500249147415161,
+      "logps/chosen": -1.616499662399292,
+      "logps/rejected": -1.7777793407440186,
+      "loss": 1.105,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.616499662399292,
+      "rewards/margins": 0.16127988696098328,
+      "rewards/rejected": -1.7777793407440186,
+      "step": 3320
+    },
+    {
+      "epoch": 0.5737422467263956,
+      "grad_norm": 13.944554058075337,
+      "learning_rate": 4.486791759444111e-08,
+      "logits/chosen": -3.564995527267456,
+      "logits/rejected": -3.546194076538086,
+      "logps/chosen": -1.5792338848114014,
+      "logps/rejected": -1.8268985748291016,
+      "loss": 1.068,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.5792338848114014,
+      "rewards/margins": 0.24766476452350616,
+      "rewards/rejected": -1.8268985748291016,
+      "step": 3330
+    },
+    {
+      "epoch": 0.5754651964162646,
+      "grad_norm": 9.557628474324263,
+      "learning_rate": 4.482219545754672e-08,
+      "logits/chosen": -3.5236308574676514,
+      "logits/rejected": -3.513779401779175,
+      "logps/chosen": -1.663000464439392,
+      "logps/rejected": -1.7964719533920288,
+      "loss": 1.1501,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -1.663000464439392,
+      "rewards/margins": 0.1334715187549591,
+      "rewards/rejected": -1.7964719533920288,
+      "step": 3340
+    },
+    {
+      "epoch": 0.5771881461061337,
+      "grad_norm": 9.912766524066669,
+      "learning_rate": 4.4776294067012546e-08,
+      "logits/chosen": -3.4631073474884033,
+      "logits/rejected": -3.4542746543884277,
+      "logps/chosen": -1.6885662078857422,
+      "logps/rejected": -1.9765657186508179,
+      "loss": 1.0675,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.6885662078857422,
+      "rewards/margins": 0.28799954056739807,
+      "rewards/rejected": -1.9765657186508179,
+      "step": 3350
+    },
+    {
+      "epoch": 0.5789110957960028,
+      "grad_norm": 9.455322101813575,
+      "learning_rate": 4.473021383792838e-08,
+      "logits/chosen": -3.542515277862549,
+      "logits/rejected": -3.5222764015197754,
+      "logps/chosen": -1.587283968925476,
+      "logps/rejected": -1.8880081176757812,
+      "loss": 1.0527,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.587283968925476,
+      "rewards/margins": 0.3007240891456604,
+      "rewards/rejected": -1.8880081176757812,
+      "step": 3360
+    },
+    {
+      "epoch": 0.5806340454858718,
+      "grad_norm": 9.753362783025247,
+      "learning_rate": 4.468395518700129e-08,
+      "logits/chosen": -3.489945888519287,
+      "logits/rejected": -3.4834723472595215,
+      "logps/chosen": -1.6052488088607788,
+      "logps/rejected": -1.833269476890564,
+      "loss": 1.0792,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.6052488088607788,
+      "rewards/margins": 0.2280208319425583,
+      "rewards/rejected": -1.833269476890564,
+      "step": 3370
+    },
+    {
+      "epoch": 0.5823569951757409,
+      "grad_norm": 10.318108078875047,
+      "learning_rate": 4.463751853255182e-08,
+      "logits/chosen": -3.5478878021240234,
+      "logits/rejected": -3.527463912963867,
+      "logps/chosen": -1.628976821899414,
+      "logps/rejected": -1.8107759952545166,
+      "loss": 1.0972,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.628976821899414,
+      "rewards/margins": 0.18179893493652344,
+      "rewards/rejected": -1.8107759952545166,
+      "step": 3380
+    },
+    {
+      "epoch": 0.5840799448656099,
+      "grad_norm": 9.175249694255482,
+      "learning_rate": 4.45909042945102e-08,
+      "logits/chosen": -3.511514663696289,
+      "logits/rejected": -3.496006488800049,
+      "logps/chosen": -1.6418826580047607,
+      "logps/rejected": -1.8208776712417603,
+      "loss": 1.1036,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.6418826580047607,
+      "rewards/margins": 0.17899474501609802,
+      "rewards/rejected": -1.8208776712417603,
+      "step": 3390
+    },
+    {
+      "epoch": 0.585802894555479,
+      "grad_norm": 9.20143692502329,
+      "learning_rate": 4.454411289441259e-08,
+      "logits/chosen": -3.5489230155944824,
+      "logits/rejected": -3.526597499847412,
+      "logps/chosen": -1.5647203922271729,
+      "logps/rejected": -1.8457648754119873,
+      "loss": 1.0437,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.5647203922271729,
+      "rewards/margins": 0.28104472160339355,
+      "rewards/rejected": -1.8457648754119873,
+      "step": 3400
+    },
+    {
+      "epoch": 0.585802894555479,
+      "eval_logits/chosen": -3.584557294845581,
+      "eval_logits/rejected": -3.5806984901428223,
+      "eval_logps/chosen": -1.543709397315979,
+      "eval_logps/rejected": -1.6900385618209839,
+      "eval_loss": 1.1070204973220825,
+      "eval_rewards/accuracies": 0.6080390214920044,
+      "eval_rewards/chosen": -1.543709397315979,
+      "eval_rewards/margins": 0.14632916450500488,
+      "eval_rewards/rejected": -1.6900385618209839,
+      "eval_runtime": 155.1501,
+      "eval_samples_per_second": 27.741,
+      "eval_steps_per_second": 3.468,
+      "step": 3400
+    },
+    {
+      "epoch": 0.587525844245348,
+      "grad_norm": 7.971440871171622,
+      "learning_rate": 4.4497144755397215e-08,
+      "logits/chosen": -3.4743003845214844,
+      "logits/rejected": -3.4578194618225098,
+      "logps/chosen": -1.5522679090499878,
+      "logps/rejected": -1.7532596588134766,
+      "loss": 1.0903,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.5522679090499878,
+      "rewards/margins": 0.20099177956581116,
+      "rewards/rejected": -1.7532596588134766,
+      "step": 3410
+    },
+    {
+      "epoch": 0.5892487939352171,
+      "grad_norm": 9.345193819803288,
+      "learning_rate": 4.4450000302200574e-08,
+      "logits/chosen": -3.4939608573913574,
+      "logits/rejected": -3.478551149368286,
+      "logps/chosen": -1.5343397855758667,
+      "logps/rejected": -1.8078359365463257,
+      "loss": 1.0451,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.5343397855758667,
+      "rewards/margins": 0.27349621057510376,
+      "rewards/rejected": -1.8078359365463257,
+      "step": 3420
+    },
+    {
+      "epoch": 0.5909717436250862,
+      "grad_norm": 8.382649918163924,
+      "learning_rate": 4.440267996115359e-08,
+      "logits/chosen": -3.5295989513397217,
+      "logits/rejected": -3.5176353454589844,
+      "logps/chosen": -1.5676885843276978,
+      "logps/rejected": -1.8371127843856812,
+      "loss": 1.0518,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.5676885843276978,
+      "rewards/margins": 0.26942428946495056,
+      "rewards/rejected": -1.8371127843856812,
+      "step": 3430
+    },
+    {
+      "epoch": 0.5926946933149552,
+      "grad_norm": 8.467363003204087,
+      "learning_rate": 4.435518416017774e-08,
+      "logits/chosen": -3.48310923576355,
+      "logits/rejected": -3.4738152027130127,
+      "logps/chosen": -1.6629886627197266,
+      "logps/rejected": -1.9278119802474976,
+      "loss": 1.0785,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.6629886627197266,
+      "rewards/margins": 0.26482346653938293,
+      "rewards/rejected": -1.9278119802474976,
+      "step": 3440
+    },
+    {
+      "epoch": 0.5944176430048242,
+      "grad_norm": 9.418139827801163,
+      "learning_rate": 4.430751332878122e-08,
+      "logits/chosen": -3.6283226013183594,
+      "logits/rejected": -3.606818437576294,
+      "logps/chosen": -1.6628334522247314,
+      "logps/rejected": -1.9258276224136353,
+      "loss": 1.0561,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.6628334522247314,
+      "rewards/margins": 0.2629939913749695,
+      "rewards/rejected": -1.9258276224136353,
+      "step": 3450
+    },
+    {
+      "epoch": 0.5961405926946933,
+      "grad_norm": 10.083525903149365,
+      "learning_rate": 4.425966789805503e-08,
+      "logits/chosen": -3.5130038261413574,
+      "logits/rejected": -3.503751039505005,
+      "logps/chosen": -1.5865585803985596,
+      "logps/rejected": -1.8138000965118408,
+      "loss": 1.0803,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.5865585803985596,
+      "rewards/margins": 0.22724118828773499,
+      "rewards/rejected": -1.8138000965118408,
+      "step": 3460
+    },
+    {
+      "epoch": 0.5978635423845624,
+      "grad_norm": 8.796000678467392,
+      "learning_rate": 4.4211648300669074e-08,
+      "logits/chosen": -3.556635618209839,
+      "logits/rejected": -3.5471878051757812,
+      "logps/chosen": -1.6412687301635742,
+      "logps/rejected": -1.8160209655761719,
+      "loss": 1.1054,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.6412687301635742,
+      "rewards/margins": 0.1747523695230484,
+      "rewards/rejected": -1.8160209655761719,
+      "step": 3470
+    },
+    {
+      "epoch": 0.5995864920744314,
+      "grad_norm": 10.193730339069141,
+      "learning_rate": 4.416345497086827e-08,
+      "logits/chosen": -3.5053036212921143,
+      "logits/rejected": -3.4847323894500732,
+      "logps/chosen": -1.610018014907837,
+      "logps/rejected": -1.8443092107772827,
+      "loss": 1.0929,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.610018014907837,
+      "rewards/margins": 0.23429112136363983,
+      "rewards/rejected": -1.8443092107772827,
+      "step": 3480
+    },
+    {
+      "epoch": 0.6013094417643005,
+      "grad_norm": 10.324069697933579,
+      "learning_rate": 4.411508834446863e-08,
+      "logits/chosen": -3.511176347732544,
+      "logits/rejected": -3.495032548904419,
+      "logps/chosen": -1.6053279638290405,
+      "logps/rejected": -1.837648630142212,
+      "loss": 1.0877,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.6053279638290405,
+      "rewards/margins": 0.23232051730155945,
+      "rewards/rejected": -1.837648630142212,
+      "step": 3490
+    },
+    {
+      "epoch": 0.6030323914541695,
+      "grad_norm": 7.152139016664346,
+      "learning_rate": 4.406654885885326e-08,
+      "logits/chosen": -3.475548505783081,
+      "logits/rejected": -3.4712727069854736,
+      "logps/chosen": -1.6130670309066772,
+      "logps/rejected": -1.804485559463501,
+      "loss": 1.099,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.6130670309066772,
+      "rewards/margins": 0.1914181411266327,
+      "rewards/rejected": -1.804485559463501,
+      "step": 3500
+    },
+    {
+      "epoch": 0.6030323914541695,
+      "eval_logits/chosen": -3.5800795555114746,
+      "eval_logits/rejected": -3.5762269496917725,
+      "eval_logps/chosen": -1.5523650646209717,
+      "eval_logps/rejected": -1.699596643447876,
+      "eval_loss": 1.1067214012145996,
+      "eval_rewards/accuracies": 0.6105948090553284,
+      "eval_rewards/chosen": -1.5523650646209717,
+      "eval_rewards/margins": 0.14723153412342072,
+      "eval_rewards/rejected": -1.699596643447876,
+      "eval_runtime": 155.3185,
+      "eval_samples_per_second": 27.711,
+      "eval_steps_per_second": 3.464,
+      "step": 3500
+    },
+    {
+      "epoch": 0.6047553411440386,
+      "grad_norm": 9.935228913015843,
+      "learning_rate": 4.401783695296847e-08,
+      "logits/chosen": -3.447859287261963,
+      "logits/rejected": -3.4360461235046387,
+      "logps/chosen": -1.7676012516021729,
+      "logps/rejected": -1.9062551259994507,
+      "loss": 1.1365,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.7676012516021729,
+      "rewards/margins": 0.13865378499031067,
+      "rewards/rejected": -1.9062551259994507,
+      "step": 3510
+    },
+    {
+      "epoch": 0.6064782908339077,
+      "grad_norm": 8.472846275195014,
+      "learning_rate": 4.3968953067319766e-08,
+      "logits/chosen": -3.467944383621216,
+      "logits/rejected": -3.4582438468933105,
+      "logps/chosen": -1.6749337911605835,
+      "logps/rejected": -1.8136667013168335,
+      "loss": 1.133,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -1.6749337911605835,
+      "rewards/margins": 0.13873298466205597,
+      "rewards/rejected": -1.8136667013168335,
+      "step": 3520
+    },
+    {
+      "epoch": 0.6082012405237767,
+      "grad_norm": 9.378804717937525,
+      "learning_rate": 4.391989764396792e-08,
+      "logits/chosen": -3.569918155670166,
+      "logits/rejected": -3.546748638153076,
+      "logps/chosen": -1.61806321144104,
+      "logps/rejected": -1.8390001058578491,
+      "loss": 1.0836,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.61806321144104,
+      "rewards/margins": 0.22093670070171356,
+      "rewards/rejected": -1.8390001058578491,
+      "step": 3530
+    },
+    {
+      "epoch": 0.6099241902136457,
+      "grad_norm": 9.10499545903494,
+      "learning_rate": 4.387067112652487e-08,
+      "logits/chosen": -3.456853151321411,
+      "logits/rejected": -3.44356107711792,
+      "logps/chosen": -1.6680368185043335,
+      "logps/rejected": -1.8816606998443604,
+      "loss": 1.0869,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.6680368185043335,
+      "rewards/margins": 0.2136237919330597,
+      "rewards/rejected": -1.8816606998443604,
+      "step": 3540
+    },
+    {
+      "epoch": 0.6116471399035148,
+      "grad_norm": 11.218396222951657,
+      "learning_rate": 4.382127396014982e-08,
+      "logits/chosen": -3.51519775390625,
+      "logits/rejected": -3.508298873901367,
+      "logps/chosen": -1.699819803237915,
+      "logps/rejected": -1.8098218441009521,
+      "loss": 1.1415,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -1.699819803237915,
+      "rewards/margins": 0.11000212281942368,
+      "rewards/rejected": -1.8098218441009521,
+      "step": 3550
+    },
+    {
+      "epoch": 0.6133700895933839,
+      "grad_norm": 9.367545920038426,
+      "learning_rate": 4.377170659154514e-08,
+      "logits/chosen": -3.4701626300811768,
+      "logits/rejected": -3.452913761138916,
+      "logps/chosen": -1.6075149774551392,
+      "logps/rejected": -1.8662782907485962,
+      "loss": 1.0505,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.6075149774551392,
+      "rewards/margins": 0.25876325368881226,
+      "rewards/rejected": -1.8662782907485962,
+      "step": 3560
+    },
+    {
+      "epoch": 0.6150930392832529,
+      "grad_norm": 10.124046178021791,
+      "learning_rate": 4.372196946895238e-08,
+      "logits/chosen": -3.5569965839385986,
+      "logits/rejected": -3.53851318359375,
+      "logps/chosen": -1.6315815448760986,
+      "logps/rejected": -1.8075681924819946,
+      "loss": 1.1094,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.6315815448760986,
+      "rewards/margins": 0.17598679661750793,
+      "rewards/rejected": -1.8075681924819946,
+      "step": 3570
+    },
+    {
+      "epoch": 0.616815988973122,
+      "grad_norm": 9.80623110586067,
+      "learning_rate": 4.367206304214815e-08,
+      "logits/chosen": -3.5138161182403564,
+      "logits/rejected": -3.4991650581359863,
+      "logps/chosen": -1.6682888269424438,
+      "logps/rejected": -1.9168756008148193,
+      "loss": 1.0542,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.6682888269424438,
+      "rewards/margins": 0.24858692288398743,
+      "rewards/rejected": -1.9168756008148193,
+      "step": 3580
+    },
+    {
+      "epoch": 0.618538938662991,
+      "grad_norm": 9.283209695188653,
+      "learning_rate": 4.3621987762440115e-08,
+      "logits/chosen": -3.521824359893799,
+      "logits/rejected": -3.512800931930542,
+      "logps/chosen": -1.7147804498672485,
+      "logps/rejected": -1.8777177333831787,
+      "loss": 1.1153,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.7147804498672485,
+      "rewards/margins": 0.1629372537136078,
+      "rewards/rejected": -1.8777177333831787,
+      "step": 3590
+    },
+    {
+      "epoch": 0.6202618883528601,
+      "grad_norm": 12.030257639191648,
+      "learning_rate": 4.3571744082662884e-08,
+      "logits/chosen": -3.4929957389831543,
+      "logits/rejected": -3.4806008338928223,
+      "logps/chosen": -1.7134592533111572,
+      "logps/rejected": -1.8542486429214478,
+      "loss": 1.1365,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.7134592533111572,
+      "rewards/margins": 0.1407892256975174,
+      "rewards/rejected": -1.8542486429214478,
+      "step": 3600
+    },
+    {
+      "epoch": 0.6202618883528601,
+      "eval_logits/chosen": -3.5749573707580566,
+      "eval_logits/rejected": -3.5711004734039307,
+      "eval_logps/chosen": -1.5625951290130615,
+      "eval_logps/rejected": -1.711233139038086,
+      "eval_loss": 1.1062034368515015,
+      "eval_rewards/accuracies": 0.609897792339325,
+      "eval_rewards/chosen": -1.5625951290130615,
+      "eval_rewards/margins": 0.14863790571689606,
+      "eval_rewards/rejected": -1.711233139038086,
+      "eval_runtime": 155.2766,
+      "eval_samples_per_second": 27.718,
+      "eval_steps_per_second": 3.465,
+      "step": 3600
+    },
+    {
+      "epoch": 0.6219848380427292,
+      "grad_norm": 8.289458762138988,
+      "learning_rate": 4.352133245717393e-08,
+      "logits/chosen": -3.479661226272583,
+      "logits/rejected": -3.469113826751709,
+      "logps/chosen": -1.7122876644134521,
+      "logps/rejected": -1.801745057106018,
+      "loss": 1.1562,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -1.7122876644134521,
+      "rewards/margins": 0.08945748955011368,
+      "rewards/rejected": -1.801745057106018,
+      "step": 3610
+    },
+    {
+      "epoch": 0.6237077877325982,
+      "grad_norm": 9.226763002334959,
+      "learning_rate": 4.347075334184946e-08,
+      "logits/chosen": -3.466191053390503,
+      "logits/rejected": -3.4528281688690186,
+      "logps/chosen": -1.5847923755645752,
+      "logps/rejected": -1.7951018810272217,
+      "loss": 1.0931,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.5847923755645752,
+      "rewards/margins": 0.21030957996845245,
+      "rewards/rejected": -1.7951018810272217,
+      "step": 3620
+    },
+    {
+      "epoch": 0.6254307374224672,
+      "grad_norm": 8.80860085286088,
+      "learning_rate": 4.34200071940803e-08,
+      "logits/chosen": -3.555785655975342,
+      "logits/rejected": -3.5508244037628174,
+      "logps/chosen": -1.6465049982070923,
+      "logps/rejected": -1.850594162940979,
+      "loss": 1.0861,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.6465049982070923,
+      "rewards/margins": 0.20408916473388672,
+      "rewards/rejected": -1.850594162940979,
+      "step": 3630
+    },
+    {
+      "epoch": 0.6271536871123363,
+      "grad_norm": 8.936442208399624,
+      "learning_rate": 4.3369094472767786e-08,
+      "logits/chosen": -3.5195987224578857,
+      "logits/rejected": -3.5109035968780518,
+      "logps/chosen": -1.6439142227172852,
+      "logps/rejected": -1.8350632190704346,
+      "loss": 1.0972,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.6439142227172852,
+      "rewards/margins": 0.19114910066127777,
+      "rewards/rejected": -1.8350632190704346,
+      "step": 3640
+    },
+    {
+      "epoch": 0.6288766368022054,
+      "grad_norm": 8.857829418667775,
+      "learning_rate": 4.331801563831956e-08,
+      "logits/chosen": -3.4837257862091064,
+      "logits/rejected": -3.484900951385498,
+      "logps/chosen": -1.632882833480835,
+      "logps/rejected": -1.750328779220581,
+      "loss": 1.1346,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -1.632882833480835,
+      "rewards/margins": 0.11744587123394012,
+      "rewards/rejected": -1.750328779220581,
+      "step": 3650
+    },
+    {
+      "epoch": 0.6305995864920745,
+      "grad_norm": 8.051027916275816,
+      "learning_rate": 4.326677115264547e-08,
+      "logits/chosen": -3.5261611938476562,
+      "logits/rejected": -3.5106654167175293,
+      "logps/chosen": -1.6454684734344482,
+      "logps/rejected": -1.9025354385375977,
+      "loss": 1.0504,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.6454684734344482,
+      "rewards/margins": 0.2570669949054718,
+      "rewards/rejected": -1.9025354385375977,
+      "step": 3660
+    },
+    {
+      "epoch": 0.6323225361819435,
+      "grad_norm": 8.592353504020787,
+      "learning_rate": 4.321536147915334e-08,
+      "logits/chosen": -3.4993038177490234,
+      "logits/rejected": -3.4843826293945312,
+      "logps/chosen": -1.6026842594146729,
+      "logps/rejected": -1.8083784580230713,
+      "loss": 1.0821,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.6026842594146729,
+      "rewards/margins": 0.20569436252117157,
+      "rewards/rejected": -1.8083784580230713,
+      "step": 3670
+    },
+    {
+      "epoch": 0.6340454858718125,
+      "grad_norm": 8.17713095660827,
+      "learning_rate": 4.3163787082744806e-08,
+      "logits/chosen": -3.510178327560425,
+      "logits/rejected": -3.4868252277374268,
+      "logps/chosen": -1.6334466934204102,
+      "logps/rejected": -1.871146559715271,
+      "loss": 1.0748,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.6334466934204102,
+      "rewards/margins": 0.237699955701828,
+      "rewards/rejected": -1.871146559715271,
+      "step": 3680
+    },
+    {
+      "epoch": 0.6357684355616816,
+      "grad_norm": 11.00577788306391,
+      "learning_rate": 4.31120484298111e-08,
+      "logits/chosen": -3.472620725631714,
+      "logits/rejected": -3.4772751331329346,
+      "logps/chosen": -1.6956323385238647,
+      "logps/rejected": -1.8623136281967163,
+      "loss": 1.1186,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.6956323385238647,
+      "rewards/margins": 0.16668136417865753,
+      "rewards/rejected": -1.8623136281967163,
+      "step": 3690
+    },
+    {
+      "epoch": 0.6374913852515507,
+      "grad_norm": 9.232030006222626,
+      "learning_rate": 4.306014598822886e-08,
+      "logits/chosen": -3.48614501953125,
+      "logits/rejected": -3.471806287765503,
+      "logps/chosen": -1.6301186084747314,
+      "logps/rejected": -1.9545173645019531,
+      "loss": 1.0205,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.6301186084747314,
+      "rewards/margins": 0.32439857721328735,
+      "rewards/rejected": -1.9545173645019531,
+      "step": 3700
+    },
+    {
+      "epoch": 0.6374913852515507,
+      "eval_logits/chosen": -3.554898977279663,
+      "eval_logits/rejected": -3.550968885421753,
+      "eval_logps/chosen": -1.572847604751587,
+      "eval_logps/rejected": -1.7227364778518677,
+      "eval_loss": 1.1057591438293457,
+      "eval_rewards/accuracies": 0.609433114528656,
+      "eval_rewards/chosen": -1.572847604751587,
+      "eval_rewards/margins": 0.1498887836933136,
+      "eval_rewards/rejected": -1.7227364778518677,
+      "eval_runtime": 155.4531,
+      "eval_samples_per_second": 27.687,
+      "eval_steps_per_second": 3.461,
+      "step": 3700
+    },
+    {
+      "epoch": 0.6392143349414197,
+      "grad_norm": 10.033465382554441,
+      "learning_rate": 4.300808022735584e-08,
+      "logits/chosen": -3.5150482654571533,
+      "logits/rejected": -3.4935264587402344,
+      "logps/chosen": -1.615647315979004,
+      "logps/rejected": -1.8444297313690186,
+      "loss": 1.0809,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.615647315979004,
+      "rewards/margins": 0.22878257930278778,
+      "rewards/rejected": -1.8444297313690186,
+      "step": 3710
+    },
+    {
+      "epoch": 0.6409372846312887,
+      "grad_norm": 9.115330707559751,
+      "learning_rate": 4.295585161802674e-08,
+      "logits/chosen": -3.5287811756134033,
+      "logits/rejected": -3.519094467163086,
+      "logps/chosen": -1.5981851816177368,
+      "logps/rejected": -1.8639476299285889,
+      "loss": 1.0556,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.5981851816177368,
+      "rewards/margins": 0.2657622694969177,
+      "rewards/rejected": -1.8639476299285889,
+      "step": 3720
+    },
+    {
+      "epoch": 0.6426602343211578,
+      "grad_norm": 9.925396702227012,
+      "learning_rate": 4.290346063254889e-08,
+      "logits/chosen": -3.4983506202697754,
+      "logits/rejected": -3.4871037006378174,
+      "logps/chosen": -1.6853961944580078,
+      "logps/rejected": -1.8356300592422485,
+      "loss": 1.1179,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.6853961944580078,
+      "rewards/margins": 0.1502339243888855,
+      "rewards/rejected": -1.8356300592422485,
+      "step": 3730
+    },
+    {
+      "epoch": 0.6443831840110269,
+      "grad_norm": 9.424676132624052,
+      "learning_rate": 4.285090774469802e-08,
+      "logits/chosen": -3.496593475341797,
+      "logits/rejected": -3.483675718307495,
+      "logps/chosen": -1.658524513244629,
+      "logps/rejected": -1.9211466312408447,
+      "loss": 1.065,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.658524513244629,
+      "rewards/margins": 0.26262176036834717,
+      "rewards/rejected": -1.9211466312408447,
+      "step": 3740
+    },
+    {
+      "epoch": 0.646106133700896,
+      "grad_norm": 9.221981977743791,
+      "learning_rate": 4.279819342971391e-08,
+      "logits/chosen": -3.5442001819610596,
+      "logits/rejected": -3.52791166305542,
+      "logps/chosen": -1.5842949151992798,
+      "logps/rejected": -1.8248989582061768,
+      "loss": 1.0611,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.5842949151992798,
+      "rewards/margins": 0.24060411751270294,
+      "rewards/rejected": -1.8248989582061768,
+      "step": 3750
+    },
+    {
+      "epoch": 0.647829083390765,
+      "grad_norm": 10.744880345548928,
+      "learning_rate": 4.27453181642962e-08,
+      "logits/chosen": -3.5217597484588623,
+      "logits/rejected": -3.520879030227661,
+      "logps/chosen": -1.754352331161499,
+      "logps/rejected": -1.8518718481063843,
+      "loss": 1.1624,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.754352331161499,
+      "rewards/margins": 0.09751953929662704,
+      "rewards/rejected": -1.8518718481063843,
+      "step": 3760
+    },
+    {
+      "epoch": 0.649552033080634,
+      "grad_norm": 8.888599416858558,
+      "learning_rate": 4.269228242659997e-08,
+      "logits/chosen": -3.496748685836792,
+      "logits/rejected": -3.4846904277801514,
+      "logps/chosen": -1.663551688194275,
+      "logps/rejected": -1.9043327569961548,
+      "loss": 1.0825,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.663551688194275,
+      "rewards/margins": 0.24078097939491272,
+      "rewards/rejected": -1.9043327569961548,
+      "step": 3770
+    },
+    {
+      "epoch": 0.6512749827705031,
+      "grad_norm": 9.842722627072968,
+      "learning_rate": 4.2639086696231486e-08,
+      "logits/chosen": -3.5215556621551514,
+      "logits/rejected": -3.50062894821167,
+      "logps/chosen": -1.6893303394317627,
+      "logps/rejected": -1.868909478187561,
+      "loss": 1.1054,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -1.6893303394317627,
+      "rewards/margins": 0.17957928776741028,
+      "rewards/rejected": -1.868909478187561,
+      "step": 3780
+    },
+    {
+      "epoch": 0.6529979324603722,
+      "grad_norm": 9.295395274133893,
+      "learning_rate": 4.2585731454243836e-08,
+      "logits/chosen": -3.4921905994415283,
+      "logits/rejected": -3.4751923084259033,
+      "logps/chosen": -1.683331847190857,
+      "logps/rejected": -1.9565273523330688,
+      "loss": 1.1083,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.683331847190857,
+      "rewards/margins": 0.2731954753398895,
+      "rewards/rejected": -1.9565273523330688,
+      "step": 3790
+    },
+    {
+      "epoch": 0.6547208821502413,
+      "grad_norm": 12.398227765090523,
+      "learning_rate": 4.2532217183132566e-08,
+      "logits/chosen": -3.5037143230438232,
+      "logits/rejected": -3.484870433807373,
+      "logps/chosen": -1.7076663970947266,
+      "logps/rejected": -1.85005784034729,
+      "loss": 1.1328,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -1.7076663970947266,
+      "rewards/margins": 0.1423913687467575,
+      "rewards/rejected": -1.85005784034729,
+      "step": 3800
+    },
+    {
+      "epoch": 0.6547208821502413,
+      "eval_logits/chosen": -3.562772274017334,
+      "eval_logits/rejected": -3.558901071548462,
+      "eval_logps/chosen": -1.5860168933868408,
+      "eval_logps/rejected": -1.7378623485565186,
+      "eval_loss": 1.1048774719238281,
+      "eval_rewards/accuracies": 0.6126858592033386,
+      "eval_rewards/chosen": -1.5860168933868408,
+      "eval_rewards/margins": 0.15184549987316132,
+      "eval_rewards/rejected": -1.7378623485565186,
+      "eval_runtime": 155.5319,
+      "eval_samples_per_second": 27.673,
+      "eval_steps_per_second": 3.459,
+      "step": 3800
+    },
+    {
+      "epoch": 0.6564438318401102,
+      "grad_norm": 9.280014183637519,
+      "learning_rate": 4.247854436683137e-08,
+      "logits/chosen": -3.5277743339538574,
+      "logits/rejected": -3.5024020671844482,
+      "logps/chosen": -1.6548550128936768,
+      "logps/rejected": -1.8574676513671875,
+      "loss": 1.0853,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.6548550128936768,
+      "rewards/margins": 0.2026127278804779,
+      "rewards/rejected": -1.8574676513671875,
+      "step": 3810
+    },
+    {
+      "epoch": 0.6581667815299793,
+      "grad_norm": 9.588015325598322,
+      "learning_rate": 4.242471349070765e-08,
+      "logits/chosen": -3.5054659843444824,
+      "logits/rejected": -3.4965972900390625,
+      "logps/chosen": -1.6443849802017212,
+      "logps/rejected": -1.8933079242706299,
+      "loss": 1.0884,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -1.6443849802017212,
+      "rewards/margins": 0.2489229440689087,
+      "rewards/rejected": -1.8933079242706299,
+      "step": 3820
+    },
+    {
+      "epoch": 0.6598897312198484,
+      "grad_norm": 10.300984170236005,
+      "learning_rate": 4.237072504155817e-08,
+      "logits/chosen": -3.5393989086151123,
+      "logits/rejected": -3.514063596725464,
+      "logps/chosen": -1.6767990589141846,
+      "logps/rejected": -1.8298031091690063,
+      "loss": 1.1184,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.6767990589141846,
+      "rewards/margins": 0.153003990650177,
+      "rewards/rejected": -1.8298031091690063,
+      "step": 3830
+    },
+    {
+      "epoch": 0.6616126809097175,
+      "grad_norm": 11.176803781206093,
+      "learning_rate": 4.231657950760461e-08,
+      "logits/chosen": -3.486666202545166,
+      "logits/rejected": -3.479262113571167,
+      "logps/chosen": -1.623412847518921,
+      "logps/rejected": -1.8921505212783813,
+      "loss": 1.0445,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.623412847518921,
+      "rewards/margins": 0.2687375247478485,
+      "rewards/rejected": -1.8921505212783813,
+      "step": 3840
+    },
+    {
+      "epoch": 0.6633356305995864,
+      "grad_norm": 9.241298635937218,
+      "learning_rate": 4.2262277378489225e-08,
+      "logits/chosen": -3.5609130859375,
+      "logits/rejected": -3.5515542030334473,
+      "logps/chosen": -1.6977201700210571,
+      "logps/rejected": -1.8079506158828735,
+      "loss": 1.1458,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -1.6977201700210571,
+      "rewards/margins": 0.11023037135601044,
+      "rewards/rejected": -1.8079506158828735,
+      "step": 3850
+    },
+    {
+      "epoch": 0.6650585802894555,
+      "grad_norm": 9.096163175550023,
+      "learning_rate": 4.220781914527035e-08,
+      "logits/chosen": -3.5616440773010254,
+      "logits/rejected": -3.5486788749694824,
+      "logps/chosen": -1.694867730140686,
+      "logps/rejected": -1.8679559230804443,
+      "loss": 1.1074,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.694867730140686,
+      "rewards/margins": 0.17308831214904785,
+      "rewards/rejected": -1.8679559230804443,
+      "step": 3860
+    },
+    {
+      "epoch": 0.6667815299793246,
+      "grad_norm": 12.386903457085221,
+      "learning_rate": 4.2153205300417966e-08,
+      "logits/chosen": -3.499711513519287,
+      "logits/rejected": -3.485344409942627,
+      "logps/chosen": -1.6886955499649048,
+      "logps/rejected": -1.8365328311920166,
+      "loss": 1.1135,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.6886955499649048,
+      "rewards/margins": 0.14783743023872375,
+      "rewards/rejected": -1.8365328311920166,
+      "step": 3870
+    },
+    {
+      "epoch": 0.6685044796691937,
+      "grad_norm": 10.562089047704522,
+      "learning_rate": 4.209843633780929e-08,
+      "logits/chosen": -3.5167980194091797,
+      "logits/rejected": -3.5195045471191406,
+      "logps/chosen": -1.7487472295761108,
+      "logps/rejected": -1.8800337314605713,
+      "loss": 1.1374,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.7487472295761108,
+      "rewards/margins": 0.1312863826751709,
+      "rewards/rejected": -1.8800337314605713,
+      "step": 3880
+    },
+    {
+      "epoch": 0.6702274293590628,
+      "grad_norm": 9.773622663865877,
+      "learning_rate": 4.2043512752724265e-08,
+      "logits/chosen": -3.537513017654419,
+      "logits/rejected": -3.5205817222595215,
+      "logps/chosen": -1.6788158416748047,
+      "logps/rejected": -1.9103587865829468,
+      "loss": 1.0822,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.6788158416748047,
+      "rewards/margins": 0.23154297471046448,
+      "rewards/rejected": -1.9103587865829468,
+      "step": 3890
+    },
+    {
+      "epoch": 0.6719503790489317,
+      "grad_norm": 11.265522895012584,
+      "learning_rate": 4.19884350418411e-08,
+      "logits/chosen": -3.5284111499786377,
+      "logits/rejected": -3.4977753162384033,
+      "logps/chosen": -1.6665042638778687,
+      "logps/rejected": -1.9607744216918945,
+      "loss": 1.0318,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.6665042638778687,
+      "rewards/margins": 0.2942701280117035,
+      "rewards/rejected": -1.9607744216918945,
+      "step": 3900
+    },
+    {
+      "epoch": 0.6719503790489317,
+      "eval_logits/chosen": -3.5620007514953613,
+      "eval_logits/rejected": -3.558152437210083,
+      "eval_logps/chosen": -1.5994967222213745,
+      "eval_logps/rejected": -1.753262996673584,
+      "eval_loss": 1.103891372680664,
+      "eval_rewards/accuracies": 0.6126858592033386,
+      "eval_rewards/chosen": -1.5994967222213745,
+      "eval_rewards/margins": 0.15376614034175873,
+      "eval_rewards/rejected": -1.753262996673584,
+      "eval_runtime": 155.3909,
+      "eval_samples_per_second": 27.698,
+      "eval_steps_per_second": 3.462,
+      "step": 3900
+    },
+    {
+      "epoch": 0.6736733287388008,
+      "grad_norm": 11.380342604713436,
+      "learning_rate": 4.1933203703231764e-08,
+      "logits/chosen": -3.5039591789245605,
+      "logits/rejected": -3.494279384613037,
+      "logps/chosen": -1.7250614166259766,
+      "logps/rejected": -1.9287946224212646,
+      "loss": 1.0829,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.7250614166259766,
+      "rewards/margins": 0.20373348891735077,
+      "rewards/rejected": -1.9287946224212646,
+      "step": 3910
+    },
+    {
+      "epoch": 0.6753962784286699,
+      "grad_norm": 11.803752755950043,
+      "learning_rate": 4.187781923635753e-08,
+      "logits/chosen": -3.555239200592041,
+      "logits/rejected": -3.5302090644836426,
+      "logps/chosen": -1.6503461599349976,
+      "logps/rejected": -1.9322706460952759,
+      "loss": 1.0745,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.6503461599349976,
+      "rewards/margins": 0.281924307346344,
+      "rewards/rejected": -1.9322706460952759,
+      "step": 3920
+    },
+    {
+      "epoch": 0.677119228118539,
+      "grad_norm": 10.379543220711142,
+      "learning_rate": 4.182228214206437e-08,
+      "logits/chosen": -3.562861919403076,
+      "logits/rejected": -3.5602402687072754,
+      "logps/chosen": -1.6659927368164062,
+      "logps/rejected": -1.857887864112854,
+      "loss": 1.0974,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.6659927368164062,
+      "rewards/margins": 0.19189508259296417,
+      "rewards/rejected": -1.857887864112854,
+      "step": 3930
+    },
+    {
+      "epoch": 0.6788421778084079,
+      "grad_norm": 9.991547142898089,
+      "learning_rate": 4.176659292257853e-08,
+      "logits/chosen": -3.4460785388946533,
+      "logits/rejected": -3.4367377758026123,
+      "logps/chosen": -1.6825830936431885,
+      "logps/rejected": -1.9281036853790283,
+      "loss": 1.0585,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.6825830936431885,
+      "rewards/margins": 0.24552056193351746,
+      "rewards/rejected": -1.9281036853790283,
+      "step": 3940
+    },
+    {
+      "epoch": 0.680565127498277,
+      "grad_norm": 10.866985341526144,
+      "learning_rate": 4.1710752081501877e-08,
+      "logits/chosen": -3.4561667442321777,
+      "logits/rejected": -3.429983615875244,
+      "logps/chosen": -1.6621586084365845,
+      "logps/rejected": -1.9186832904815674,
+      "loss": 1.0549,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.6621586084365845,
+      "rewards/margins": 0.25652453303337097,
+      "rewards/rejected": -1.9186832904815674,
+      "step": 3950
+    },
+    {
+      "epoch": 0.6822880771881461,
+      "grad_norm": 8.400064679666926,
+      "learning_rate": 4.1654760123807465e-08,
+      "logits/chosen": -3.5064749717712402,
+      "logits/rejected": -3.507322311401367,
+      "logps/chosen": -1.6336743831634521,
+      "logps/rejected": -1.8801929950714111,
+      "loss": 1.0832,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.6336743831634521,
+      "rewards/margins": 0.2465183287858963,
+      "rewards/rejected": -1.8801929950714111,
+      "step": 3960
+    },
+    {
+      "epoch": 0.6840110268780152,
+      "grad_norm": 9.845851740342123,
+      "learning_rate": 4.1598617555834866e-08,
+      "logits/chosen": -3.527547836303711,
+      "logits/rejected": -3.512822389602661,
+      "logps/chosen": -1.6713902950286865,
+      "logps/rejected": -1.934462547302246,
+      "loss": 1.0726,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.6713902950286865,
+      "rewards/margins": 0.26307258009910583,
+      "rewards/rejected": -1.934462547302246,
+      "step": 3970
+    },
+    {
+      "epoch": 0.6857339765678843,
+      "grad_norm": 11.605061684464864,
+      "learning_rate": 4.1542324885285656e-08,
+      "logits/chosen": -3.4362540245056152,
+      "logits/rejected": -3.4242451190948486,
+      "logps/chosen": -1.7763773202896118,
+      "logps/rejected": -1.987749695777893,
+      "loss": 1.0977,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.7763773202896118,
+      "rewards/margins": 0.2113722562789917,
+      "rewards/rejected": -1.987749695777893,
+      "step": 3980
+    },
+    {
+      "epoch": 0.6874569262577532,
+      "grad_norm": 9.640041452559977,
+      "learning_rate": 4.148588262121877e-08,
+      "logits/chosen": -3.516035556793213,
+      "logits/rejected": -3.5099308490753174,
+      "logps/chosen": -1.6792304515838623,
+      "logps/rejected": -1.9388787746429443,
+      "loss": 1.0519,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.6792304515838623,
+      "rewards/margins": 0.25964832305908203,
+      "rewards/rejected": -1.9388787746429443,
+      "step": 3990
+    },
+    {
+      "epoch": 0.6891798759476223,
+      "grad_norm": 11.420832986418416,
+      "learning_rate": 4.1429291274045966e-08,
+      "logits/chosen": -3.5490174293518066,
+      "logits/rejected": -3.5300650596618652,
+      "logps/chosen": -1.7465951442718506,
+      "logps/rejected": -1.9396660327911377,
+      "loss": 1.1154,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -1.7465951442718506,
+      "rewards/margins": 0.1930708885192871,
+      "rewards/rejected": -1.9396660327911377,
+      "step": 4000
+    },
+    {
+      "epoch": 0.6891798759476223,
+      "eval_logits/chosen": -3.5610930919647217,
+      "eval_logits/rejected": -3.557279109954834,
+      "eval_logps/chosen": -1.615576148033142,
+      "eval_logps/rejected": -1.7711799144744873,
+      "eval_loss": 1.102990984916687,
+      "eval_rewards/accuracies": 0.6166356801986694,
+      "eval_rewards/chosen": -1.615576148033142,
+      "eval_rewards/margins": 0.1556037813425064,
+      "eval_rewards/rejected": -1.7711799144744873,
+      "eval_runtime": 155.426,
+      "eval_samples_per_second": 27.692,
+      "eval_steps_per_second": 3.461,
+      "step": 4000
+    },
+    {
+      "epoch": 0.6909028256374914,
+      "grad_norm": 9.375781669957787,
+      "learning_rate": 4.137255135552714e-08,
+      "logits/chosen": -3.4829628467559814,
+      "logits/rejected": -3.478179454803467,
+      "logps/chosen": -1.6291637420654297,
+      "logps/rejected": -1.932287573814392,
+      "loss": 1.0403,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.6291637420654297,
+      "rewards/margins": 0.30312392115592957,
+      "rewards/rejected": -1.932287573814392,
+      "step": 4010
+    },
+    {
+      "epoch": 0.6926257753273605,
+      "grad_norm": 8.950147667903238,
+      "learning_rate": 4.131566337876575e-08,
+      "logits/chosen": -3.488407611846924,
+      "logits/rejected": -3.479485273361206,
+      "logps/chosen": -1.7210981845855713,
+      "logps/rejected": -1.9516842365264893,
+      "loss": 1.0948,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.7210981845855713,
+      "rewards/margins": 0.23058578372001648,
+      "rewards/rejected": -1.9516842365264893,
+      "step": 4020
+    },
+    {
+      "epoch": 0.6943487250172296,
+      "grad_norm": 8.668064339379706,
+      "learning_rate": 4.1258627858204156e-08,
+      "logits/chosen": -3.470398426055908,
+      "logits/rejected": -3.4570069313049316,
+      "logps/chosen": -1.707562804222107,
+      "logps/rejected": -2.0530216693878174,
+      "loss": 1.0246,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.707562804222107,
+      "rewards/margins": 0.3454587459564209,
+      "rewards/rejected": -2.0530216693878174,
+      "step": 4030
+    },
+    {
+      "epoch": 0.6960716747070985,
+      "grad_norm": 9.871767376782332,
+      "learning_rate": 4.1201445309618957e-08,
+      "logits/chosen": -3.5441811084747314,
+      "logits/rejected": -3.537202835083008,
+      "logps/chosen": -1.7741960287094116,
+      "logps/rejected": -1.8475137948989868,
+      "loss": 1.1712,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -1.7741960287094116,
+      "rewards/margins": 0.07331773638725281,
+      "rewards/rejected": -1.8475137948989868,
+      "step": 4040
+    },
+    {
+      "epoch": 0.6977946243969676,
+      "grad_norm": 9.884890273300808,
+      "learning_rate": 4.114411625011634e-08,
+      "logits/chosen": -3.465610980987549,
+      "logits/rejected": -3.4555821418762207,
+      "logps/chosen": -1.6617422103881836,
+      "logps/rejected": -1.915371298789978,
+      "loss": 1.0744,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.6617422103881836,
+      "rewards/margins": 0.25362899899482727,
+      "rewards/rejected": -1.915371298789978,
+      "step": 4050
+    },
+    {
+      "epoch": 0.6995175740868367,
+      "grad_norm": 10.090306673046,
+      "learning_rate": 4.10866411981274e-08,
+      "logits/chosen": -3.484386920928955,
+      "logits/rejected": -3.466063976287842,
+      "logps/chosen": -1.7678349018096924,
+      "logps/rejected": -1.9891035556793213,
+      "loss": 1.0865,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.7678349018096924,
+      "rewards/margins": 0.22126856446266174,
+      "rewards/rejected": -1.9891035556793213,
+      "step": 4060
+    },
+    {
+      "epoch": 0.7012405237767058,
+      "grad_norm": 10.848614583509438,
+      "learning_rate": 4.102902067340348e-08,
+      "logits/chosen": -3.4902634620666504,
+      "logits/rejected": -3.477780818939209,
+      "logps/chosen": -1.7855924367904663,
+      "logps/rejected": -1.9826233386993408,
+      "loss": 1.0992,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.7855924367904663,
+      "rewards/margins": 0.1970309317111969,
+      "rewards/rejected": -1.9826233386993408,
+      "step": 4070
+    },
+    {
+      "epoch": 0.7029634734665747,
+      "grad_norm": 10.017771478135856,
+      "learning_rate": 4.0971255197011395e-08,
+      "logits/chosen": -3.4687752723693848,
+      "logits/rejected": -3.4648425579071045,
+      "logps/chosen": -1.6485637426376343,
+      "logps/rejected": -1.8432992696762085,
+      "loss": 1.0897,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.6485637426376343,
+      "rewards/margins": 0.19473548233509064,
+      "rewards/rejected": -1.8432992696762085,
+      "step": 4080
+    },
+    {
+      "epoch": 0.7046864231564438,
+      "grad_norm": 10.60201994127666,
+      "learning_rate": 4.091334529132881e-08,
+      "logits/chosen": -3.5255050659179688,
+      "logits/rejected": -3.5078017711639404,
+      "logps/chosen": -1.7701562643051147,
+      "logps/rejected": -2.0106351375579834,
+      "loss": 1.0648,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.7701562643051147,
+      "rewards/margins": 0.24047903716564178,
+      "rewards/rejected": -2.0106351375579834,
+      "step": 4090
+    },
+    {
+      "epoch": 0.7064093728463129,
+      "grad_norm": 10.80016247748972,
+      "learning_rate": 4.085529148003945e-08,
+      "logits/chosen": -3.4800727367401123,
+      "logits/rejected": -3.464503049850464,
+      "logps/chosen": -1.5958360433578491,
+      "logps/rejected": -1.8165090084075928,
+      "loss": 1.0646,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.5958360433578491,
+      "rewards/margins": 0.2206730842590332,
+      "rewards/rejected": -1.8165090084075928,
+      "step": 4100
+    },
+    {
+      "epoch": 0.7064093728463129,
+      "eval_logits/chosen": -3.548285484313965,
+      "eval_logits/rejected": -3.544435977935791,
+      "eval_logps/chosen": -1.6233590841293335,
+      "eval_logps/rejected": -1.7803820371627808,
+      "eval_loss": 1.1022820472717285,
+      "eval_rewards/accuracies": 0.6177973747253418,
+      "eval_rewards/chosen": -1.6233590841293335,
+      "eval_rewards/margins": 0.15702292323112488,
+      "eval_rewards/rejected": -1.7803820371627808,
+      "eval_runtime": 155.2084,
+      "eval_samples_per_second": 27.73,
+      "eval_steps_per_second": 3.466,
+      "step": 4100
+    },
+    {
+      "epoch": 0.708132322536182,
+      "grad_norm": 11.16798297478165,
+      "learning_rate": 4.079709428812842e-08,
+      "logits/chosen": -3.477321147918701,
+      "logits/rejected": -3.470987319946289,
+      "logps/chosen": -1.8325397968292236,
+      "logps/rejected": -1.878116250038147,
+      "loss": 1.1889,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -1.8325397968292236,
+      "rewards/margins": 0.04557663947343826,
+      "rewards/rejected": -1.878116250038147,
+      "step": 4110
+    },
+    {
+      "epoch": 0.709855272226051,
+      "grad_norm": 9.341495636777616,
+      "learning_rate": 4.073875424187739e-08,
+      "logits/chosen": -3.447930097579956,
+      "logits/rejected": -3.446903705596924,
+      "logps/chosen": -1.7248780727386475,
+      "logps/rejected": -1.9381649494171143,
+      "loss": 1.0893,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.7248780727386475,
+      "rewards/margins": 0.21328675746917725,
+      "rewards/rejected": -1.9381649494171143,
+      "step": 4120
+    },
+    {
+      "epoch": 0.71157822191592,
+      "grad_norm": 10.866076457068672,
+      "learning_rate": 4.06802718688599e-08,
+      "logits/chosen": -3.507199764251709,
+      "logits/rejected": -3.492269515991211,
+      "logps/chosen": -1.7427746057510376,
+      "logps/rejected": -1.9649732112884521,
+      "loss": 1.0842,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.7427746057510376,
+      "rewards/margins": 0.22219884395599365,
+      "rewards/rejected": -1.9649732112884521,
+      "step": 4130
+    },
+    {
+      "epoch": 0.7133011716057891,
+      "grad_norm": 10.138709987096712,
+      "learning_rate": 4.0621647697936555e-08,
+      "logits/chosen": -3.504094362258911,
+      "logits/rejected": -3.4866111278533936,
+      "logps/chosen": -1.7355225086212158,
+      "logps/rejected": -1.8569231033325195,
+      "loss": 1.1524,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -1.7355225086212158,
+      "rewards/margins": 0.12140093743801117,
+      "rewards/rejected": -1.8569231033325195,
+      "step": 4140
+    },
+    {
+      "epoch": 0.7150241212956582,
+      "grad_norm": 10.110785807757093,
+      "learning_rate": 4.056288225925023e-08,
+      "logits/chosen": -3.5140902996063232,
+      "logits/rejected": -3.4977307319641113,
+      "logps/chosen": -1.803004503250122,
+      "logps/rejected": -2.011648654937744,
+      "loss": 1.1066,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -1.803004503250122,
+      "rewards/margins": 0.2086443454027176,
+      "rewards/rejected": -2.011648654937744,
+      "step": 4150
+    },
+    {
+      "epoch": 0.7167470709855273,
+      "grad_norm": 9.317425005209996,
+      "learning_rate": 4.050397608422132e-08,
+      "logits/chosen": -3.4422011375427246,
+      "logits/rejected": -3.4241394996643066,
+      "logps/chosen": -1.7173618078231812,
+      "logps/rejected": -1.9993969202041626,
+      "loss": 1.0304,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.7173618078231812,
+      "rewards/margins": 0.28203505277633667,
+      "rewards/rejected": -1.9993969202041626,
+      "step": 4160
+    },
+    {
+      "epoch": 0.7184700206753962,
+      "grad_norm": 10.856149991959583,
+      "learning_rate": 4.044492970554292e-08,
+      "logits/chosen": -3.472992420196533,
+      "logits/rejected": -3.4685134887695312,
+      "logps/chosen": -1.7469717264175415,
+      "logps/rejected": -1.8684556484222412,
+      "loss": 1.1372,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.7469717264175415,
+      "rewards/margins": 0.12148382514715195,
+      "rewards/rejected": -1.8684556484222412,
+      "step": 4170
+    },
+    {
+      "epoch": 0.7201929703652653,
+      "grad_norm": 11.558604120378405,
+      "learning_rate": 4.038574365717594e-08,
+      "logits/chosen": -3.487257719039917,
+      "logits/rejected": -3.4806315898895264,
+      "logps/chosen": -1.744690179824829,
+      "logps/rejected": -1.946305274963379,
+      "loss": 1.1017,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.744690179824829,
+      "rewards/margins": 0.20161540806293488,
+      "rewards/rejected": -1.946305274963379,
+      "step": 4180
+    },
+    {
+      "epoch": 0.7219159200551344,
+      "grad_norm": 9.508676908072454,
+      "learning_rate": 4.0326418474344414e-08,
+      "logits/chosen": -3.4731414318084717,
+      "logits/rejected": -3.464491605758667,
+      "logps/chosen": -1.66921067237854,
+      "logps/rejected": -1.9249536991119385,
+      "loss": 1.0566,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.66921067237854,
+      "rewards/margins": 0.25574296712875366,
+      "rewards/rejected": -1.9249536991119385,
+      "step": 4190
+    },
+    {
+      "epoch": 0.7236388697450035,
+      "grad_norm": 11.141194012745999,
+      "learning_rate": 4.026695469353051e-08,
+      "logits/chosen": -3.501142978668213,
+      "logits/rejected": -3.492615222930908,
+      "logps/chosen": -1.7372791767120361,
+      "logps/rejected": -1.8630377054214478,
+      "loss": 1.1369,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.7372791767120361,
+      "rewards/margins": 0.12575849890708923,
+      "rewards/rejected": -1.8630377054214478,
+      "step": 4200
+    },
+    {
+      "epoch": 0.7236388697450035,
+      "eval_logits/chosen": -3.5471296310424805,
+      "eval_logits/rejected": -3.543299913406372,
+      "eval_logps/chosen": -1.6359500885009766,
+      "eval_logps/rejected": -1.7943590879440308,
+      "eval_loss": 1.1016526222229004,
+      "eval_rewards/accuracies": 0.6171003580093384,
+      "eval_rewards/chosen": -1.6359500885009766,
+      "eval_rewards/margins": 0.1584090292453766,
+      "eval_rewards/rejected": -1.7943590879440308,
+      "eval_runtime": 155.4433,
+      "eval_samples_per_second": 27.689,
+      "eval_steps_per_second": 3.461,
+      "step": 4200
+    },
+    {
+      "epoch": 0.7253618194348725,
+      "grad_norm": 9.530345882187067,
+      "learning_rate": 4.020735285246979e-08,
+      "logits/chosen": -3.4964547157287598,
+      "logits/rejected": -3.481815814971924,
+      "logps/chosen": -1.762404441833496,
+      "logps/rejected": -1.9780311584472656,
+      "loss": 1.1009,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.762404441833496,
+      "rewards/margins": 0.21562668681144714,
+      "rewards/rejected": -1.9780311584472656,
+      "step": 4210
+    },
+    {
+      "epoch": 0.7270847691247415,
+      "grad_norm": 8.474988982917598,
+      "learning_rate": 4.0147613490146285e-08,
+      "logits/chosen": -3.4035377502441406,
+      "logits/rejected": -3.3922953605651855,
+      "logps/chosen": -1.728161096572876,
+      "logps/rejected": -2.0109848976135254,
+      "loss": 1.0325,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.728161096572876,
+      "rewards/margins": 0.28282418847084045,
+      "rewards/rejected": -2.0109848976135254,
+      "step": 4220
+    },
+    {
+      "epoch": 0.7288077188146106,
+      "grad_norm": 9.651494514007785,
+      "learning_rate": 4.0087737146787653e-08,
+      "logits/chosen": -3.5112946033477783,
+      "logits/rejected": -3.5007729530334473,
+      "logps/chosen": -1.786633849143982,
+      "logps/rejected": -2.0032341480255127,
+      "loss": 1.0873,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.786633849143982,
+      "rewards/margins": 0.21660038828849792,
+      "rewards/rejected": -2.0032341480255127,
+      "step": 4230
+    },
+    {
+      "epoch": 0.7305306685044797,
+      "grad_norm": 10.092504996087309,
+      "learning_rate": 4.002772436386027e-08,
+      "logits/chosen": -3.4621481895446777,
+      "logits/rejected": -3.447965145111084,
+      "logps/chosen": -1.751190423965454,
+      "logps/rejected": -2.0880682468414307,
+      "loss": 1.0131,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.751190423965454,
+      "rewards/margins": 0.33687785267829895,
+      "rewards/rejected": -2.0880682468414307,
+      "step": 4240
+    },
+    {
+      "epoch": 0.7322536181943488,
+      "grad_norm": 11.905020099904768,
+      "learning_rate": 3.996757568406437e-08,
+      "logits/chosen": -3.4715542793273926,
+      "logits/rejected": -3.4636166095733643,
+      "logps/chosen": -1.8017956018447876,
+      "logps/rejected": -1.9378858804702759,
+      "loss": 1.135,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.8017956018447876,
+      "rewards/margins": 0.13609029352664948,
+      "rewards/rejected": -1.9378858804702759,
+      "step": 4250
+    },
+    {
+      "epoch": 0.7339765678842178,
+      "grad_norm": 10.033774745911208,
+      "learning_rate": 3.990729165132907e-08,
+      "logits/chosen": -3.444941282272339,
+      "logits/rejected": -3.4354660511016846,
+      "logps/chosen": -1.7430578470230103,
+      "logps/rejected": -1.9357764720916748,
+      "loss": 1.1027,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.7430578470230103,
+      "rewards/margins": 0.19271886348724365,
+      "rewards/rejected": -1.9357764720916748,
+      "step": 4260
+    },
+    {
+      "epoch": 0.7356995175740868,
+      "grad_norm": 10.2233898859266,
+      "learning_rate": 3.9846872810807543e-08,
+      "logits/chosen": -3.4620585441589355,
+      "logits/rejected": -3.4471962451934814,
+      "logps/chosen": -1.6581662893295288,
+      "logps/rejected": -1.9226531982421875,
+      "loss": 1.0682,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.6581662893295288,
+      "rewards/margins": 0.2644868493080139,
+      "rewards/rejected": -1.9226531982421875,
+      "step": 4270
+    },
+    {
+      "epoch": 0.7374224672639559,
+      "grad_norm": 8.316274323432712,
+      "learning_rate": 3.978631970887201e-08,
+      "logits/chosen": -3.4317240715026855,
+      "logits/rejected": -3.4152092933654785,
+      "logps/chosen": -1.744306206703186,
+      "logps/rejected": -2.059614896774292,
+      "loss": 1.0276,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.744306206703186,
+      "rewards/margins": 0.3153088390827179,
+      "rewards/rejected": -2.059614896774292,
+      "step": 4280
+    },
+    {
+      "epoch": 0.739145416953825,
+      "grad_norm": 10.386076944721982,
+      "learning_rate": 3.9725632893108816e-08,
+      "logits/chosen": -3.472008466720581,
+      "logits/rejected": -3.4581007957458496,
+      "logps/chosen": -1.7657129764556885,
+      "logps/rejected": -1.943905234336853,
+      "loss": 1.1136,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.7657129764556885,
+      "rewards/margins": 0.17819233238697052,
+      "rewards/rejected": -1.943905234336853,
+      "step": 4290
+    },
+    {
+      "epoch": 0.740868366643694,
+      "grad_norm": 8.973853034703392,
+      "learning_rate": 3.9664812912313536e-08,
+      "logits/chosen": -3.474567413330078,
+      "logits/rejected": -3.4687671661376953,
+      "logps/chosen": -1.787796974182129,
+      "logps/rejected": -1.9654314517974854,
+      "loss": 1.0954,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.787796974182129,
+      "rewards/margins": 0.1776348203420639,
+      "rewards/rejected": -1.9654314517974854,
+      "step": 4300
+    },
+    {
+      "epoch": 0.740868366643694,
+      "eval_logits/chosen": -3.5243797302246094,
+      "eval_logits/rejected": -3.520461082458496,
+      "eval_logps/chosen": -1.6440356969833374,
+      "eval_logps/rejected": -1.8032548427581787,
+      "eval_loss": 1.1013259887695312,
+      "eval_rewards/accuracies": 0.6182620525360107,
+      "eval_rewards/chosen": -1.6440356969833374,
+      "eval_rewards/margins": 0.15921920537948608,
+      "eval_rewards/rejected": -1.8032548427581787,
+      "eval_runtime": 155.4234,
+      "eval_samples_per_second": 27.692,
+      "eval_steps_per_second": 3.462,
+      "step": 4300
+    },
+    {
+      "epoch": 0.742591316333563,
+      "grad_norm": 11.16489937908736,
+      "learning_rate": 3.960386031648592e-08,
+      "logits/chosen": -3.4517784118652344,
+      "logits/rejected": -3.440960645675659,
+      "logps/chosen": -1.7798258066177368,
+      "logps/rejected": -1.8897091150283813,
+      "loss": 1.1436,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.7798258066177368,
+      "rewards/margins": 0.10988356173038483,
+      "rewards/rejected": -1.8897091150283813,
+      "step": 4310
+    },
+    {
+      "epoch": 0.7443142660234321,
+      "grad_norm": 10.52673288740027,
+      "learning_rate": 3.9542775656825e-08,
+      "logits/chosen": -3.501256227493286,
+      "logits/rejected": -3.485825777053833,
+      "logps/chosen": -1.8022257089614868,
+      "logps/rejected": -2.0387346744537354,
+      "loss": 1.0709,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.8022257089614868,
+      "rewards/margins": 0.2365090399980545,
+      "rewards/rejected": -2.0387346744537354,
+      "step": 4320
+    },
+    {
+      "epoch": 0.7460372157133012,
+      "grad_norm": 9.667572318920904,
+      "learning_rate": 3.9481559485724046e-08,
+      "logits/chosen": -3.4316906929016113,
+      "logits/rejected": -3.4139599800109863,
+      "logps/chosen": -1.8384513854980469,
+      "logps/rejected": -2.0212197303771973,
+      "loss": 1.0969,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.8384513854980469,
+      "rewards/margins": 0.18276794254779816,
+      "rewards/rejected": -2.0212197303771973,
+      "step": 4330
+    },
+    {
+      "epoch": 0.7477601654031703,
+      "grad_norm": 11.030480508990632,
+      "learning_rate": 3.942021235676561e-08,
+      "logits/chosen": -3.4466850757598877,
+      "logits/rejected": -3.438957929611206,
+      "logps/chosen": -1.7432401180267334,
+      "logps/rejected": -2.0190703868865967,
+      "loss": 1.0633,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.7432401180267334,
+      "rewards/margins": 0.2758301794528961,
+      "rewards/rejected": -2.0190703868865967,
+      "step": 4340
+    },
+    {
+      "epoch": 0.7494831150930393,
+      "grad_norm": 10.945711552714549,
+      "learning_rate": 3.93587348247165e-08,
+      "logits/chosen": -3.45192289352417,
+      "logits/rejected": -3.441998243331909,
+      "logps/chosen": -1.635253667831421,
+      "logps/rejected": -1.8944756984710693,
+      "loss": 1.0513,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.635253667831421,
+      "rewards/margins": 0.2592221200466156,
+      "rewards/rejected": -1.8944756984710693,
+      "step": 4350
+    },
+    {
+      "epoch": 0.7512060647829083,
+      "grad_norm": 8.955615395258187,
+      "learning_rate": 3.929712744552278e-08,
+      "logits/chosen": -3.47450590133667,
+      "logits/rejected": -3.462526321411133,
+      "logps/chosen": -1.7715518474578857,
+      "logps/rejected": -1.9477853775024414,
+      "loss": 1.1038,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.7715518474578857,
+      "rewards/margins": 0.1762334555387497,
+      "rewards/rejected": -1.9477853775024414,
+      "step": 4360
+    },
+    {
+      "epoch": 0.7529290144727774,
+      "grad_norm": 9.238012621315479,
+      "learning_rate": 3.923539077630471e-08,
+      "logits/chosen": -3.4835751056671143,
+      "logits/rejected": -3.4768428802490234,
+      "logps/chosen": -1.7702239751815796,
+      "logps/rejected": -1.8950402736663818,
+      "loss": 1.1377,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.7702239751815796,
+      "rewards/margins": 0.12481584399938583,
+      "rewards/rejected": -1.8950402736663818,
+      "step": 4370
+    },
+    {
+      "epoch": 0.7546519641626465,
+      "grad_norm": 11.260804783703891,
+      "learning_rate": 3.917352537535176e-08,
+      "logits/chosen": -3.495016098022461,
+      "logits/rejected": -3.4817092418670654,
+      "logps/chosen": -1.7002780437469482,
+      "logps/rejected": -1.8760570287704468,
+      "loss": 1.0978,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.7002780437469482,
+      "rewards/margins": 0.17577899992465973,
+      "rewards/rejected": -1.8760570287704468,
+      "step": 4380
+    },
+    {
+      "epoch": 0.7563749138525155,
+      "grad_norm": 10.414830409687998,
+      "learning_rate": 3.91115318021175e-08,
+      "logits/chosen": -3.455268383026123,
+      "logits/rejected": -3.4477951526641846,
+      "logps/chosen": -1.7695268392562866,
+      "logps/rejected": -1.9809064865112305,
+      "loss": 1.088,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.7695268392562866,
+      "rewards/margins": 0.21137969195842743,
+      "rewards/rejected": -1.9809064865112305,
+      "step": 4390
+    },
+    {
+      "epoch": 0.7580978635423845,
+      "grad_norm": 10.315026323112427,
+      "learning_rate": 3.9049410617214604e-08,
+      "logits/chosen": -3.465050458908081,
+      "logits/rejected": -3.4550247192382812,
+      "logps/chosen": -1.7654104232788086,
+      "logps/rejected": -1.9352848529815674,
+      "loss": 1.1088,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.7654104232788086,
+      "rewards/margins": 0.16987448930740356,
+      "rewards/rejected": -1.9352848529815674,
+      "step": 4400
+    },
+    {
+      "epoch": 0.7580978635423845,
+      "eval_logits/chosen": -3.530872344970703,
+      "eval_logits/rejected": -3.527012825012207,
+      "eval_logps/chosen": -1.6538678407669067,
+      "eval_logps/rejected": -1.8142768144607544,
+      "eval_loss": 1.1007695198059082,
+      "eval_rewards/accuracies": 0.6175650358200073,
+      "eval_rewards/chosen": -1.6538678407669067,
+      "eval_rewards/margins": 0.16040906310081482,
+      "eval_rewards/rejected": -1.8142768144607544,
+      "eval_runtime": 155.669,
+      "eval_samples_per_second": 27.648,
+      "eval_steps_per_second": 3.456,
+      "step": 4400
+    },
+    {
+      "epoch": 0.7598208132322536,
+      "grad_norm": 10.365162277940419,
+      "learning_rate": 3.898716238240971e-08,
+      "logits/chosen": -3.4519762992858887,
+      "logits/rejected": -3.443540096282959,
+      "logps/chosen": -1.862732172012329,
+      "logps/rejected": -1.9926360845565796,
+      "loss": 1.138,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.862732172012329,
+      "rewards/margins": 0.12990400195121765,
+      "rewards/rejected": -1.9926360845565796,
+      "step": 4410
+    },
+    {
+      "epoch": 0.7615437629221227,
+      "grad_norm": 10.046481990807624,
+      "learning_rate": 3.892478766061841e-08,
+      "logits/chosen": -3.514650344848633,
+      "logits/rejected": -3.4912891387939453,
+      "logps/chosen": -1.6688110828399658,
+      "logps/rejected": -1.9862983226776123,
+      "loss": 1.0198,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.6688110828399658,
+      "rewards/margins": 0.3174872100353241,
+      "rewards/rejected": -1.9862983226776123,
+      "step": 4420
+    },
+    {
+      "epoch": 0.7632667126119917,
+      "grad_norm": 9.744721537221654,
+      "learning_rate": 3.886228701590011e-08,
+      "logits/chosen": -3.4433434009552,
+      "logits/rejected": -3.426525831222534,
+      "logps/chosen": -1.7158515453338623,
+      "logps/rejected": -1.944561243057251,
+      "loss": 1.0817,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.7158515453338623,
+      "rewards/margins": 0.22870993614196777,
+      "rewards/rejected": -1.944561243057251,
+      "step": 4430
+    },
+    {
+      "epoch": 0.7649896623018608,
+      "grad_norm": 11.163981788135068,
+      "learning_rate": 3.879966101345296e-08,
+      "logits/chosen": -3.4932219982147217,
+      "logits/rejected": -3.4750800132751465,
+      "logps/chosen": -1.819690465927124,
+      "logps/rejected": -2.046234369277954,
+      "loss": 1.0728,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.819690465927124,
+      "rewards/margins": 0.22654375433921814,
+      "rewards/rejected": -2.046234369277954,
+      "step": 4440
+    },
+    {
+      "epoch": 0.7667126119917298,
+      "grad_norm": 10.210732278445336,
+      "learning_rate": 3.8736910219608706e-08,
+      "logits/chosen": -3.411233901977539,
+      "logits/rejected": -3.4038376808166504,
+      "logps/chosen": -1.7966740131378174,
+      "logps/rejected": -2.003030300140381,
+      "loss": 1.0996,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.7966740131378174,
+      "rewards/margins": 0.206356480717659,
+      "rewards/rejected": -2.003030300140381,
+      "step": 4450
+    },
+    {
+      "epoch": 0.7684355616815989,
+      "grad_norm": 11.775207876383964,
+      "learning_rate": 3.867403520182762e-08,
+      "logits/chosen": -3.4809887409210205,
+      "logits/rejected": -3.4763259887695312,
+      "logps/chosen": -1.842728853225708,
+      "logps/rejected": -1.9935181140899658,
+      "loss": 1.138,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.842728853225708,
+      "rewards/margins": 0.15078909695148468,
+      "rewards/rejected": -1.9935181140899658,
+      "step": 4460
+    },
+    {
+      "epoch": 0.770158511371468,
+      "grad_norm": 9.631865411731418,
+      "learning_rate": 3.861103652869334e-08,
+      "logits/chosen": -3.5009982585906982,
+      "logits/rejected": -3.4830079078674316,
+      "logps/chosen": -1.8187406063079834,
+      "logps/rejected": -2.04398775100708,
+      "loss": 1.0823,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.8187406063079834,
+      "rewards/margins": 0.22524718940258026,
+      "rewards/rejected": -2.04398775100708,
+      "step": 4470
+    },
+    {
+      "epoch": 0.771881461061337,
+      "grad_norm": 10.10665641289171,
+      "learning_rate": 3.854791476990771e-08,
+      "logits/chosen": -3.525630235671997,
+      "logits/rejected": -3.5232086181640625,
+      "logps/chosen": -1.8044286966323853,
+      "logps/rejected": -1.962071418762207,
+      "loss": 1.1127,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.8044286966323853,
+      "rewards/margins": 0.15764297544956207,
+      "rewards/rejected": -1.962071418762207,
+      "step": 4480
+    },
+    {
+      "epoch": 0.7736044107512061,
+      "grad_norm": 10.980121504826828,
+      "learning_rate": 3.848467049628564e-08,
+      "logits/chosen": -3.448246717453003,
+      "logits/rejected": -3.4335598945617676,
+      "logps/chosen": -1.7602516412734985,
+      "logps/rejected": -2.0321569442749023,
+      "loss": 1.0786,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.7602516412734985,
+      "rewards/margins": 0.27190548181533813,
+      "rewards/rejected": -2.0321569442749023,
+      "step": 4490
+    },
+    {
+      "epoch": 0.7753273604410751,
+      "grad_norm": 10.64098101902454,
+      "learning_rate": 3.842130427974998e-08,
+      "logits/chosen": -3.4661858081817627,
+      "logits/rejected": -3.4588005542755127,
+      "logps/chosen": -1.8359178304672241,
+      "logps/rejected": -1.9210259914398193,
+      "loss": 1.1572,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.8359178304672241,
+      "rewards/margins": 0.08510830998420715,
+      "rewards/rejected": -1.9210259914398193,
+      "step": 4500
+    },
+    {
+      "epoch": 0.7753273604410751,
+      "eval_logits/chosen": -3.5393993854522705,
+      "eval_logits/rejected": -3.535609483718872,
+      "eval_logps/chosen": -1.6681013107299805,
+      "eval_logps/rejected": -1.8300743103027344,
+      "eval_loss": 1.0999354124069214,
+      "eval_rewards/accuracies": 0.6205855011940002,
+      "eval_rewards/chosen": -1.6681013107299805,
+      "eval_rewards/margins": 0.1619730144739151,
+      "eval_rewards/rejected": -1.8300743103027344,
+      "eval_runtime": 155.4868,
+      "eval_samples_per_second": 27.681,
+      "eval_steps_per_second": 3.46,
+      "step": 4500
+    },
+    {
+      "epoch": 0.7770503101309442,
+      "grad_norm": 11.359497041220056,
+      "learning_rate": 3.835781669332631e-08,
+      "logits/chosen": -3.53251576423645,
+      "logits/rejected": -3.516831159591675,
+      "logps/chosen": -1.7082977294921875,
+      "logps/rejected": -2.032041311264038,
+      "loss": 1.0242,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.7082977294921875,
+      "rewards/margins": 0.3237438499927521,
+      "rewards/rejected": -2.032041311264038,
+      "step": 4510
+    },
+    {
+      "epoch": 0.7787732598208132,
+      "grad_norm": 11.762334160896607,
+      "learning_rate": 3.829420831113775e-08,
+      "logits/chosen": -3.485935688018799,
+      "logits/rejected": -3.4743430614471436,
+      "logps/chosen": -1.8578441143035889,
+      "logps/rejected": -2.0093960762023926,
+      "loss": 1.1217,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.8578441143035889,
+      "rewards/margins": 0.15155203640460968,
+      "rewards/rejected": -2.0093960762023926,
+      "step": 4520
+    },
+    {
+      "epoch": 0.7804962095106823,
+      "grad_norm": 10.317834472825727,
+      "learning_rate": 3.823047970839981e-08,
+      "logits/chosen": -3.492619276046753,
+      "logits/rejected": -3.4876441955566406,
+      "logps/chosen": -1.832643747329712,
+      "logps/rejected": -1.9411132335662842,
+      "loss": 1.1464,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -1.832643747329712,
+      "rewards/margins": 0.10846952348947525,
+      "rewards/rejected": -1.9411132335662842,
+      "step": 4530
+    },
+    {
+      "epoch": 0.7822191592005513,
+      "grad_norm": 13.18831571560031,
+      "learning_rate": 3.816663146141514e-08,
+      "logits/chosen": -3.4298501014709473,
+      "logits/rejected": -3.4227242469787598,
+      "logps/chosen": -1.7794711589813232,
+      "logps/rejected": -1.9804744720458984,
+      "loss": 1.1073,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.7794711589813232,
+      "rewards/margins": 0.20100298523902893,
+      "rewards/rejected": -1.9804744720458984,
+      "step": 4540
+    },
+    {
+      "epoch": 0.7839421088904204,
+      "grad_norm": 10.21827263065561,
+      "learning_rate": 3.810266414756836e-08,
+      "logits/chosen": -3.4949429035186768,
+      "logits/rejected": -3.48186993598938,
+      "logps/chosen": -1.761635184288025,
+      "logps/rejected": -1.9614530801773071,
+      "loss": 1.0759,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.761635184288025,
+      "rewards/margins": 0.19981786608695984,
+      "rewards/rejected": -1.9614530801773071,
+      "step": 4550
+    },
+    {
+      "epoch": 0.7856650585802895,
+      "grad_norm": 9.436789718524336,
+      "learning_rate": 3.803857834532081e-08,
+      "logits/chosen": -3.4266293048858643,
+      "logits/rejected": -3.4059841632843018,
+      "logps/chosen": -1.6917593479156494,
+      "logps/rejected": -1.9685176610946655,
+      "loss": 1.0464,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.6917593479156494,
+      "rewards/margins": 0.2767585813999176,
+      "rewards/rejected": -1.9685176610946655,
+      "step": 4560
+    },
+    {
+      "epoch": 0.7873880082701585,
+      "grad_norm": 10.274661649660851,
+      "learning_rate": 3.7974374634205344e-08,
+      "logits/chosen": -3.460648775100708,
+      "logits/rejected": -3.447291135787964,
+      "logps/chosen": -1.7827314138412476,
+      "logps/rejected": -2.049365520477295,
+      "loss": 1.0482,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.7827314138412476,
+      "rewards/margins": 0.2666339576244354,
+      "rewards/rejected": -2.049365520477295,
+      "step": 4570
+    },
+    {
+      "epoch": 0.7891109579600276,
+      "grad_norm": 9.173192155816976,
+      "learning_rate": 3.791005359482106e-08,
+      "logits/chosen": -3.4273324012756348,
+      "logits/rejected": -3.4114456176757812,
+      "logps/chosen": -1.6942014694213867,
+      "logps/rejected": -1.8647798299789429,
+      "loss": 1.0993,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.6942014694213867,
+      "rewards/margins": 0.1705784797668457,
+      "rewards/rejected": -1.8647798299789429,
+      "step": 4580
+    },
+    {
+      "epoch": 0.7908339076498966,
+      "grad_norm": 11.743004104057253,
+      "learning_rate": 3.7845615808828055e-08,
+      "logits/chosen": -3.492399215698242,
+      "logits/rejected": -3.47672963142395,
+      "logps/chosen": -1.8643858432769775,
+      "logps/rejected": -2.0631701946258545,
+      "loss": 1.1155,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.8643858432769775,
+      "rewards/margins": 0.19878420233726501,
+      "rewards/rejected": -2.0631701946258545,
+      "step": 4590
+    },
+    {
+      "epoch": 0.7925568573397657,
+      "grad_norm": 13.017389106214816,
+      "learning_rate": 3.7781061858942206e-08,
+      "logits/chosen": -3.421992778778076,
+      "logits/rejected": -3.410595417022705,
+      "logps/chosen": -1.83063542842865,
+      "logps/rejected": -2.1120753288269043,
+      "loss": 1.0346,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.83063542842865,
+      "rewards/margins": 0.28143978118896484,
+      "rewards/rejected": -2.1120753288269043,
+      "step": 4600
+    },
+    {
+      "epoch": 0.7925568573397657,
+      "eval_logits/chosen": -3.5342118740081787,
+      "eval_logits/rejected": -3.530413866043091,
+      "eval_logps/chosen": -1.6779311895370483,
+      "eval_logps/rejected": -1.8418591022491455,
+      "eval_loss": 1.099021553993225,
+      "eval_rewards/accuracies": 0.6240706443786621,
+      "eval_rewards/chosen": -1.6779311895370483,
+      "eval_rewards/margins": 0.16392803192138672,
+      "eval_rewards/rejected": -1.8418591022491455,
+      "eval_runtime": 155.4355,
+      "eval_samples_per_second": 27.69,
+      "eval_steps_per_second": 3.461,
+      "step": 4600
+    },
+    {
+      "epoch": 0.7942798070296347,
+      "grad_norm": 12.71927555827521,
+      "learning_rate": 3.7716392328929864e-08,
+      "logits/chosen": -3.437431812286377,
+      "logits/rejected": -3.431269407272339,
+      "logps/chosen": -1.8536970615386963,
+      "logps/rejected": -2.0196056365966797,
+      "loss": 1.1294,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -1.8536970615386963,
+      "rewards/margins": 0.16590848565101624,
+      "rewards/rejected": -2.0196056365966797,
+      "step": 4610
+    },
+    {
+      "epoch": 0.7960027567195038,
+      "grad_norm": 10.596318540251149,
+      "learning_rate": 3.765160780360254e-08,
+      "logits/chosen": -3.438406467437744,
+      "logits/rejected": -3.427164077758789,
+      "logps/chosen": -1.7847334146499634,
+      "logps/rejected": -2.006922960281372,
+      "loss": 1.0793,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.7847334146499634,
+      "rewards/margins": 0.22218947112560272,
+      "rewards/rejected": -2.006922960281372,
+      "step": 4620
+    },
+    {
+      "epoch": 0.7977257064093728,
+      "grad_norm": 7.187401305727788,
+      "learning_rate": 3.7586708868811703e-08,
+      "logits/chosen": -3.477475643157959,
+      "logits/rejected": -3.459272861480713,
+      "logps/chosen": -1.7941644191741943,
+      "logps/rejected": -2.146414279937744,
+      "loss": 1.0189,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.7941644191741943,
+      "rewards/margins": 0.3522498607635498,
+      "rewards/rejected": -2.146414279937744,
+      "step": 4630
+    },
+    {
+      "epoch": 0.7994486560992419,
+      "grad_norm": 10.447932431725114,
+      "learning_rate": 3.7521696111443416e-08,
+      "logits/chosen": -3.4839344024658203,
+      "logits/rejected": -3.474947690963745,
+      "logps/chosen": -1.898906946182251,
+      "logps/rejected": -2.1014225482940674,
+      "loss": 1.0843,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.898906946182251,
+      "rewards/margins": 0.20251551270484924,
+      "rewards/rejected": -2.1014225482940674,
+      "step": 4640
+    },
+    {
+      "epoch": 0.801171605789111,
+      "grad_norm": 8.87060566985378,
+      "learning_rate": 3.7456570119413035e-08,
+      "logits/chosen": -3.4818172454833984,
+      "logits/rejected": -3.4648044109344482,
+      "logps/chosen": -1.772060751914978,
+      "logps/rejected": -2.0345969200134277,
+      "loss": 1.0551,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.772060751914978,
+      "rewards/margins": 0.2625361979007721,
+      "rewards/rejected": -2.0345969200134277,
+      "step": 4650
+    },
+    {
+      "epoch": 0.80289455547898,
+      "grad_norm": 10.706028250836447,
+      "learning_rate": 3.739133148165994e-08,
+      "logits/chosen": -3.478705883026123,
+      "logits/rejected": -3.4694931507110596,
+      "logps/chosen": -1.8079332113265991,
+      "logps/rejected": -2.016995668411255,
+      "loss": 1.0823,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.8079332113265991,
+      "rewards/margins": 0.20906230807304382,
+      "rewards/rejected": -2.016995668411255,
+      "step": 4660
+    },
+    {
+      "epoch": 0.8046175051688491,
+      "grad_norm": 8.374699377562777,
+      "learning_rate": 3.732598078814215e-08,
+      "logits/chosen": -3.469174861907959,
+      "logits/rejected": -3.455202579498291,
+      "logps/chosen": -1.7618134021759033,
+      "logps/rejected": -1.8775997161865234,
+      "loss": 1.15,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.7618134021759033,
+      "rewards/margins": 0.11578632891178131,
+      "rewards/rejected": -1.8775997161865234,
+      "step": 4670
+    },
+    {
+      "epoch": 0.8063404548587181,
+      "grad_norm": 10.610444130204232,
+      "learning_rate": 3.7260518629831006e-08,
+      "logits/chosen": -3.4527640342712402,
+      "logits/rejected": -3.4350101947784424,
+      "logps/chosen": -1.803868055343628,
+      "logps/rejected": -2.065181255340576,
+      "loss": 1.0486,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.803868055343628,
+      "rewards/margins": 0.2613135278224945,
+      "rewards/rejected": -2.065181255340576,
+      "step": 4680
+    },
+    {
+      "epoch": 0.8080634045485872,
+      "grad_norm": 12.549069643319683,
+      "learning_rate": 3.7194945598705865e-08,
+      "logits/chosen": -3.4877171516418457,
+      "logits/rejected": -3.476414442062378,
+      "logps/chosen": -1.887328863143921,
+      "logps/rejected": -2.1108367443084717,
+      "loss": 1.0996,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.887328863143921,
+      "rewards/margins": 0.22350779175758362,
+      "rewards/rejected": -2.1108367443084717,
+      "step": 4690
+    },
+    {
+      "epoch": 0.8097863542384562,
+      "grad_norm": 9.329068768182776,
+      "learning_rate": 3.712926228774868e-08,
+      "logits/chosen": -3.470322847366333,
+      "logits/rejected": -3.465893507003784,
+      "logps/chosen": -1.8403066396713257,
+      "logps/rejected": -2.100289821624756,
+      "loss": 1.0589,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.8403066396713257,
+      "rewards/margins": 0.25998321175575256,
+      "rewards/rejected": -2.100289821624756,
+      "step": 4700
+    },
+    {
+      "epoch": 0.8097863542384562,
+      "eval_logits/chosen": -3.521972894668579,
+      "eval_logits/rejected": -3.5181398391723633,
+      "eval_logps/chosen": -1.6891604661941528,
+      "eval_logps/rejected": -1.8544057607650757,
+      "eval_loss": 1.098482370376587,
+      "eval_rewards/accuracies": 0.6247676610946655,
+      "eval_rewards/chosen": -1.6891604661941528,
+      "eval_rewards/margins": 0.1652453988790512,
+      "eval_rewards/rejected": -1.8544057607650757,
+      "eval_runtime": 155.3476,
+      "eval_samples_per_second": 27.706,
+      "eval_steps_per_second": 3.463,
+      "step": 4700
+    },
+    {
+      "epoch": 0.8115093039283253,
+      "grad_norm": 11.489317381359594,
+      "learning_rate": 3.70634692909387e-08,
+      "logits/chosen": -3.504046678543091,
+      "logits/rejected": -3.481678009033203,
+      "logps/chosen": -1.8243032693862915,
+      "logps/rejected": -2.0477938652038574,
+      "loss": 1.0964,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.8243032693862915,
+      "rewards/margins": 0.22349052131175995,
+      "rewards/rejected": -2.0477938652038574,
+      "step": 4710
+    },
+    {
+      "epoch": 0.8132322536181944,
+      "grad_norm": 10.7801876808542,
+      "learning_rate": 3.699756720324706e-08,
+      "logits/chosen": -3.4160354137420654,
+      "logits/rejected": -3.398965358734131,
+      "logps/chosen": -1.7882091999053955,
+      "logps/rejected": -1.9930884838104248,
+      "loss": 1.1003,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.7882091999053955,
+      "rewards/margins": 0.20487961173057556,
+      "rewards/rejected": -1.9930884838104248,
+      "step": 4720
+    },
+    {
+      "epoch": 0.8149552033080634,
+      "grad_norm": 10.882406681123793,
+      "learning_rate": 3.693155662063141e-08,
+      "logits/chosen": -3.41571044921875,
+      "logits/rejected": -3.40269136428833,
+      "logps/chosen": -1.8120486736297607,
+      "logps/rejected": -2.0196385383605957,
+      "loss": 1.084,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.8120486736297607,
+      "rewards/margins": 0.2075900286436081,
+      "rewards/rejected": -2.0196385383605957,
+      "step": 4730
+    },
+    {
+      "epoch": 0.8166781529979324,
+      "grad_norm": 9.79004729998889,
+      "learning_rate": 3.686543814003053e-08,
+      "logits/chosen": -3.4647223949432373,
+      "logits/rejected": -3.4584319591522217,
+      "logps/chosen": -1.7621009349822998,
+      "logps/rejected": -1.972400426864624,
+      "loss": 1.0833,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.7621009349822998,
+      "rewards/margins": 0.21029968559741974,
+      "rewards/rejected": -1.972400426864624,
+      "step": 4740
+    },
+    {
+      "epoch": 0.8184011026878015,
+      "grad_norm": 10.514306365665844,
+      "learning_rate": 3.6799212359358935e-08,
+      "logits/chosen": -3.451145648956299,
+      "logits/rejected": -3.439783811569214,
+      "logps/chosen": -1.7921940088272095,
+      "logps/rejected": -1.972425103187561,
+      "loss": 1.0917,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.7921940088272095,
+      "rewards/margins": 0.18023106455802917,
+      "rewards/rejected": -1.972425103187561,
+      "step": 4750
+    },
+    {
+      "epoch": 0.8201240523776706,
+      "grad_norm": 12.628543169030669,
+      "learning_rate": 3.673287987750146e-08,
+      "logits/chosen": -3.4607250690460205,
+      "logits/rejected": -3.4474411010742188,
+      "logps/chosen": -1.8683372735977173,
+      "logps/rejected": -2.102221727371216,
+      "loss": 1.0841,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -1.8683372735977173,
+      "rewards/margins": 0.23388464748859406,
+      "rewards/rejected": -2.102221727371216,
+      "step": 4760
+    },
+    {
+      "epoch": 0.8218470020675396,
+      "grad_norm": 10.990294829033317,
+      "learning_rate": 3.6666441294307835e-08,
+      "logits/chosen": -3.4972541332244873,
+      "logits/rejected": -3.48504376411438,
+      "logps/chosen": -1.8638255596160889,
+      "logps/rejected": -1.9491316080093384,
+      "loss": 1.1565,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -1.8638255596160889,
+      "rewards/margins": 0.0853060632944107,
+      "rewards/rejected": -1.9491316080093384,
+      "step": 4770
+    },
+    {
+      "epoch": 0.8235699517574087,
+      "grad_norm": 9.89769963833377,
+      "learning_rate": 3.65998972105873e-08,
+      "logits/chosen": -3.4187679290771484,
+      "logits/rejected": -3.407043933868408,
+      "logps/chosen": -1.7698347568511963,
+      "logps/rejected": -2.0666236877441406,
+      "loss": 1.0373,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.7698347568511963,
+      "rewards/margins": 0.29678890109062195,
+      "rewards/rejected": -2.0666236877441406,
+      "step": 4780
+    },
+    {
+      "epoch": 0.8252929014472777,
+      "grad_norm": 10.147895946018714,
+      "learning_rate": 3.6533248228103114e-08,
+      "logits/chosen": -3.476608991622925,
+      "logits/rejected": -3.460615634918213,
+      "logps/chosen": -1.8804343938827515,
+      "logps/rejected": -2.0244762897491455,
+      "loss": 1.1228,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.8804343938827515,
+      "rewards/margins": 0.14404162764549255,
+      "rewards/rejected": -2.0244762897491455,
+      "step": 4790
+    },
+    {
+      "epoch": 0.8270158511371468,
+      "grad_norm": 11.587589154259321,
+      "learning_rate": 3.6466494949567175e-08,
+      "logits/chosen": -3.4297099113464355,
+      "logits/rejected": -3.4212441444396973,
+      "logps/chosen": -1.7943837642669678,
+      "logps/rejected": -1.9477494955062866,
+      "loss": 1.1169,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.7943837642669678,
+      "rewards/margins": 0.15336564183235168,
+      "rewards/rejected": -1.9477494955062866,
+      "step": 4800
+    },
+    {
+      "epoch": 0.8270158511371468,
+      "eval_logits/chosen": -3.523972749710083,
+      "eval_logits/rejected": -3.520176410675049,
+      "eval_logps/chosen": -1.704317569732666,
+      "eval_logps/rejected": -1.870858073234558,
+      "eval_loss": 1.0978317260742188,
+      "eval_rewards/accuracies": 0.625,
+      "eval_rewards/chosen": -1.704317569732666,
+      "eval_rewards/margins": 0.16654060781002045,
+      "eval_rewards/rejected": -1.870858073234558,
+      "eval_runtime": 155.6855,
+      "eval_samples_per_second": 27.645,
+      "eval_steps_per_second": 3.456,
+      "step": 4800
+    },
+    {
+      "epoch": 0.8287388008270159,
+      "grad_norm": 9.975353693480042,
+      "learning_rate": 3.639963797863449e-08,
+      "logits/chosen": -3.427691698074341,
+      "logits/rejected": -3.4088191986083984,
+      "logps/chosen": -1.8019860982894897,
+      "logps/rejected": -2.050698757171631,
+      "loss": 1.0644,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.8019860982894897,
+      "rewards/margins": 0.2487124651670456,
+      "rewards/rejected": -2.050698757171631,
+      "step": 4810
+    },
+    {
+      "epoch": 0.8304617505168849,
+      "grad_norm": 9.41958694163124,
+      "learning_rate": 3.633267791989782e-08,
+      "logits/chosen": -3.4526565074920654,
+      "logits/rejected": -3.4463284015655518,
+      "logps/chosen": -1.8088480234146118,
+      "logps/rejected": -2.0138120651245117,
+      "loss": 1.0893,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.8088480234146118,
+      "rewards/margins": 0.20496425032615662,
+      "rewards/rejected": -2.0138120651245117,
+      "step": 4820
+    },
+    {
+      "epoch": 0.832184700206754,
+      "grad_norm": 10.95353644889666,
+      "learning_rate": 3.626561537888214e-08,
+      "logits/chosen": -3.462636947631836,
+      "logits/rejected": -3.4480385780334473,
+      "logps/chosen": -1.8283278942108154,
+      "logps/rejected": -2.045539379119873,
+      "loss": 1.0852,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.8283278942108154,
+      "rewards/margins": 0.2172117680311203,
+      "rewards/rejected": -2.045539379119873,
+      "step": 4830
+    },
+    {
+      "epoch": 0.833907649896623,
+      "grad_norm": 11.550857386595428,
+      "learning_rate": 3.6198450962039146e-08,
+      "logits/chosen": -3.440760374069214,
+      "logits/rejected": -3.4235153198242188,
+      "logps/chosen": -1.8547710180282593,
+      "logps/rejected": -2.088658571243286,
+      "loss": 1.0759,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.8547710180282593,
+      "rewards/margins": 0.23388762772083282,
+      "rewards/rejected": -2.088658571243286,
+      "step": 4840
+    },
+    {
+      "epoch": 0.8356305995864921,
+      "grad_norm": 9.51896653396322,
+      "learning_rate": 3.613118527674184e-08,
+      "logits/chosen": -3.4820303916931152,
+      "logits/rejected": -3.4730496406555176,
+      "logps/chosen": -1.8217700719833374,
+      "logps/rejected": -2.0458619594573975,
+      "loss": 1.0777,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.8217700719833374,
+      "rewards/margins": 0.22409196197986603,
+      "rewards/rejected": -2.0458619594573975,
+      "step": 4850
+    },
+    {
+      "epoch": 0.8373535492763611,
+      "grad_norm": 10.780666946145095,
+      "learning_rate": 3.6063818931279e-08,
+      "logits/chosen": -3.4889492988586426,
+      "logits/rejected": -3.471437931060791,
+      "logps/chosen": -1.919257402420044,
+      "logps/rejected": -2.04201602935791,
+      "loss": 1.1427,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -1.919257402420044,
+      "rewards/margins": 0.12275861203670502,
+      "rewards/rejected": -2.04201602935791,
+      "step": 4860
+    },
+    {
+      "epoch": 0.8390764989662302,
+      "grad_norm": 11.618466747389865,
+      "learning_rate": 3.599635253484967e-08,
+      "logits/chosen": -3.495753526687622,
+      "logits/rejected": -3.481552839279175,
+      "logps/chosen": -1.857842206954956,
+      "logps/rejected": -2.000894546508789,
+      "loss": 1.1206,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.857842206954956,
+      "rewards/margins": 0.1430523693561554,
+      "rewards/rejected": -2.000894546508789,
+      "step": 4870
+    },
+    {
+      "epoch": 0.8407994486560992,
+      "grad_norm": 9.049310243913338,
+      "learning_rate": 3.5928786697557667e-08,
+      "logits/chosen": -3.4209632873535156,
+      "logits/rejected": -3.401214599609375,
+      "logps/chosen": -1.8374961614608765,
+      "logps/rejected": -2.1090145111083984,
+      "loss": 1.0669,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.8374961614608765,
+      "rewards/margins": 0.2715182304382324,
+      "rewards/rejected": -2.1090145111083984,
+      "step": 4880
+    },
+    {
+      "epoch": 0.8425223983459683,
+      "grad_norm": 10.069449838927268,
+      "learning_rate": 3.586112203040607e-08,
+      "logits/chosen": -3.457656145095825,
+      "logits/rejected": -3.446042537689209,
+      "logps/chosen": -1.8194310665130615,
+      "logps/rejected": -2.0697271823883057,
+      "loss": 1.0736,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.8194310665130615,
+      "rewards/margins": 0.250296026468277,
+      "rewards/rejected": -2.0697271823883057,
+      "step": 4890
+    },
+    {
+      "epoch": 0.8442453480358374,
+      "grad_norm": 11.100246001054074,
+      "learning_rate": 3.579335914529166e-08,
+      "logits/chosen": -3.444554090499878,
+      "logits/rejected": -3.4266979694366455,
+      "logps/chosen": -1.8450673818588257,
+      "logps/rejected": -2.1378536224365234,
+      "loss": 1.0477,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.8450673818588257,
+      "rewards/margins": 0.2927860617637634,
+      "rewards/rejected": -2.1378536224365234,
+      "step": 4900
+    },
+    {
+      "epoch": 0.8442453480358374,
+      "eval_logits/chosen": -3.52335262298584,
+      "eval_logits/rejected": -3.5195674896240234,
+      "eval_logps/chosen": -1.7174824476242065,
+      "eval_logps/rejected": -1.885353922843933,
+      "eval_loss": 1.0971500873565674,
+      "eval_rewards/accuracies": 0.6259293556213379,
+      "eval_rewards/chosen": -1.7174824476242065,
+      "eval_rewards/margins": 0.1678716242313385,
+      "eval_rewards/rejected": -1.885353922843933,
+      "eval_runtime": 155.5147,
+      "eval_samples_per_second": 27.676,
+      "eval_steps_per_second": 3.459,
+      "step": 4900
+    },
+    {
+      "epoch": 0.8459682977257064,
+      "grad_norm": 10.088161042471414,
+      "learning_rate": 3.572549865499944e-08,
+      "logits/chosen": -3.527315855026245,
+      "logits/rejected": -3.5107274055480957,
+      "logps/chosen": -1.7859948873519897,
+      "logps/rejected": -2.0366015434265137,
+      "loss": 1.0633,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.7859948873519897,
+      "rewards/margins": 0.2506069540977478,
+      "rewards/rejected": -2.0366015434265137,
+      "step": 4910
+    },
+    {
+      "epoch": 0.8476912474155754,
+      "grad_norm": 10.236821116057465,
+      "learning_rate": 3.5657541173197025e-08,
+      "logits/chosen": -3.4185824394226074,
+      "logits/rejected": -3.4089291095733643,
+      "logps/chosen": -1.9017324447631836,
+      "logps/rejected": -2.14628267288208,
+      "loss": 1.0766,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.9017324447631836,
+      "rewards/margins": 0.24455030262470245,
+      "rewards/rejected": -2.14628267288208,
+      "step": 4920
+    },
+    {
+      "epoch": 0.8494141971054445,
+      "grad_norm": 10.02948244979294,
+      "learning_rate": 3.558948731442918e-08,
+      "logits/chosen": -3.5285065174102783,
+      "logits/rejected": -3.5236244201660156,
+      "logps/chosen": -1.9051845073699951,
+      "logps/rejected": -2.1489899158477783,
+      "loss": 1.0742,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.9051845073699951,
+      "rewards/margins": 0.24380531907081604,
+      "rewards/rejected": -2.1489899158477783,
+      "step": 4930
+    },
+    {
+      "epoch": 0.8511371467953136,
+      "grad_norm": 12.732879017118586,
+      "learning_rate": 3.5521337694112176e-08,
+      "logits/chosen": -3.4858059883117676,
+      "logits/rejected": -3.4685959815979004,
+      "logps/chosen": -1.9179147481918335,
+      "logps/rejected": -2.199037551879883,
+      "loss": 1.0501,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.9179147481918335,
+      "rewards/margins": 0.28112298250198364,
+      "rewards/rejected": -2.199037551879883,
+      "step": 4940
+    },
+    {
+      "epoch": 0.8528600964851827,
+      "grad_norm": 10.202735014407434,
+      "learning_rate": 3.5453092928528286e-08,
+      "logits/chosen": -3.3665173053741455,
+      "logits/rejected": -3.3575751781463623,
+      "logps/chosen": -1.8939135074615479,
+      "logps/rejected": -2.0629172325134277,
+      "loss": 1.1326,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.8939135074615479,
+      "rewards/margins": 0.16900362074375153,
+      "rewards/rejected": -2.0629172325134277,
+      "step": 4950
+    },
+    {
+      "epoch": 0.8545830461750517,
+      "grad_norm": 10.770528768433191,
+      "learning_rate": 3.538475363482017e-08,
+      "logits/chosen": -3.4357192516326904,
+      "logits/rejected": -3.4347729682922363,
+      "logps/chosen": -1.9288291931152344,
+      "logps/rejected": -2.1294703483581543,
+      "loss": 1.0978,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.9288291931152344,
+      "rewards/margins": 0.2006412297487259,
+      "rewards/rejected": -2.1294703483581543,
+      "step": 4960
+    },
+    {
+      "epoch": 0.8563059958649207,
+      "grad_norm": 12.183799181586775,
+      "learning_rate": 3.531632043098533e-08,
+      "logits/chosen": -3.434633731842041,
+      "logits/rejected": -3.429356813430786,
+      "logps/chosen": -1.9184478521347046,
+      "logps/rejected": -2.1462960243225098,
+      "loss": 1.0869,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.9184478521347046,
+      "rewards/margins": 0.2278481423854828,
+      "rewards/rejected": -2.1462960243225098,
+      "step": 4970
+    },
+    {
+      "epoch": 0.8580289455547898,
+      "grad_norm": 9.300657245099485,
+      "learning_rate": 3.524779393587049e-08,
+      "logits/chosen": -3.476921796798706,
+      "logits/rejected": -3.4737744331359863,
+      "logps/chosen": -1.8450511693954468,
+      "logps/rejected": -2.005983352661133,
+      "loss": 1.1165,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.8450511693954468,
+      "rewards/margins": 0.16093197464942932,
+      "rewards/rejected": -2.005983352661133,
+      "step": 4980
+    },
+    {
+      "epoch": 0.8597518952446589,
+      "grad_norm": 9.393837366728443,
+      "learning_rate": 3.517917476916604e-08,
+      "logits/chosen": -3.412691593170166,
+      "logits/rejected": -3.407773971557617,
+      "logps/chosen": -1.8522074222564697,
+      "logps/rejected": -2.064918279647827,
+      "loss": 1.1002,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.8522074222564697,
+      "rewards/margins": 0.21271082758903503,
+      "rewards/rejected": -2.064918279647827,
+      "step": 4990
+    },
+    {
+      "epoch": 0.8614748449345279,
+      "grad_norm": 9.767606572010324,
+      "learning_rate": 3.511046355140036e-08,
+      "logits/chosen": -3.4416301250457764,
+      "logits/rejected": -3.4308021068573,
+      "logps/chosen": -1.8950364589691162,
+      "logps/rejected": -2.074068307876587,
+      "loss": 1.1388,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.8950364589691162,
+      "rewards/margins": 0.17903196811676025,
+      "rewards/rejected": -2.074068307876587,
+      "step": 5000
+    },
+    {
+      "epoch": 0.8614748449345279,
+      "eval_logits/chosen": -3.516158103942871,
+      "eval_logits/rejected": -3.5123584270477295,
+      "eval_logps/chosen": -1.7191039323806763,
+      "eval_logps/rejected": -1.8874883651733398,
+      "eval_loss": 1.0969411134719849,
+      "eval_rewards/accuracies": 0.6240706443786621,
+      "eval_rewards/chosen": -1.7191039323806763,
+      "eval_rewards/margins": 0.16838450729846954,
+      "eval_rewards/rejected": -1.8874883651733398,
+      "eval_runtime": 155.5389,
+      "eval_samples_per_second": 27.672,
+      "eval_steps_per_second": 3.459,
+      "step": 5000
+    },
+    {
+      "epoch": 0.8631977946243969,
+      "grad_norm": 10.784548487892827,
+      "learning_rate": 3.5041660903934306e-08,
+      "logits/chosen": -3.440232753753662,
+      "logits/rejected": -3.4253745079040527,
+      "logps/chosen": -1.8520838022232056,
+      "logps/rejected": -2.0750606060028076,
+      "loss": 1.0722,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.8520838022232056,
+      "rewards/margins": 0.22297687828540802,
+      "rewards/rejected": -2.0750606060028076,
+      "step": 5010
+    },
+    {
+      "epoch": 0.864920744314266,
+      "grad_norm": 10.864566849972507,
+      "learning_rate": 3.4972767448955513e-08,
+      "logits/chosen": -3.4276816844940186,
+      "logits/rejected": -3.413456678390503,
+      "logps/chosen": -1.8161275386810303,
+      "logps/rejected": -2.0597078800201416,
+      "loss": 1.0802,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.8161275386810303,
+      "rewards/margins": 0.24358026683330536,
+      "rewards/rejected": -2.0597078800201416,
+      "step": 5020
+    },
+    {
+      "epoch": 0.8666436940041351,
+      "grad_norm": 10.310077029578373,
+      "learning_rate": 3.490378380947279e-08,
+      "logits/chosen": -3.39629864692688,
+      "logits/rejected": -3.3909077644348145,
+      "logps/chosen": -1.9290459156036377,
+      "logps/rejected": -2.08693265914917,
+      "loss": 1.1181,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.9290459156036377,
+      "rewards/margins": 0.1578865945339203,
+      "rewards/rejected": -2.08693265914917,
+      "step": 5030
+    },
+    {
+      "epoch": 0.8683666436940042,
+      "grad_norm": 11.70436596432896,
+      "learning_rate": 3.483471060931051e-08,
+      "logits/chosen": -3.5457847118377686,
+      "logits/rejected": -3.522904634475708,
+      "logps/chosen": -1.8110673427581787,
+      "logps/rejected": -2.006075620651245,
+      "loss": 1.0904,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.8110673427581787,
+      "rewards/margins": 0.19500815868377686,
+      "rewards/rejected": -2.006075620651245,
+      "step": 5040
+    },
+    {
+      "epoch": 0.8700895933838731,
+      "grad_norm": 10.799957850009978,
+      "learning_rate": 3.476554847310294e-08,
+      "logits/chosen": -3.459956645965576,
+      "logits/rejected": -3.4460136890411377,
+      "logps/chosen": -1.8513752222061157,
+      "logps/rejected": -2.1486268043518066,
+      "loss": 1.042,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.8513752222061157,
+      "rewards/margins": 0.2972516715526581,
+      "rewards/rejected": -2.1486268043518066,
+      "step": 5050
+    },
+    {
+      "epoch": 0.8718125430737422,
+      "grad_norm": 11.366286937941489,
+      "learning_rate": 3.4696298026288585e-08,
+      "logits/chosen": -3.371398448944092,
+      "logits/rejected": -3.366403102874756,
+      "logps/chosen": -1.9085171222686768,
+      "logps/rejected": -2.1194987297058105,
+      "loss": 1.0925,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.9085171222686768,
+      "rewards/margins": 0.21098187565803528,
+      "rewards/rejected": -2.1194987297058105,
+      "step": 5060
+    },
+    {
+      "epoch": 0.8735354927636113,
+      "grad_norm": 11.0935151448377,
+      "learning_rate": 3.462695989510459e-08,
+      "logits/chosen": -3.4451606273651123,
+      "logits/rejected": -3.4302074909210205,
+      "logps/chosen": -1.8246638774871826,
+      "logps/rejected": -2.008059024810791,
+      "loss": 1.1037,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.8246638774871826,
+      "rewards/margins": 0.1833951771259308,
+      "rewards/rejected": -2.008059024810791,
+      "step": 5070
+    },
+    {
+      "epoch": 0.8752584424534804,
+      "grad_norm": 10.766024057853658,
+      "learning_rate": 3.4557534706580996e-08,
+      "logits/chosen": -3.555269718170166,
+      "logits/rejected": -3.5363457202911377,
+      "logps/chosen": -1.846384048461914,
+      "logps/rejected": -2.037414073944092,
+      "loss": 1.1011,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.846384048461914,
+      "rewards/margins": 0.1910301148891449,
+      "rewards/rejected": -2.037414073944092,
+      "step": 5080
+    },
+    {
+      "epoch": 0.8769813921433495,
+      "grad_norm": 10.63080313475581,
+      "learning_rate": 3.448802308853515e-08,
+      "logits/chosen": -3.4811336994171143,
+      "logits/rejected": -3.463364839553833,
+      "logps/chosen": -1.8247401714324951,
+      "logps/rejected": -2.124480724334717,
+      "loss": 1.0351,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.8247401714324951,
+      "rewards/margins": 0.29974043369293213,
+      "rewards/rejected": -2.124480724334717,
+      "step": 5090
+    },
+    {
+      "epoch": 0.8787043418332184,
+      "grad_norm": 11.386407316648656,
+      "learning_rate": 3.441842566956595e-08,
+      "logits/chosen": -3.3809986114501953,
+      "logits/rejected": -3.3611502647399902,
+      "logps/chosen": -1.840951681137085,
+      "logps/rejected": -2.114793300628662,
+      "loss": 1.0556,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.840951681137085,
+      "rewards/margins": 0.2738417387008667,
+      "rewards/rejected": -2.114793300628662,
+      "step": 5100
+    },
+    {
+      "epoch": 0.8787043418332184,
+      "eval_logits/chosen": -3.509974718093872,
+      "eval_logits/rejected": -3.5061683654785156,
+      "eval_logps/chosen": -1.7340697050094604,
+      "eval_logps/rejected": -1.9039703607559204,
+      "eval_loss": 1.0962203741073608,
+      "eval_rewards/accuracies": 0.6236059665679932,
+      "eval_rewards/chosen": -1.7340697050094604,
+      "eval_rewards/margins": 0.16990065574645996,
+      "eval_rewards/rejected": -1.9039703607559204,
+      "eval_runtime": 155.7084,
+      "eval_samples_per_second": 27.641,
+      "eval_steps_per_second": 3.455,
+      "step": 5100
+    },
+    {
+      "epoch": 0.8804272915230875,
+      "grad_norm": 9.195198225517151,
+      "learning_rate": 3.434874307904822e-08,
+      "logits/chosen": -3.4599297046661377,
+      "logits/rejected": -3.4392902851104736,
+      "logps/chosen": -1.8822624683380127,
+      "logps/rejected": -2.1494736671447754,
+      "loss": 1.0759,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.8822624683380127,
+      "rewards/margins": 0.267211377620697,
+      "rewards/rejected": -2.1494736671447754,
+      "step": 5110
+    },
+    {
+      "epoch": 0.8821502412129566,
+      "grad_norm": 11.46622853543848,
+      "learning_rate": 3.427897594712699e-08,
+      "logits/chosen": -3.5026047229766846,
+      "logits/rejected": -3.486032009124756,
+      "logps/chosen": -1.8500789403915405,
+      "logps/rejected": -2.046753168106079,
+      "loss": 1.0996,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.8500789403915405,
+      "rewards/margins": 0.1966741532087326,
+      "rewards/rejected": -2.046753168106079,
+      "step": 5120
+    },
+    {
+      "epoch": 0.8838731909028257,
+      "grad_norm": 11.043723928967824,
+      "learning_rate": 3.4209124904711807e-08,
+      "logits/chosen": -3.507479429244995,
+      "logits/rejected": -3.49006724357605,
+      "logps/chosen": -1.848258376121521,
+      "logps/rejected": -2.0551950931549072,
+      "loss": 1.0905,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.848258376121521,
+      "rewards/margins": 0.20693644881248474,
+      "rewards/rejected": -2.0551950931549072,
+      "step": 5130
+    },
+    {
+      "epoch": 0.8855961405926946,
+      "grad_norm": 8.683839558551309,
+      "learning_rate": 3.413919058347102e-08,
+      "logits/chosen": -3.44372820854187,
+      "logits/rejected": -3.4244582653045654,
+      "logps/chosen": -1.8640495538711548,
+      "logps/rejected": -2.159275531768799,
+      "loss": 1.0537,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.8640495538711548,
+      "rewards/margins": 0.29522597789764404,
+      "rewards/rejected": -2.159275531768799,
+      "step": 5140
+    },
+    {
+      "epoch": 0.8873190902825637,
+      "grad_norm": 10.907062356148389,
+      "learning_rate": 3.40691736158261e-08,
+      "logits/chosen": -3.489828586578369,
+      "logits/rejected": -3.4860777854919434,
+      "logps/chosen": -1.889171838760376,
+      "logps/rejected": -2.0647807121276855,
+      "loss": 1.1002,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.889171838760376,
+      "rewards/margins": 0.17560862004756927,
+      "rewards/rejected": -2.0647807121276855,
+      "step": 5150
+    },
+    {
+      "epoch": 0.8890420399724328,
+      "grad_norm": 11.49873107639848,
+      "learning_rate": 3.399907463494585e-08,
+      "logits/chosen": -3.4602599143981934,
+      "logits/rejected": -3.4421768188476562,
+      "logps/chosen": -1.9217841625213623,
+      "logps/rejected": -2.18563175201416,
+      "loss": 1.062,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.9217841625213623,
+      "rewards/margins": 0.26384735107421875,
+      "rewards/rejected": -2.18563175201416,
+      "step": 5160
+    },
+    {
+      "epoch": 0.8907649896623019,
+      "grad_norm": 12.965075632481144,
+      "learning_rate": 3.392889427474077e-08,
+      "logits/chosen": -3.4490599632263184,
+      "logits/rejected": -3.436666488647461,
+      "logps/chosen": -1.8231680393218994,
+      "logps/rejected": -2.038656711578369,
+      "loss": 1.0827,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.8231680393218994,
+      "rewards/margins": 0.21548855304718018,
+      "rewards/rejected": -2.038656711578369,
+      "step": 5170
+    },
+    {
+      "epoch": 0.892487939352171,
+      "grad_norm": 15.336728803442595,
+      "learning_rate": 3.385863316985726e-08,
+      "logits/chosen": -3.50866436958313,
+      "logits/rejected": -3.5045535564422607,
+      "logps/chosen": -1.9037015438079834,
+      "logps/rejected": -2.0562126636505127,
+      "loss": 1.1162,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.9037015438079834,
+      "rewards/margins": 0.1525108814239502,
+      "rewards/rejected": -2.0562126636505127,
+      "step": 5180
+    },
+    {
+      "epoch": 0.8942108890420399,
+      "grad_norm": 9.596640081973934,
+      "learning_rate": 3.3788291955671886e-08,
+      "logits/chosen": -3.4354805946350098,
+      "logits/rejected": -3.4284794330596924,
+      "logps/chosen": -1.8797311782836914,
+      "logps/rejected": -2.136137008666992,
+      "loss": 1.0772,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.8797311782836914,
+      "rewards/margins": 0.2564057409763336,
+      "rewards/rejected": -2.136137008666992,
+      "step": 5190
+    },
+    {
+      "epoch": 0.895933838731909,
+      "grad_norm": 10.751372143919253,
+      "learning_rate": 3.371787126828568e-08,
+      "logits/chosen": -3.463665008544922,
+      "logits/rejected": -3.452936887741089,
+      "logps/chosen": -1.9167280197143555,
+      "logps/rejected": -2.225165367126465,
+      "loss": 1.0387,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.9167280197143555,
+      "rewards/margins": 0.3084375560283661,
+      "rewards/rejected": -2.225165367126465,
+      "step": 5200
+    },
+    {
+      "epoch": 0.895933838731909,
+      "eval_logits/chosen": -3.510218858718872,
+      "eval_logits/rejected": -3.5064303874969482,
+      "eval_logps/chosen": -1.748279094696045,
+      "eval_logps/rejected": -1.9201085567474365,
+      "eval_loss": 1.0952982902526855,
+      "eval_rewards/accuracies": 0.6240706443786621,
+      "eval_rewards/chosen": -1.748279094696045,
+      "eval_rewards/margins": 0.17182937264442444,
+      "eval_rewards/rejected": -1.9201085567474365,
+      "eval_runtime": 155.5524,
+      "eval_samples_per_second": 27.669,
+      "eval_steps_per_second": 3.459,
+      "step": 5200
+    },
+    {
+      "epoch": 0.8976567884217781,
+      "grad_norm": 12.602269797690315,
+      "learning_rate": 3.3647371744518336e-08,
+      "logits/chosen": -3.4514846801757812,
+      "logits/rejected": -3.446376085281372,
+      "logps/chosen": -1.9301488399505615,
+      "logps/rejected": -2.0280745029449463,
+      "loss": 1.157,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -1.9301488399505615,
+      "rewards/margins": 0.09792548418045044,
+      "rewards/rejected": -2.0280745029449463,
+      "step": 5210
+    },
+    {
+      "epoch": 0.8993797381116472,
+      "grad_norm": 10.592019994050698,
+      "learning_rate": 3.3576794021902476e-08,
+      "logits/chosen": -3.4708290100097656,
+      "logits/rejected": -3.4650192260742188,
+      "logps/chosen": -1.9144752025604248,
+      "logps/rejected": -2.086632251739502,
+      "loss": 1.122,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.9144752025604248,
+      "rewards/margins": 0.1721574366092682,
+      "rewards/rejected": -2.086632251739502,
+      "step": 5220
+    },
+    {
+      "epoch": 0.9011026878015161,
+      "grad_norm": 9.523003350183393,
+      "learning_rate": 3.350613873867788e-08,
+      "logits/chosen": -3.439107894897461,
+      "logits/rejected": -3.433020830154419,
+      "logps/chosen": -1.8364006280899048,
+      "logps/rejected": -2.1201889514923096,
+      "loss": 1.0396,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.8364006280899048,
+      "rewards/margins": 0.28378820419311523,
+      "rewards/rejected": -2.1201889514923096,
+      "step": 5230
+    },
+    {
+      "epoch": 0.9028256374913852,
+      "grad_norm": 11.27601720434268,
+      "learning_rate": 3.343540653378571e-08,
+      "logits/chosen": -3.4345855712890625,
+      "logits/rejected": -3.4218361377716064,
+      "logps/chosen": -1.84024977684021,
+      "logps/rejected": -2.1707839965820312,
+      "loss": 1.0083,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.84024977684021,
+      "rewards/margins": 0.3305343687534332,
+      "rewards/rejected": -2.1707839965820312,
+      "step": 5240
+    },
+    {
+      "epoch": 0.9045485871812543,
+      "grad_norm": 8.499277770921038,
+      "learning_rate": 3.336459804686275e-08,
+      "logits/chosen": -3.4054856300354004,
+      "logits/rejected": -3.3983733654022217,
+      "logps/chosen": -1.8640401363372803,
+      "logps/rejected": -2.0644259452819824,
+      "loss": 1.0912,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.8640401363372803,
+      "rewards/margins": 0.20038612186908722,
+      "rewards/rejected": -2.0644259452819824,
+      "step": 5250
+    },
+    {
+      "epoch": 0.9062715368711234,
+      "grad_norm": 11.648828391282816,
+      "learning_rate": 3.3293713918235594e-08,
+      "logits/chosen": -3.4625744819641113,
+      "logits/rejected": -3.4403750896453857,
+      "logps/chosen": -1.8789939880371094,
+      "logps/rejected": -2.158292531967163,
+      "loss": 1.0396,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.8789939880371094,
+      "rewards/margins": 0.2792988121509552,
+      "rewards/rejected": -2.158292531967163,
+      "step": 5260
+    },
+    {
+      "epoch": 0.9079944865609925,
+      "grad_norm": 10.06693651421846,
+      "learning_rate": 3.3222754788914874e-08,
+      "logits/chosen": -3.5154833793640137,
+      "logits/rejected": -3.508295774459839,
+      "logps/chosen": -1.8637443780899048,
+      "logps/rejected": -2.093592405319214,
+      "loss": 1.0777,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.8637443780899048,
+      "rewards/margins": 0.22984810173511505,
+      "rewards/rejected": -2.093592405319214,
+      "step": 5270
+    },
+    {
+      "epoch": 0.9097174362508614,
+      "grad_norm": 12.840617621741774,
+      "learning_rate": 3.315172130058946e-08,
+      "logits/chosen": -3.455533981323242,
+      "logits/rejected": -3.4398772716522217,
+      "logps/chosen": -1.9827829599380493,
+      "logps/rejected": -2.159951686859131,
+      "loss": 1.1215,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.9827829599380493,
+      "rewards/margins": 0.1771688014268875,
+      "rewards/rejected": -2.159951686859131,
+      "step": 5280
+    },
+    {
+      "epoch": 0.9114403859407305,
+      "grad_norm": 11.221059716648375,
+      "learning_rate": 3.308061409562065e-08,
+      "logits/chosen": -3.4159839153289795,
+      "logits/rejected": -3.3912606239318848,
+      "logps/chosen": -1.8212134838104248,
+      "logps/rejected": -2.1456894874572754,
+      "loss": 1.0176,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.8212134838104248,
+      "rewards/margins": 0.3244761824607849,
+      "rewards/rejected": -2.1456894874572754,
+      "step": 5290
+    },
+    {
+      "epoch": 0.9131633356305996,
+      "grad_norm": 11.879851828822925,
+      "learning_rate": 3.300943381703639e-08,
+      "logits/chosen": -3.4281439781188965,
+      "logits/rejected": -3.4192709922790527,
+      "logps/chosen": -1.8593593835830688,
+      "logps/rejected": -2.1571288108825684,
+      "loss": 1.066,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.8593593835830688,
+      "rewards/margins": 0.29776954650878906,
+      "rewards/rejected": -2.1571288108825684,
+      "step": 5300
+    },
+    {
+      "epoch": 0.9131633356305996,
+      "eval_logits/chosen": -3.5094215869903564,
+      "eval_logits/rejected": -3.5056581497192383,
+      "eval_logps/chosen": -1.7532655000686646,
+      "eval_logps/rejected": -1.9255872964859009,
+      "eval_loss": 1.0951677560806274,
+      "eval_rewards/accuracies": 0.6240706443786621,
+      "eval_rewards/chosen": -1.7532655000686646,
+      "eval_rewards/margins": 0.17232167720794678,
+      "eval_rewards/rejected": -1.9255872964859009,
+      "eval_runtime": 155.5043,
+      "eval_samples_per_second": 27.678,
+      "eval_steps_per_second": 3.46,
+      "step": 5300
+    },
+    {
+      "epoch": 0.9148862853204687,
+      "grad_norm": 10.15932150532339,
+      "learning_rate": 3.293818110852541e-08,
+      "logits/chosen": -3.506436586380005,
+      "logits/rejected": -3.5008697509765625,
+      "logps/chosen": -1.9167277812957764,
+      "logps/rejected": -2.0137596130371094,
+      "loss": 1.1557,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.9167277812957764,
+      "rewards/margins": 0.09703131020069122,
+      "rewards/rejected": -2.0137596130371094,
+      "step": 5310
+    },
+    {
+      "epoch": 0.9166092350103378,
+      "grad_norm": 11.121729351319173,
+      "learning_rate": 3.286685661443144e-08,
+      "logits/chosen": -3.4443728923797607,
+      "logits/rejected": -3.417250156402588,
+      "logps/chosen": -1.8658548593521118,
+      "logps/rejected": -2.160815715789795,
+      "loss": 1.0377,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.8658548593521118,
+      "rewards/margins": 0.29496094584465027,
+      "rewards/rejected": -2.160815715789795,
+      "step": 5320
+    },
+    {
+      "epoch": 0.9183321847002067,
+      "grad_norm": 10.436071331396837,
+      "learning_rate": 3.279546097974738e-08,
+      "logits/chosen": -3.4100749492645264,
+      "logits/rejected": -3.412144184112549,
+      "logps/chosen": -1.8982923030853271,
+      "logps/rejected": -2.154775857925415,
+      "loss": 1.097,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.8982923030853271,
+      "rewards/margins": 0.25648361444473267,
+      "rewards/rejected": -2.154775857925415,
+      "step": 5330
+    },
+    {
+      "epoch": 0.9200551343900758,
+      "grad_norm": 8.521022590555152,
+      "learning_rate": 3.272399485010943e-08,
+      "logits/chosen": -3.4371304512023926,
+      "logits/rejected": -3.4096484184265137,
+      "logps/chosen": -1.8424017429351807,
+      "logps/rejected": -2.2565503120422363,
+      "loss": 0.9728,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -1.8424017429351807,
+      "rewards/margins": 0.41414886713027954,
+      "rewards/rejected": -2.2565503120422363,
+      "step": 5340
+    },
+    {
+      "epoch": 0.9217780840799449,
+      "grad_norm": 11.841750191462822,
+      "learning_rate": 3.265245887179133e-08,
+      "logits/chosen": -3.3989288806915283,
+      "logits/rejected": -3.378321886062622,
+      "logps/chosen": -1.9137271642684937,
+      "logps/rejected": -2.2054860591888428,
+      "loss": 1.0484,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.9137271642684937,
+      "rewards/margins": 0.29175904393196106,
+      "rewards/rejected": -2.2054860591888428,
+      "step": 5350
+    },
+    {
+      "epoch": 0.923501033769814,
+      "grad_norm": 10.631639009279985,
+      "learning_rate": 3.2580853691698416e-08,
+      "logits/chosen": -3.4863650798797607,
+      "logits/rejected": -3.480780839920044,
+      "logps/chosen": -1.8278077840805054,
+      "logps/rejected": -2.1186389923095703,
+      "loss": 1.041,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.8278077840805054,
+      "rewards/margins": 0.2908310294151306,
+      "rewards/rejected": -2.1186389923095703,
+      "step": 5360
+    },
+    {
+      "epoch": 0.9252239834596829,
+      "grad_norm": 12.15756839149027,
+      "learning_rate": 3.2509179957361865e-08,
+      "logits/chosen": -3.424440383911133,
+      "logits/rejected": -3.416226863861084,
+      "logps/chosen": -1.8795299530029297,
+      "logps/rejected": -2.1417577266693115,
+      "loss": 1.0606,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.8795299530029297,
+      "rewards/margins": 0.26222795248031616,
+      "rewards/rejected": -2.1417577266693115,
+      "step": 5370
+    },
+    {
+      "epoch": 0.926946933149552,
+      "grad_norm": 11.203327352968003,
+      "learning_rate": 3.2437438316932765e-08,
+      "logits/chosen": -3.48832631111145,
+      "logits/rejected": -3.466151475906372,
+      "logps/chosen": -1.8584846258163452,
+      "logps/rejected": -2.208665370941162,
+      "loss": 1.0174,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.8584846258163452,
+      "rewards/margins": 0.3501807153224945,
+      "rewards/rejected": -2.208665370941162,
+      "step": 5380
+    },
+    {
+      "epoch": 0.9286698828394211,
+      "grad_norm": 11.308508126228416,
+      "learning_rate": 3.2365629419176294e-08,
+      "logits/chosen": -3.43654203414917,
+      "logits/rejected": -3.4139163494110107,
+      "logps/chosen": -1.953155517578125,
+      "logps/rejected": -2.1329164505004883,
+      "loss": 1.1047,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.953155517578125,
+      "rewards/margins": 0.17976100742816925,
+      "rewards/rejected": -2.1329164505004883,
+      "step": 5390
+    },
+    {
+      "epoch": 0.9303928325292902,
+      "grad_norm": 10.745316691702477,
+      "learning_rate": 3.2293753913465856e-08,
+      "logits/chosen": -3.4513535499572754,
+      "logits/rejected": -3.442833662033081,
+      "logps/chosen": -1.8372522592544556,
+      "logps/rejected": -2.1769673824310303,
+      "loss": 1.0191,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.8372522592544556,
+      "rewards/margins": 0.3397153913974762,
+      "rewards/rejected": -2.1769673824310303,
+      "step": 5400
+    },
+    {
+      "epoch": 0.9303928325292902,
+      "eval_logits/chosen": -3.499239683151245,
+      "eval_logits/rejected": -3.495443105697632,
+      "eval_logps/chosen": -1.7615408897399902,
+      "eval_logps/rejected": -1.9350817203521729,
+      "eval_loss": 1.0946146249771118,
+      "eval_rewards/accuracies": 0.6259293556213379,
+      "eval_rewards/chosen": -1.7615408897399902,
+      "eval_rewards/margins": 0.1735408902168274,
+      "eval_rewards/rejected": -1.9350817203521729,
+      "eval_runtime": 155.3933,
+      "eval_samples_per_second": 27.697,
+      "eval_steps_per_second": 3.462,
+      "step": 5400
+    },
+    {
+      "epoch": 0.9321157822191593,
+      "grad_norm": 12.66308177107042,
+      "learning_rate": 3.2221812449777164e-08,
+      "logits/chosen": -3.4604580402374268,
+      "logits/rejected": -3.4538683891296387,
+      "logps/chosen": -1.9688161611557007,
+      "logps/rejected": -2.0536344051361084,
+      "loss": 1.1644,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.9688161611557007,
+      "rewards/margins": 0.08481796085834503,
+      "rewards/rejected": -2.0536344051361084,
+      "step": 5410
+    },
+    {
+      "epoch": 0.9338387319090282,
+      "grad_norm": 14.427262644656707,
+      "learning_rate": 3.214980567868242e-08,
+      "logits/chosen": -3.48225474357605,
+      "logits/rejected": -3.4708423614501953,
+      "logps/chosen": -1.8916904926300049,
+      "logps/rejected": -2.120081901550293,
+      "loss": 1.0731,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.8916904926300049,
+      "rewards/margins": 0.22839117050170898,
+      "rewards/rejected": -2.120081901550293,
+      "step": 5420
+    },
+    {
+      "epoch": 0.9355616815988973,
+      "grad_norm": 10.594352241139715,
+      "learning_rate": 3.2077734251344407e-08,
+      "logits/chosen": -3.4372878074645996,
+      "logits/rejected": -3.4297080039978027,
+      "logps/chosen": -2.0560569763183594,
+      "logps/rejected": -2.2732059955596924,
+      "loss": 1.1039,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -2.0560569763183594,
+      "rewards/margins": 0.217149019241333,
+      "rewards/rejected": -2.2732059955596924,
+      "step": 5430
+    },
+    {
+      "epoch": 0.9372846312887664,
+      "grad_norm": 13.592875439881716,
+      "learning_rate": 3.200559881951059e-08,
+      "logits/chosen": -3.4480159282684326,
+      "logits/rejected": -3.438587188720703,
+      "logps/chosen": -1.9630777835845947,
+      "logps/rejected": -2.1919970512390137,
+      "loss": 1.0776,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.9630777835845947,
+      "rewards/margins": 0.22891950607299805,
+      "rewards/rejected": -2.1919970512390137,
+      "step": 5440
+    },
+    {
+      "epoch": 0.9390075809786355,
+      "grad_norm": 10.853119723509183,
+      "learning_rate": 3.193340003550722e-08,
+      "logits/chosen": -3.40104603767395,
+      "logits/rejected": -3.392597198486328,
+      "logps/chosen": -1.9699275493621826,
+      "logps/rejected": -2.1187949180603027,
+      "loss": 1.1259,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.9699275493621826,
+      "rewards/margins": 0.14886775612831116,
+      "rewards/rejected": -2.1187949180603027,
+      "step": 5450
+    },
+    {
+      "epoch": 0.9407305306685044,
+      "grad_norm": 11.669213870754593,
+      "learning_rate": 3.186113855223348e-08,
+      "logits/chosen": -3.468956708908081,
+      "logits/rejected": -3.460797071456909,
+      "logps/chosen": -1.9514243602752686,
+      "logps/rejected": -2.0923609733581543,
+      "loss": 1.1365,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.9514243602752686,
+      "rewards/margins": 0.14093635976314545,
+      "rewards/rejected": -2.0923609733581543,
+      "step": 5460
+    },
+    {
+      "epoch": 0.9424534803583735,
+      "grad_norm": 13.072502115865696,
+      "learning_rate": 3.1788815023155517e-08,
+      "logits/chosen": -3.4268689155578613,
+      "logits/rejected": -3.414977550506592,
+      "logps/chosen": -1.9471731185913086,
+      "logps/rejected": -2.14667010307312,
+      "loss": 1.1025,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.9471731185913086,
+      "rewards/margins": 0.19949708878993988,
+      "rewards/rejected": -2.14667010307312,
+      "step": 5470
+    },
+    {
+      "epoch": 0.9441764300482426,
+      "grad_norm": 11.225776167415457,
+      "learning_rate": 3.171643010230057e-08,
+      "logits/chosen": -3.4417812824249268,
+      "logits/rejected": -3.428140163421631,
+      "logps/chosen": -1.9100406169891357,
+      "logps/rejected": -2.162745237350464,
+      "loss": 1.0657,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.9100406169891357,
+      "rewards/margins": 0.2527046203613281,
+      "rewards/rejected": -2.162745237350464,
+      "step": 5480
+    },
+    {
+      "epoch": 0.9458993797381117,
+      "grad_norm": 10.507610030267275,
+      "learning_rate": 3.1643984444251056e-08,
+      "logits/chosen": -3.4320228099823,
+      "logits/rejected": -3.41591215133667,
+      "logps/chosen": -1.9064576625823975,
+      "logps/rejected": -2.156628131866455,
+      "loss": 1.0759,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.9064576625823975,
+      "rewards/margins": 0.2501705586910248,
+      "rewards/rejected": -2.156628131866455,
+      "step": 5490
+    },
+    {
+      "epoch": 0.9476223294279807,
+      "grad_norm": 10.071652507065258,
+      "learning_rate": 3.157147870413864e-08,
+      "logits/chosen": -3.4817211627960205,
+      "logits/rejected": -3.471518039703369,
+      "logps/chosen": -1.8754154443740845,
+      "logps/rejected": -2.1743979454040527,
+      "loss": 1.0353,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.8754154443740845,
+      "rewards/margins": 0.2989824414253235,
+      "rewards/rejected": -2.1743979454040527,
+      "step": 5500
+    },
+    {
+      "epoch": 0.9476223294279807,
+      "eval_logits/chosen": -3.5040760040283203,
+      "eval_logits/rejected": -3.5003292560577393,
+      "eval_logps/chosen": -1.7636494636535645,
+      "eval_logps/rejected": -1.9373699426651,
+      "eval_loss": 1.0946518182754517,
+      "eval_rewards/accuracies": 0.625,
+      "eval_rewards/chosen": -1.7636494636535645,
+      "eval_rewards/margins": 0.1737208068370819,
+      "eval_rewards/rejected": -1.9373699426651,
+      "eval_runtime": 155.6083,
+      "eval_samples_per_second": 27.659,
+      "eval_steps_per_second": 3.457,
+      "step": 5500
+    },
+    {
+      "epoch": 0.9493452791178497,
+      "grad_norm": 10.436468774296491,
+      "learning_rate": 3.149891353763832e-08,
+      "logits/chosen": -3.435248613357544,
+      "logits/rejected": -3.4259541034698486,
+      "logps/chosen": -1.9294788837432861,
+      "logps/rejected": -2.1228866577148438,
+      "loss": 1.104,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.9294788837432861,
+      "rewards/margins": 0.1934075951576233,
+      "rewards/rejected": -2.1228866577148438,
+      "step": 5510
+    },
+    {
+      "epoch": 0.9510682288077188,
+      "grad_norm": 11.867708837662768,
+      "learning_rate": 3.142628960096246e-08,
+      "logits/chosen": -3.4226555824279785,
+      "logits/rejected": -3.4067091941833496,
+      "logps/chosen": -1.818102478981018,
+      "logps/rejected": -2.084951400756836,
+      "loss": 1.0608,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.818102478981018,
+      "rewards/margins": 0.2668488323688507,
+      "rewards/rejected": -2.084951400756836,
+      "step": 5520
+    },
+    {
+      "epoch": 0.9527911784975879,
+      "grad_norm": 11.528775493564515,
+      "learning_rate": 3.1353607550854935e-08,
+      "logits/chosen": -3.4436473846435547,
+      "logits/rejected": -3.425518751144409,
+      "logps/chosen": -1.8911097049713135,
+      "logps/rejected": -2.153690814971924,
+      "loss": 1.0732,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.8911097049713135,
+      "rewards/margins": 0.2625811696052551,
+      "rewards/rejected": -2.153690814971924,
+      "step": 5530
+    },
+    {
+      "epoch": 0.954514128187457,
+      "grad_norm": 10.707382038502208,
+      "learning_rate": 3.12808680445851e-08,
+      "logits/chosen": -3.4300131797790527,
+      "logits/rejected": -3.4323909282684326,
+      "logps/chosen": -1.9335225820541382,
+      "logps/rejected": -2.185945987701416,
+      "loss": 1.0767,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.9335225820541382,
+      "rewards/margins": 0.2524234652519226,
+      "rewards/rejected": -2.185945987701416,
+      "step": 5540
+    },
+    {
+      "epoch": 0.956237077877326,
+      "grad_norm": 11.604228956484173,
+      "learning_rate": 3.120807173994194e-08,
+      "logits/chosen": -3.3598690032958984,
+      "logits/rejected": -3.350546360015869,
+      "logps/chosen": -1.9253263473510742,
+      "logps/rejected": -2.138554334640503,
+      "loss": 1.0782,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.9253263473510742,
+      "rewards/margins": 0.21322818100452423,
+      "rewards/rejected": -2.138554334640503,
+      "step": 5550
+    },
+    {
+      "epoch": 0.957960027567195,
+      "grad_norm": 11.40847419800857,
+      "learning_rate": 3.1135219295228014e-08,
+      "logits/chosen": -3.442878246307373,
+      "logits/rejected": -3.429008960723877,
+      "logps/chosen": -1.9418971538543701,
+      "logps/rejected": -2.144742965698242,
+      "loss": 1.094,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.9418971538543701,
+      "rewards/margins": 0.20284590125083923,
+      "rewards/rejected": -2.144742965698242,
+      "step": 5560
+    },
+    {
+      "epoch": 0.9596829772570641,
+      "grad_norm": 10.209874208915254,
+      "learning_rate": 3.1062311369253604e-08,
+      "logits/chosen": -3.456080198287964,
+      "logits/rejected": -3.4506983757019043,
+      "logps/chosen": -1.7942874431610107,
+      "logps/rejected": -2.149440050125122,
+      "loss": 1.0021,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.7942874431610107,
+      "rewards/margins": 0.35515251755714417,
+      "rewards/rejected": -2.149440050125122,
+      "step": 5570
+    },
+    {
+      "epoch": 0.9614059269469332,
+      "grad_norm": 10.852560990985202,
+      "learning_rate": 3.0989348621330694e-08,
+      "logits/chosen": -3.381937026977539,
+      "logits/rejected": -3.3736672401428223,
+      "logps/chosen": -1.977386474609375,
+      "logps/rejected": -2.187953472137451,
+      "loss": 1.0986,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.977386474609375,
+      "rewards/margins": 0.21056702733039856,
+      "rewards/rejected": -2.187953472137451,
+      "step": 5580
+    },
+    {
+      "epoch": 0.9631288766368022,
+      "grad_norm": 9.980379999333222,
+      "learning_rate": 3.091633171126703e-08,
+      "logits/chosen": -3.48820161819458,
+      "logits/rejected": -3.4641709327697754,
+      "logps/chosen": -1.8324105739593506,
+      "logps/rejected": -2.250082015991211,
+      "loss": 0.9707,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.8324105739593506,
+      "rewards/margins": 0.4176712930202484,
+      "rewards/rejected": -2.250082015991211,
+      "step": 5590
+    },
+    {
+      "epoch": 0.9648518263266712,
+      "grad_norm": 11.10077534984661,
+      "learning_rate": 3.0843261299360165e-08,
+      "logits/chosen": -3.465712785720825,
+      "logits/rejected": -3.4645752906799316,
+      "logps/chosen": -1.959644079208374,
+      "logps/rejected": -2.152742624282837,
+      "loss": 1.0994,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.959644079208374,
+      "rewards/margins": 0.19309823215007782,
+      "rewards/rejected": -2.152742624282837,
+      "step": 5600
+    },
+    {
+      "epoch": 0.9648518263266712,
+      "eval_logits/chosen": -3.486161470413208,
+      "eval_logits/rejected": -3.482330560684204,
+      "eval_logps/chosen": -1.7649174928665161,
+      "eval_logps/rejected": -1.9397294521331787,
+      "eval_loss": 1.0941563844680786,
+      "eval_rewards/accuracies": 0.625464677810669,
+      "eval_rewards/chosen": -1.7649174928665161,
+      "eval_rewards/margins": 0.1748119443655014,
+      "eval_rewards/rejected": -1.9397294521331787,
+      "eval_runtime": 155.6904,
+      "eval_samples_per_second": 27.645,
+      "eval_steps_per_second": 3.456,
+      "step": 5600
+    },
+    {
+      "epoch": 0.9665747760165403,
+      "grad_norm": 10.407552898649113,
+      "learning_rate": 3.077013804639144e-08,
+      "logits/chosen": -3.469942092895508,
+      "logits/rejected": -3.45995831489563,
+      "logps/chosen": -1.893750786781311,
+      "logps/rejected": -2.2904114723205566,
+      "loss": 0.9838,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.893750786781311,
+      "rewards/margins": 0.3966609835624695,
+      "rewards/rejected": -2.2904114723205566,
+      "step": 5610
+    },
+    {
+      "epoch": 0.9682977257064094,
+      "grad_norm": 12.979362827210618,
+      "learning_rate": 3.069696261362008e-08,
+      "logits/chosen": -3.4271857738494873,
+      "logits/rejected": -3.4148871898651123,
+      "logps/chosen": -1.9377062320709229,
+      "logps/rejected": -2.10105562210083,
+      "loss": 1.1075,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.9377062320709229,
+      "rewards/margins": 0.1633491963148117,
+      "rewards/rejected": -2.10105562210083,
+      "step": 5620
+    },
+    {
+      "epoch": 0.9700206753962785,
+      "grad_norm": 10.811015647184098,
+      "learning_rate": 3.062373566277715e-08,
+      "logits/chosen": -3.4621880054473877,
+      "logits/rejected": -3.447049379348755,
+      "logps/chosen": -1.893935203552246,
+      "logps/rejected": -2.037522077560425,
+      "loss": 1.122,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.893935203552246,
+      "rewards/margins": 0.14358706772327423,
+      "rewards/rejected": -2.037522077560425,
+      "step": 5630
+    },
+    {
+      "epoch": 0.9717436250861475,
+      "grad_norm": 12.15690827405645,
+      "learning_rate": 3.0550457856059594e-08,
+      "logits/chosen": -3.4393699169158936,
+      "logits/rejected": -3.4313175678253174,
+      "logps/chosen": -1.8234927654266357,
+      "logps/rejected": -2.07340407371521,
+      "loss": 1.0638,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.8234927654266357,
+      "rewards/margins": 0.24991145730018616,
+      "rewards/rejected": -2.07340407371521,
+      "step": 5640
+    },
+    {
+      "epoch": 0.9734665747760165,
+      "grad_norm": 12.551500951847954,
+      "learning_rate": 3.047712985612428e-08,
+      "logits/chosen": -3.3871376514434814,
+      "logits/rejected": -3.3833553791046143,
+      "logps/chosen": -1.9479678869247437,
+      "logps/rejected": -2.074371814727783,
+      "loss": 1.1409,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -1.9479678869247437,
+      "rewards/margins": 0.12640398740768433,
+      "rewards/rejected": -2.074371814727783,
+      "step": 5650
+    },
+    {
+      "epoch": 0.9751895244658856,
+      "grad_norm": 11.23586763218128,
+      "learning_rate": 3.040375232608194e-08,
+      "logits/chosen": -3.3921051025390625,
+      "logits/rejected": -3.3900933265686035,
+      "logps/chosen": -1.9238033294677734,
+      "logps/rejected": -1.9988281726837158,
+      "loss": 1.1692,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.9238033294677734,
+      "rewards/margins": 0.07502470910549164,
+      "rewards/rejected": -1.9988281726837158,
+      "step": 5660
+    },
+    {
+      "epoch": 0.9769124741557547,
+      "grad_norm": 10.80610834080807,
+      "learning_rate": 3.033032592949125e-08,
+      "logits/chosen": -3.424856185913086,
+      "logits/rejected": -3.410578966140747,
+      "logps/chosen": -1.862491250038147,
+      "logps/rejected": -2.119480609893799,
+      "loss": 1.0565,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.862491250038147,
+      "rewards/margins": 0.25698941946029663,
+      "rewards/rejected": -2.119480609893799,
+      "step": 5670
+    },
+    {
+      "epoch": 0.9786354238456237,
+      "grad_norm": 10.923425421515237,
+      "learning_rate": 3.025685133035275e-08,
+      "logits/chosen": -3.450428009033203,
+      "logits/rejected": -3.428873062133789,
+      "logps/chosen": -1.9678428173065186,
+      "logps/rejected": -2.2834839820861816,
+      "loss": 1.0139,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.9678428173065186,
+      "rewards/margins": 0.31564104557037354,
+      "rewards/rejected": -2.2834839820861816,
+      "step": 5680
+    },
+    {
+      "epoch": 0.9803583735354927,
+      "grad_norm": 10.692385651577954,
+      "learning_rate": 3.0183329193102894e-08,
+      "logits/chosen": -3.46785044670105,
+      "logits/rejected": -3.4515082836151123,
+      "logps/chosen": -1.9069669246673584,
+      "logps/rejected": -2.066882610321045,
+      "loss": 1.106,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.9069669246673584,
+      "rewards/margins": 0.1599154770374298,
+      "rewards/rejected": -2.066882610321045,
+      "step": 5690
+    },
+    {
+      "epoch": 0.9820813232253618,
+      "grad_norm": 12.963419407251164,
+      "learning_rate": 3.0109760182608054e-08,
+      "logits/chosen": -3.337448835372925,
+      "logits/rejected": -3.326279401779175,
+      "logps/chosen": -1.9296462535858154,
+      "logps/rejected": -2.109297275543213,
+      "loss": 1.1142,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.9296462535858154,
+      "rewards/margins": 0.17965126037597656,
+      "rewards/rejected": -2.109297275543213,
+      "step": 5700
+    },
+    {
+      "epoch": 0.9820813232253618,
+      "eval_logits/chosen": -3.504169225692749,
+      "eval_logits/rejected": -3.500457763671875,
+      "eval_logps/chosen": -1.770466685295105,
+      "eval_logps/rejected": -1.9459874629974365,
+      "eval_loss": 1.093935251235962,
+      "eval_rewards/accuracies": 0.6252323389053345,
+      "eval_rewards/chosen": -1.770466685295105,
+      "eval_rewards/margins": 0.17552082240581512,
+      "eval_rewards/rejected": -1.9459874629974365,
+      "eval_runtime": 155.7936,
+      "eval_samples_per_second": 27.626,
+      "eval_steps_per_second": 3.453,
+      "step": 5700
+    },
+    {
+      "epoch": 0.9838042729152309,
+      "grad_norm": 11.966025959217951,
+      "learning_rate": 3.0036144964158425e-08,
+      "logits/chosen": -3.4585750102996826,
+      "logits/rejected": -3.442483425140381,
+      "logps/chosen": -1.936847448348999,
+      "logps/rejected": -2.222085475921631,
+      "loss": 1.0433,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.936847448348999,
+      "rewards/margins": 0.2852379381656647,
+      "rewards/rejected": -2.222085475921631,
+      "step": 5710
+    },
+    {
+      "epoch": 0.9855272226051,
+      "grad_norm": 11.466403351800416,
+      "learning_rate": 2.9962484203462114e-08,
+      "logits/chosen": -3.4492945671081543,
+      "logits/rejected": -3.4369537830352783,
+      "logps/chosen": -1.9131911993026733,
+      "logps/rejected": -2.1684608459472656,
+      "loss": 1.0843,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.9131911993026733,
+      "rewards/margins": 0.25526970624923706,
+      "rewards/rejected": -2.1684608459472656,
+      "step": 5720
+    },
+    {
+      "epoch": 0.987250172294969,
+      "grad_norm": 11.073662433550027,
+      "learning_rate": 2.988877856663905e-08,
+      "logits/chosen": -3.471259355545044,
+      "logits/rejected": -3.464627504348755,
+      "logps/chosen": -1.8858661651611328,
+      "logps/rejected": -2.107440710067749,
+      "loss": 1.0848,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.8858661651611328,
+      "rewards/margins": 0.2215747833251953,
+      "rewards/rejected": -2.107440710067749,
+      "step": 5730
+    },
+    {
+      "epoch": 0.988973121984838,
+      "grad_norm": 10.296051618193514,
+      "learning_rate": 2.9815028720214984e-08,
+      "logits/chosen": -3.416987657546997,
+      "logits/rejected": -3.399951934814453,
+      "logps/chosen": -1.9954261779785156,
+      "logps/rejected": -2.28566837310791,
+      "loss": 1.0468,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.9954261779785156,
+      "rewards/margins": 0.29024219512939453,
+      "rewards/rejected": -2.28566837310791,
+      "step": 5740
+    },
+    {
+      "epoch": 0.9906960716747071,
+      "grad_norm": 12.234968687641018,
+      "learning_rate": 2.974123533111545e-08,
+      "logits/chosen": -3.5080199241638184,
+      "logits/rejected": -3.4912285804748535,
+      "logps/chosen": -1.9873214960098267,
+      "logps/rejected": -2.1998019218444824,
+      "loss": 1.0802,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.9873214960098267,
+      "rewards/margins": 0.212480828166008,
+      "rewards/rejected": -2.1998019218444824,
+      "step": 5750
+    },
+    {
+      "epoch": 0.9924190213645762,
+      "grad_norm": 10.664606080873053,
+      "learning_rate": 2.9667399066659754e-08,
+      "logits/chosen": -3.4059157371520996,
+      "logits/rejected": -3.3916525840759277,
+      "logps/chosen": -1.9250662326812744,
+      "logps/rejected": -2.19187068939209,
+      "loss": 1.0648,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.9250662326812744,
+      "rewards/margins": 0.26680439710617065,
+      "rewards/rejected": -2.19187068939209,
+      "step": 5760
+    },
+    {
+      "epoch": 0.9941419710544452,
+      "grad_norm": 10.017986084118135,
+      "learning_rate": 2.959352059455492e-08,
+      "logits/chosen": -3.3989086151123047,
+      "logits/rejected": -3.389204740524292,
+      "logps/chosen": -1.8752634525299072,
+      "logps/rejected": -2.137773275375366,
+      "loss": 1.0477,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.8752634525299072,
+      "rewards/margins": 0.26250970363616943,
+      "rewards/rejected": -2.137773275375366,
+      "step": 5770
+    },
+    {
+      "epoch": 0.9958649207443143,
+      "grad_norm": 12.259049062628025,
+      "learning_rate": 2.9519600582889654e-08,
+      "logits/chosen": -3.421304702758789,
+      "logits/rejected": -3.410816192626953,
+      "logps/chosen": -1.8800327777862549,
+      "logps/rejected": -2.1815009117126465,
+      "loss": 1.0319,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.8800327777862549,
+      "rewards/margins": 0.301468163728714,
+      "rewards/rejected": -2.1815009117126465,
+      "step": 5780
+    },
+    {
+      "epoch": 0.9975878704341833,
+      "grad_norm": 11.809267704977925,
+      "learning_rate": 2.944563970012831e-08,
+      "logits/chosen": -3.378391742706299,
+      "logits/rejected": -3.3610942363739014,
+      "logps/chosen": -1.906667709350586,
+      "logps/rejected": -2.2171921730041504,
+      "loss": 1.06,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.906667709350586,
+      "rewards/margins": 0.31052452325820923,
+      "rewards/rejected": -2.2171921730041504,
+      "step": 5790
+    },
+    {
+      "epoch": 0.9993108201240524,
+      "grad_norm": 13.234757427711886,
+      "learning_rate": 2.937163861510486e-08,
+      "logits/chosen": -3.41619610786438,
+      "logits/rejected": -3.4022960662841797,
+      "logps/chosen": -1.8752868175506592,
+      "logps/rejected": -2.2479007244110107,
+      "loss": 1.0105,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.8752868175506592,
+      "rewards/margins": 0.37261390686035156,
+      "rewards/rejected": -2.2479007244110107,
+      "step": 5800
+    },
+    {
+      "epoch": 0.9993108201240524,
+      "eval_logits/chosen": -3.494699478149414,
+      "eval_logits/rejected": -3.4909520149230957,
+      "eval_logps/chosen": -1.780420184135437,
+      "eval_logps/rejected": -1.957065463066101,
+      "eval_loss": 1.0933873653411865,
+      "eval_rewards/accuracies": 0.624535322189331,
+      "eval_rewards/chosen": -1.780420184135437,
+      "eval_rewards/margins": 0.17664550244808197,
+      "eval_rewards/rejected": -1.957065463066101,
+      "eval_runtime": 155.4169,
+      "eval_samples_per_second": 27.693,
+      "eval_steps_per_second": 3.462,
+      "step": 5800
+    },
+    {
+      "epoch": 1.0010337698139213,
+      "grad_norm": 9.419000379111106,
+      "learning_rate": 2.92975979970168e-08,
+      "logits/chosen": -3.4653656482696533,
+      "logits/rejected": -3.457083225250244,
+      "logps/chosen": -2.0171971321105957,
+      "logps/rejected": -2.206838369369507,
+      "loss": 1.1109,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -2.0171971321105957,
+      "rewards/margins": 0.18964163959026337,
+      "rewards/rejected": -2.206838369369507,
+      "step": 5810
+    },
+    {
+      "epoch": 1.0027567195037905,
+      "grad_norm": 12.80136945902006,
+      "learning_rate": 2.9223518515419147e-08,
+      "logits/chosen": -3.4702281951904297,
+      "logits/rejected": -3.45373272895813,
+      "logps/chosen": -1.9455616474151611,
+      "logps/rejected": -2.2249858379364014,
+      "loss": 1.049,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.9455616474151611,
+      "rewards/margins": 0.27942436933517456,
+      "rewards/rejected": -2.2249858379364014,
+      "step": 5820
+    },
+    {
+      "epoch": 1.0044796691936595,
+      "grad_norm": 10.774772380665492,
+      "learning_rate": 2.914940084021836e-08,
+      "logits/chosen": -3.3917412757873535,
+      "logits/rejected": -3.373875379562378,
+      "logps/chosen": -1.8685290813446045,
+      "logps/rejected": -2.2018942832946777,
+      "loss": 1.0216,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.8685290813446045,
+      "rewards/margins": 0.3333651125431061,
+      "rewards/rejected": -2.2018942832946777,
+      "step": 5830
+    },
+    {
+      "epoch": 1.0062026188835287,
+      "grad_norm": 10.161576810480579,
+      "learning_rate": 2.9075245641666278e-08,
+      "logits/chosen": -3.4057059288024902,
+      "logits/rejected": -3.3939995765686035,
+      "logps/chosen": -1.8997621536254883,
+      "logps/rejected": -2.3102364540100098,
+      "loss": 0.9838,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.8997621536254883,
+      "rewards/margins": 0.4104744791984558,
+      "rewards/rejected": -2.3102364540100098,
+      "step": 5840
+    },
+    {
+      "epoch": 1.0079255685733977,
+      "grad_norm": 11.407180189207624,
+      "learning_rate": 2.9001053590354075e-08,
+      "logits/chosen": -3.458336353302002,
+      "logits/rejected": -3.4463584423065186,
+      "logps/chosen": -1.8606325387954712,
+      "logps/rejected": -2.219241142272949,
+      "loss": 1.0116,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.8606325387954712,
+      "rewards/margins": 0.35860857367515564,
+      "rewards/rejected": -2.219241142272949,
+      "step": 5850
+    },
+    {
+      "epoch": 1.0096485182632666,
+      "grad_norm": 12.294297414266158,
+      "learning_rate": 2.8926825357206174e-08,
+      "logits/chosen": -3.379124879837036,
+      "logits/rejected": -3.3719000816345215,
+      "logps/chosen": -1.9951471090316772,
+      "logps/rejected": -2.27166485786438,
+      "loss": 1.0784,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.9951471090316772,
+      "rewards/margins": 0.2765175402164459,
+      "rewards/rejected": -2.27166485786438,
+      "step": 5860
+    },
+    {
+      "epoch": 1.0113714679531358,
+      "grad_norm": 11.662877799653499,
+      "learning_rate": 2.8852561613474213e-08,
+      "logits/chosen": -3.3881640434265137,
+      "logits/rejected": -3.376826524734497,
+      "logps/chosen": -1.9656829833984375,
+      "logps/rejected": -2.307656764984131,
+      "loss": 1.0596,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.9656829833984375,
+      "rewards/margins": 0.34197378158569336,
+      "rewards/rejected": -2.307656764984131,
+      "step": 5870
+    },
+    {
+      "epoch": 1.0130944176430048,
+      "grad_norm": 11.66472630756115,
+      "learning_rate": 2.8778263030730937e-08,
+      "logits/chosen": -3.437969207763672,
+      "logits/rejected": -3.431363344192505,
+      "logps/chosen": -1.914284348487854,
+      "logps/rejected": -2.1178836822509766,
+      "loss": 1.1048,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.914284348487854,
+      "rewards/margins": 0.203599214553833,
+      "rewards/rejected": -2.1178836822509766,
+      "step": 5880
+    },
+    {
+      "epoch": 1.014817367332874,
+      "grad_norm": 11.273507973414482,
+      "learning_rate": 2.8703930280864165e-08,
+      "logits/chosen": -3.450343370437622,
+      "logits/rejected": -3.4426937103271484,
+      "logps/chosen": -1.880518913269043,
+      "logps/rejected": -2.1358580589294434,
+      "loss": 1.0675,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.880518913269043,
+      "rewards/margins": 0.2553390860557556,
+      "rewards/rejected": -2.1358580589294434,
+      "step": 5890
+    },
+    {
+      "epoch": 1.016540317022743,
+      "grad_norm": 9.617253181009701,
+      "learning_rate": 2.8629564036070662e-08,
+      "logits/chosen": -3.4028522968292236,
+      "logits/rejected": -3.392329692840576,
+      "logps/chosen": -1.9530251026153564,
+      "logps/rejected": -2.2224507331848145,
+      "loss": 1.0585,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.9530251026153564,
+      "rewards/margins": 0.2694256901741028,
+      "rewards/rejected": -2.2224507331848145,
+      "step": 5900
+    },
+    {
+      "epoch": 1.016540317022743,
+      "eval_logits/chosen": -3.4888105392456055,
+      "eval_logits/rejected": -3.4850564002990723,
+      "eval_logps/chosen": -1.7831361293792725,
+      "eval_logps/rejected": -1.9605830907821655,
+      "eval_loss": 1.0931737422943115,
+      "eval_rewards/accuracies": 0.6231412887573242,
+      "eval_rewards/chosen": -1.7831361293792725,
+      "eval_rewards/margins": 0.17744717001914978,
+      "eval_rewards/rejected": -1.9605830907821655,
+      "eval_runtime": 155.6914,
+      "eval_samples_per_second": 27.644,
+      "eval_steps_per_second": 3.456,
+      "step": 5900
+    },
+    {
+      "epoch": 1.018263266712612,
+      "grad_norm": 11.67358469568351,
+      "learning_rate": 2.8555164968850108e-08,
+      "logits/chosen": -3.3998191356658936,
+      "logits/rejected": -3.401618480682373,
+      "logps/chosen": -1.9379875659942627,
+      "logps/rejected": -2.140415906906128,
+      "loss": 1.0908,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.9379875659942627,
+      "rewards/margins": 0.20242838561534882,
+      "rewards/rejected": -2.140415906906128,
+      "step": 5910
+    },
+    {
+      "epoch": 1.019986216402481,
+      "grad_norm": 11.964126464330255,
+      "learning_rate": 2.848073375199901e-08,
+      "logits/chosen": -3.4383766651153564,
+      "logits/rejected": -3.4299206733703613,
+      "logps/chosen": -1.9241117238998413,
+      "logps/rejected": -2.171543598175049,
+      "loss": 1.0849,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.9241117238998413,
+      "rewards/margins": 0.247431680560112,
+      "rewards/rejected": -2.171543598175049,
+      "step": 5920
+    },
+    {
+      "epoch": 1.02170916609235,
+      "grad_norm": 11.209567514475495,
+      "learning_rate": 2.8406271058604575e-08,
+      "logits/chosen": -3.4287784099578857,
+      "logits/rejected": -3.426023006439209,
+      "logps/chosen": -1.9696004390716553,
+      "logps/rejected": -2.1504530906677246,
+      "loss": 1.1062,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.9696004390716553,
+      "rewards/margins": 0.18085305392742157,
+      "rewards/rejected": -2.1504530906677246,
+      "step": 5930
+    },
+    {
+      "epoch": 1.0234321157822193,
+      "grad_norm": 9.635418244238455,
+      "learning_rate": 2.8331777562038677e-08,
+      "logits/chosen": -3.4429988861083984,
+      "logits/rejected": -3.420043468475342,
+      "logps/chosen": -1.8543379306793213,
+      "logps/rejected": -2.2759554386138916,
+      "loss": 0.9795,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.8543379306793213,
+      "rewards/margins": 0.4216178357601166,
+      "rewards/rejected": -2.2759554386138916,
+      "step": 5940
+    },
+    {
+      "epoch": 1.0251550654720882,
+      "grad_norm": 10.559957087735315,
+      "learning_rate": 2.8257253935951754e-08,
+      "logits/chosen": -3.353259563446045,
+      "logits/rejected": -3.3526992797851562,
+      "logps/chosen": -1.9518934488296509,
+      "logps/rejected": -2.161328077316284,
+      "loss": 1.0949,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.9518934488296509,
+      "rewards/margins": 0.20943458378314972,
+      "rewards/rejected": -2.161328077316284,
+      "step": 5950
+    },
+    {
+      "epoch": 1.0268780151619572,
+      "grad_norm": 14.699267262626657,
+      "learning_rate": 2.8182700854266677e-08,
+      "logits/chosen": -3.383129596710205,
+      "logits/rejected": -3.3555309772491455,
+      "logps/chosen": -1.8859987258911133,
+      "logps/rejected": -2.1602537631988525,
+      "loss": 1.0478,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.8859987258911133,
+      "rewards/margins": 0.2742549479007721,
+      "rewards/rejected": -2.1602537631988525,
+      "step": 5960
+    },
+    {
+      "epoch": 1.0286009648518264,
+      "grad_norm": 11.439974553793625,
+      "learning_rate": 2.8108118991172713e-08,
+      "logits/chosen": -3.382798433303833,
+      "logits/rejected": -3.374568223953247,
+      "logps/chosen": -1.9688291549682617,
+      "logps/rejected": -2.21622633934021,
+      "loss": 1.0687,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.9688291549682617,
+      "rewards/margins": 0.24739718437194824,
+      "rewards/rejected": -2.21622633934021,
+      "step": 5970
+    },
+    {
+      "epoch": 1.0303239145416954,
+      "grad_norm": 12.532823259426669,
+      "learning_rate": 2.8033509021119394e-08,
+      "logits/chosen": -3.391226291656494,
+      "logits/rejected": -3.390462875366211,
+      "logps/chosen": -1.896564245223999,
+      "logps/rejected": -2.1574392318725586,
+      "loss": 1.069,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.896564245223999,
+      "rewards/margins": 0.26087483763694763,
+      "rewards/rejected": -2.1574392318725586,
+      "step": 5980
+    },
+    {
+      "epoch": 1.0320468642315643,
+      "grad_norm": 10.934384813583685,
+      "learning_rate": 2.7958871618810432e-08,
+      "logits/chosen": -3.4284629821777344,
+      "logits/rejected": -3.409000873565674,
+      "logps/chosen": -1.9060583114624023,
+      "logps/rejected": -2.2427003383636475,
+      "loss": 1.0391,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.9060583114624023,
+      "rewards/margins": 0.3366420269012451,
+      "rewards/rejected": -2.2427003383636475,
+      "step": 5990
+    },
+    {
+      "epoch": 1.0337698139214335,
+      "grad_norm": 10.814825183508534,
+      "learning_rate": 2.7884207459197584e-08,
+      "logits/chosen": -3.422478199005127,
+      "logits/rejected": -3.4131920337677,
+      "logps/chosen": -1.9562642574310303,
+      "logps/rejected": -2.224428176879883,
+      "loss": 1.05,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.9562642574310303,
+      "rewards/margins": 0.26816365122795105,
+      "rewards/rejected": -2.224428176879883,
+      "step": 6000
+    },
+    {
+      "epoch": 1.0337698139214335,
+      "eval_logits/chosen": -3.4893136024475098,
+      "eval_logits/rejected": -3.4855756759643555,
+      "eval_logps/chosen": -1.784942865371704,
+      "eval_logps/rejected": -1.9626952409744263,
+      "eval_loss": 1.092982292175293,
+      "eval_rewards/accuracies": 0.6231412887573242,
+      "eval_rewards/chosen": -1.784942865371704,
+      "eval_rewards/margins": 0.1777522712945938,
+      "eval_rewards/rejected": -1.9626952409744263,
+      "eval_runtime": 155.3089,
+      "eval_samples_per_second": 27.713,
+      "eval_steps_per_second": 3.464,
+      "step": 6000
+    },
+    {
+      "epoch": 1.0354927636113025,
+      "grad_norm": 11.924915133819274,
+      "learning_rate": 2.780951721747461e-08,
+      "logits/chosen": -3.4151558876037598,
+      "logits/rejected": -3.405982494354248,
+      "logps/chosen": -1.973700761795044,
+      "logps/rejected": -2.2199769020080566,
+      "loss": 1.084,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.973700761795044,
+      "rewards/margins": 0.2462763786315918,
+      "rewards/rejected": -2.2199769020080566,
+      "step": 6010
+    },
+    {
+      "epoch": 1.0372157133011717,
+      "grad_norm": 10.023726158147506,
+      "learning_rate": 2.7734801569071104e-08,
+      "logits/chosen": -3.5192818641662598,
+      "logits/rejected": -3.500136137008667,
+      "logps/chosen": -1.8804823160171509,
+      "logps/rejected": -2.106300115585327,
+      "loss": 1.0942,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.8804823160171509,
+      "rewards/margins": 0.22581800818443298,
+      "rewards/rejected": -2.106300115585327,
+      "step": 6020
+    },
+    {
+      "epoch": 1.0389386629910407,
+      "grad_norm": 10.89143471900147,
+      "learning_rate": 2.766006118964644e-08,
+      "logits/chosen": -3.3081862926483154,
+      "logits/rejected": -3.305790424346924,
+      "logps/chosen": -2.0266401767730713,
+      "logps/rejected": -2.1995091438293457,
+      "loss": 1.1537,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -2.0266401767730713,
+      "rewards/margins": 0.17286917567253113,
+      "rewards/rejected": -2.1995091438293457,
+      "step": 6030
+    },
+    {
+      "epoch": 1.0406616126809096,
+      "grad_norm": 18.244287155096934,
+      "learning_rate": 2.7585296755083613e-08,
+      "logits/chosen": -3.432079792022705,
+      "logits/rejected": -3.421729326248169,
+      "logps/chosen": -1.9720739126205444,
+      "logps/rejected": -2.1752769947052,
+      "loss": 1.1041,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.9720739126205444,
+      "rewards/margins": 0.2032029926776886,
+      "rewards/rejected": -2.1752769947052,
+      "step": 6040
+    },
+    {
+      "epoch": 1.0423845623707788,
+      "grad_norm": 10.555962095137232,
+      "learning_rate": 2.751050894148317e-08,
+      "logits/chosen": -3.3654983043670654,
+      "logits/rejected": -3.3513355255126953,
+      "logps/chosen": -2.001335382461548,
+      "logps/rejected": -2.1786954402923584,
+      "loss": 1.1214,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -2.001335382461548,
+      "rewards/margins": 0.177359938621521,
+      "rewards/rejected": -2.1786954402923584,
+      "step": 6050
+    },
+    {
+      "epoch": 1.0441075120606478,
+      "grad_norm": 11.134238513965448,
+      "learning_rate": 2.7435698425157065e-08,
+      "logits/chosen": -3.4121062755584717,
+      "logits/rejected": -3.3993868827819824,
+      "logps/chosen": -1.9145278930664062,
+      "logps/rejected": -2.142988920211792,
+      "loss": 1.0747,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.9145278930664062,
+      "rewards/margins": 0.22846117615699768,
+      "rewards/rejected": -2.142988920211792,
+      "step": 6060
+    },
+    {
+      "epoch": 1.045830461750517,
+      "grad_norm": 12.558165424190443,
+      "learning_rate": 2.7360865882622558e-08,
+      "logits/chosen": -3.424063205718994,
+      "logits/rejected": -3.413264036178589,
+      "logps/chosen": -1.9536945819854736,
+      "logps/rejected": -2.1822805404663086,
+      "loss": 1.0745,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.9536945819854736,
+      "rewards/margins": 0.2285860776901245,
+      "rewards/rejected": -2.1822805404663086,
+      "step": 6070
+    },
+    {
+      "epoch": 1.047553411440386,
+      "grad_norm": 10.608590009199519,
+      "learning_rate": 2.7286011990596092e-08,
+      "logits/chosen": -3.3986144065856934,
+      "logits/rejected": -3.3866569995880127,
+      "logps/chosen": -1.961950659751892,
+      "logps/rejected": -2.2214646339416504,
+      "loss": 1.0672,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.961950659751892,
+      "rewards/margins": 0.25951382517814636,
+      "rewards/rejected": -2.2214646339416504,
+      "step": 6080
+    },
+    {
+      "epoch": 1.049276361130255,
+      "grad_norm": 12.471154227750391,
+      "learning_rate": 2.7211137425987175e-08,
+      "logits/chosen": -3.4434027671813965,
+      "logits/rejected": -3.435576915740967,
+      "logps/chosen": -1.8881969451904297,
+      "logps/rejected": -2.1372053623199463,
+      "loss": 1.0524,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.8881969451904297,
+      "rewards/margins": 0.24900826811790466,
+      "rewards/rejected": -2.1372053623199463,
+      "step": 6090
+    },
+    {
+      "epoch": 1.050999310820124,
+      "grad_norm": 10.84640819800594,
+      "learning_rate": 2.7136242865892268e-08,
+      "logits/chosen": -3.4154045581817627,
+      "logits/rejected": -3.414189100265503,
+      "logps/chosen": -2.0179309844970703,
+      "logps/rejected": -2.141340732574463,
+      "loss": 1.1418,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -2.0179309844970703,
+      "rewards/margins": 0.12340925633907318,
+      "rewards/rejected": -2.141340732574463,
+      "step": 6100
+    },
+    {
+      "epoch": 1.050999310820124,
+      "eval_logits/chosen": -3.4878828525543213,
+      "eval_logits/rejected": -3.4841556549072266,
+      "eval_logps/chosen": -1.7910230159759521,
+      "eval_logps/rejected": -1.9698712825775146,
+      "eval_loss": 1.092617392539978,
+      "eval_rewards/accuracies": 0.625,
+      "eval_rewards/chosen": -1.7910230159759521,
+      "eval_rewards/margins": 0.17884840071201324,
+      "eval_rewards/rejected": -1.9698712825775146,
+      "eval_runtime": 155.602,
+      "eval_samples_per_second": 27.66,
+      "eval_steps_per_second": 3.458,
+      "step": 6100
+    },
+    {
+      "epoch": 1.052722260509993,
+      "grad_norm": 11.444205244768941,
+      "learning_rate": 2.7061328987588627e-08,
+      "logits/chosen": -3.4320037364959717,
+      "logits/rejected": -3.422847270965576,
+      "logps/chosen": -1.928194284439087,
+      "logps/rejected": -2.1602349281311035,
+      "loss": 1.0677,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.928194284439087,
+      "rewards/margins": 0.23204056918621063,
+      "rewards/rejected": -2.1602349281311035,
+      "step": 6110
+    },
+    {
+      "epoch": 1.0544452101998623,
+      "grad_norm": 9.027586335188923,
+      "learning_rate": 2.698639646852824e-08,
+      "logits/chosen": -3.484250545501709,
+      "logits/rejected": -3.458054304122925,
+      "logps/chosen": -1.8272943496704102,
+      "logps/rejected": -2.263728380203247,
+      "loss": 0.9633,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -1.8272943496704102,
+      "rewards/margins": 0.4364340305328369,
+      "rewards/rejected": -2.263728380203247,
+      "step": 6120
+    },
+    {
+      "epoch": 1.0561681598897312,
+      "grad_norm": 12.425457884530351,
+      "learning_rate": 2.6911445986331633e-08,
+      "logits/chosen": -3.426913022994995,
+      "logits/rejected": -3.414134979248047,
+      "logps/chosen": -1.942604422569275,
+      "logps/rejected": -2.295293092727661,
+      "loss": 1.0129,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.942604422569275,
+      "rewards/margins": 0.35268840193748474,
+      "rewards/rejected": -2.295293092727661,
+      "step": 6130
+    },
+    {
+      "epoch": 1.0578911095796002,
+      "grad_norm": 13.060774775551383,
+      "learning_rate": 2.68364782187818e-08,
+      "logits/chosen": -3.4492218494415283,
+      "logits/rejected": -3.4433789253234863,
+      "logps/chosen": -2.027282953262329,
+      "logps/rejected": -2.1917781829833984,
+      "loss": 1.1343,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -2.027282953262329,
+      "rewards/margins": 0.16449519991874695,
+      "rewards/rejected": -2.1917781829833984,
+      "step": 6140
+    },
+    {
+      "epoch": 1.0596140592694694,
+      "grad_norm": 11.287223596326001,
+      "learning_rate": 2.676149384381803e-08,
+      "logits/chosen": -3.3877010345458984,
+      "logits/rejected": -3.3791916370391846,
+      "logps/chosen": -1.996636986732483,
+      "logps/rejected": -2.155397415161133,
+      "loss": 1.1182,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.996636986732483,
+      "rewards/margins": 0.15876078605651855,
+      "rewards/rejected": -2.155397415161133,
+      "step": 6150
+    },
+    {
+      "epoch": 1.0613370089593384,
+      "grad_norm": 11.939263696551004,
+      "learning_rate": 2.66864935395298e-08,
+      "logits/chosen": -3.3609116077423096,
+      "logits/rejected": -3.3547751903533936,
+      "logps/chosen": -1.8757543563842773,
+      "logps/rejected": -2.0930557250976562,
+      "loss": 1.0804,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.8757543563842773,
+      "rewards/margins": 0.2173011600971222,
+      "rewards/rejected": -2.0930557250976562,
+      "step": 6160
+    },
+    {
+      "epoch": 1.0630599586492075,
+      "grad_norm": 12.404466075329308,
+      "learning_rate": 2.6611477984150627e-08,
+      "logits/chosen": -3.4318859577178955,
+      "logits/rejected": -3.4223244190216064,
+      "logps/chosen": -2.028075933456421,
+      "logps/rejected": -2.2366268634796143,
+      "loss": 1.0835,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -2.028075933456421,
+      "rewards/margins": 0.20855078101158142,
+      "rewards/rejected": -2.2366268634796143,
+      "step": 6170
+    },
+    {
+      "epoch": 1.0647829083390765,
+      "grad_norm": 12.272863864982352,
+      "learning_rate": 2.6536447856051963e-08,
+      "logits/chosen": -3.4452197551727295,
+      "logits/rejected": -3.4310989379882812,
+      "logps/chosen": -1.943311095237732,
+      "logps/rejected": -2.1910972595214844,
+      "loss": 1.0683,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.943311095237732,
+      "rewards/margins": 0.2477862387895584,
+      "rewards/rejected": -2.1910972595214844,
+      "step": 6180
+    },
+    {
+      "epoch": 1.0665058580289455,
+      "grad_norm": 11.003867366121552,
+      "learning_rate": 2.646140383373704e-08,
+      "logits/chosen": -3.4174857139587402,
+      "logits/rejected": -3.4036972522735596,
+      "logps/chosen": -1.9612274169921875,
+      "logps/rejected": -2.203089952468872,
+      "loss": 1.08,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.9612274169921875,
+      "rewards/margins": 0.2418626993894577,
+      "rewards/rejected": -2.203089952468872,
+      "step": 6190
+    },
+    {
+      "epoch": 1.0682288077188147,
+      "grad_norm": 11.93458609457427,
+      "learning_rate": 2.638634659583472e-08,
+      "logits/chosen": -3.360623598098755,
+      "logits/rejected": -3.3496994972229004,
+      "logps/chosen": -1.9964052438735962,
+      "logps/rejected": -2.2977466583251953,
+      "loss": 1.052,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.9964052438735962,
+      "rewards/margins": 0.30134183168411255,
+      "rewards/rejected": -2.2977466583251953,
+      "step": 6200
+    },
+    {
+      "epoch": 1.0682288077188147,
+      "eval_logits/chosen": -3.4820001125335693,
+      "eval_logits/rejected": -3.4782655239105225,
+      "eval_logps/chosen": -1.7986135482788086,
+      "eval_logps/rejected": -1.9783504009246826,
+      "eval_loss": 1.0922743082046509,
+      "eval_rewards/accuracies": 0.6229089498519897,
+      "eval_rewards/chosen": -1.7986135482788086,
+      "eval_rewards/margins": 0.17973698675632477,
+      "eval_rewards/rejected": -1.9783504009246826,
+      "eval_runtime": 155.2262,
+      "eval_samples_per_second": 27.727,
+      "eval_steps_per_second": 3.466,
+      "step": 6200
+    },
+    {
+      "epoch": 1.0699517574086836,
+      "grad_norm": 11.83296228626006,
+      "learning_rate": 2.6311276821093382e-08,
+      "logits/chosen": -3.4316000938415527,
+      "logits/rejected": -3.4160022735595703,
+      "logps/chosen": -1.9361133575439453,
+      "logps/rejected": -2.2167041301727295,
+      "loss": 1.0312,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.9361133575439453,
+      "rewards/margins": 0.28059059381484985,
+      "rewards/rejected": -2.2167041301727295,
+      "step": 6210
+    },
+    {
+      "epoch": 1.0716747070985528,
+      "grad_norm": 10.836942084925994,
+      "learning_rate": 2.62361951883748e-08,
+      "logits/chosen": -3.420494556427002,
+      "logits/rejected": -3.410055160522461,
+      "logps/chosen": -1.935442328453064,
+      "logps/rejected": -2.250401020050049,
+      "loss": 1.0328,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.935442328453064,
+      "rewards/margins": 0.31495824456214905,
+      "rewards/rejected": -2.250401020050049,
+      "step": 6220
+    },
+    {
+      "epoch": 1.0733976567884218,
+      "grad_norm": 10.186872818948487,
+      "learning_rate": 2.616110237664793e-08,
+      "logits/chosen": -3.5044498443603516,
+      "logits/rejected": -3.496494770050049,
+      "logps/chosen": -1.9641250371932983,
+      "logps/rejected": -2.2214720249176025,
+      "loss": 1.0679,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.9641250371932983,
+      "rewards/margins": 0.2573469579219818,
+      "rewards/rejected": -2.2214720249176025,
+      "step": 6230
+    },
+    {
+      "epoch": 1.0751206064782908,
+      "grad_norm": 10.947831322413764,
+      "learning_rate": 2.608599906498287e-08,
+      "logits/chosen": -3.391982316970825,
+      "logits/rejected": -3.37567400932312,
+      "logps/chosen": -1.9669920206069946,
+      "logps/rejected": -2.2538466453552246,
+      "loss": 1.0632,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.9669920206069946,
+      "rewards/margins": 0.2868550419807434,
+      "rewards/rejected": -2.2538466453552246,
+      "step": 6240
+    },
+    {
+      "epoch": 1.07684355616816,
+      "grad_norm": 11.328067837813856,
+      "learning_rate": 2.6010885932544646e-08,
+      "logits/chosen": -3.439513683319092,
+      "logits/rejected": -3.4281418323516846,
+      "logps/chosen": -1.976723074913025,
+      "logps/rejected": -2.1655750274658203,
+      "loss": 1.1105,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.976723074913025,
+      "rewards/margins": 0.18885205686092377,
+      "rewards/rejected": -2.1655750274658203,
+      "step": 6250
+    },
+    {
+      "epoch": 1.078566505858029,
+      "grad_norm": 10.736167127031386,
+      "learning_rate": 2.59357636585871e-08,
+      "logits/chosen": -3.3675715923309326,
+      "logits/rejected": -3.357095718383789,
+      "logps/chosen": -1.9657106399536133,
+      "logps/rejected": -2.1753089427948,
+      "loss": 1.0909,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.9657106399536133,
+      "rewards/margins": 0.20959842205047607,
+      "rewards/rejected": -2.1753089427948,
+      "step": 6260
+    },
+    {
+      "epoch": 1.080289455547898,
+      "grad_norm": 9.437913703567157,
+      "learning_rate": 2.5860632922446733e-08,
+      "logits/chosen": -3.5268752574920654,
+      "logits/rejected": -3.5245907306671143,
+      "logps/chosen": -1.9184255599975586,
+      "logps/rejected": -2.097491979598999,
+      "loss": 1.114,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -1.9184255599975586,
+      "rewards/margins": 0.17906661331653595,
+      "rewards/rejected": -2.097491979598999,
+      "step": 6270
+    },
+    {
+      "epoch": 1.082012405237767,
+      "grad_norm": 9.813261296550406,
+      "learning_rate": 2.578549440353659e-08,
+      "logits/chosen": -3.34009051322937,
+      "logits/rejected": -3.3253073692321777,
+      "logps/chosen": -1.8897197246551514,
+      "logps/rejected": -2.1850991249084473,
+      "loss": 1.0276,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.8897197246551514,
+      "rewards/margins": 0.29537931084632874,
+      "rewards/rejected": -2.1850991249084473,
+      "step": 6280
+    },
+    {
+      "epoch": 1.083735354927636,
+      "grad_norm": 9.802679152328315,
+      "learning_rate": 2.5710348781340068e-08,
+      "logits/chosen": -3.382852077484131,
+      "logits/rejected": -3.3664093017578125,
+      "logps/chosen": -1.8846254348754883,
+      "logps/rejected": -2.3111023902893066,
+      "loss": 0.9866,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.8846254348754883,
+      "rewards/margins": 0.4264773428440094,
+      "rewards/rejected": -2.3111023902893066,
+      "step": 6290
+    },
+    {
+      "epoch": 1.0854583046175053,
+      "grad_norm": 11.731385766456182,
+      "learning_rate": 2.5635196735404818e-08,
+      "logits/chosen": -3.4360432624816895,
+      "logits/rejected": -3.4201712608337402,
+      "logps/chosen": -2.010270118713379,
+      "logps/rejected": -2.347832679748535,
+      "loss": 1.0504,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -2.010270118713379,
+      "rewards/margins": 0.3375625014305115,
+      "rewards/rejected": -2.347832679748535,
+      "step": 6300
+    },
+    {
+      "epoch": 1.0854583046175053,
+      "eval_logits/chosen": -3.4755172729492188,
+      "eval_logits/rejected": -3.4717581272125244,
+      "eval_logps/chosen": -1.802880883216858,
+      "eval_logps/rejected": -1.9832969903945923,
+      "eval_loss": 1.092018723487854,
+      "eval_rewards/accuracies": 0.6243029832839966,
+      "eval_rewards/chosen": -1.802880883216858,
+      "eval_rewards/margins": 0.18041613698005676,
+      "eval_rewards/rejected": -1.9832969903945923,
+      "eval_runtime": 155.5472,
+      "eval_samples_per_second": 27.67,
+      "eval_steps_per_second": 3.459,
+      "step": 6300
+    },
+    {
+      "epoch": 1.0871812543073742,
+      "grad_norm": 12.917354044925556,
+      "learning_rate": 2.556003894533658e-08,
+      "logits/chosen": -3.4148082733154297,
+      "logits/rejected": -3.3976902961730957,
+      "logps/chosen": -1.875796914100647,
+      "logps/rejected": -2.2086405754089355,
+      "loss": 1.0191,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.875796914100647,
+      "rewards/margins": 0.3328438699245453,
+      "rewards/rejected": -2.2086405754089355,
+      "step": 6310
+    },
+    {
+      "epoch": 1.0889042039972432,
+      "grad_norm": 10.938111467266166,
+      "learning_rate": 2.548487609079305e-08,
+      "logits/chosen": -3.391505479812622,
+      "logits/rejected": -3.379016399383545,
+      "logps/chosen": -1.9524133205413818,
+      "logps/rejected": -2.215803861618042,
+      "loss": 1.0583,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.9524133205413818,
+      "rewards/margins": 0.26339051127433777,
+      "rewards/rejected": -2.215803861618042,
+      "step": 6320
+    },
+    {
+      "epoch": 1.0906271536871124,
+      "grad_norm": 10.793902800743574,
+      "learning_rate": 2.5409708851477683e-08,
+      "logits/chosen": -3.397482395172119,
+      "logits/rejected": -3.3870747089385986,
+      "logps/chosen": -1.9796231985092163,
+      "logps/rejected": -2.406693696975708,
+      "loss": 0.9804,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.9796231985092163,
+      "rewards/margins": 0.4270703196525574,
+      "rewards/rejected": -2.406693696975708,
+      "step": 6330
+    },
+    {
+      "epoch": 1.0923501033769814,
+      "grad_norm": 10.820158864903025,
+      "learning_rate": 2.533453790713363e-08,
+      "logits/chosen": -3.3883655071258545,
+      "logits/rejected": -3.3779120445251465,
+      "logps/chosen": -1.949059247970581,
+      "logps/rejected": -2.1930179595947266,
+      "loss": 1.0584,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.949059247970581,
+      "rewards/margins": 0.2439587563276291,
+      "rewards/rejected": -2.1930179595947266,
+      "step": 6340
+    },
+    {
+      "epoch": 1.0940730530668505,
+      "grad_norm": 10.811041750728641,
+      "learning_rate": 2.5259363937537526e-08,
+      "logits/chosen": -3.389192581176758,
+      "logits/rejected": -3.383488893508911,
+      "logps/chosen": -2.0018811225891113,
+      "logps/rejected": -2.2283692359924316,
+      "loss": 1.0916,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -2.0018811225891113,
+      "rewards/margins": 0.22648802399635315,
+      "rewards/rejected": -2.2283692359924316,
+      "step": 6350
+    },
+    {
+      "epoch": 1.0957960027567195,
+      "grad_norm": 12.005534049742856,
+      "learning_rate": 2.518418762249336e-08,
+      "logits/chosen": -3.4091544151306152,
+      "logits/rejected": -3.406395673751831,
+      "logps/chosen": -1.980224847793579,
+      "logps/rejected": -2.1962087154388428,
+      "loss": 1.0866,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.980224847793579,
+      "rewards/margins": 0.21598371863365173,
+      "rewards/rejected": -2.1962087154388428,
+      "step": 6360
+    },
+    {
+      "epoch": 1.0975189524465885,
+      "grad_norm": 11.097080798978107,
+      "learning_rate": 2.5109009641826344e-08,
+      "logits/chosen": -3.420313596725464,
+      "logits/rejected": -3.418764591217041,
+      "logps/chosen": -1.9212003946304321,
+      "logps/rejected": -2.0378031730651855,
+      "loss": 1.1501,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.9212003946304321,
+      "rewards/margins": 0.11660295724868774,
+      "rewards/rejected": -2.0378031730651855,
+      "step": 6370
+    },
+    {
+      "epoch": 1.0992419021364577,
+      "grad_norm": 13.02947212406382,
+      "learning_rate": 2.5033830675376744e-08,
+      "logits/chosen": -3.4417545795440674,
+      "logits/rejected": -3.4363327026367188,
+      "logps/chosen": -1.9934208393096924,
+      "logps/rejected": -2.2335474491119385,
+      "loss": 1.0687,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.9934208393096924,
+      "rewards/margins": 0.24012641608715057,
+      "rewards/rejected": -2.2335474491119385,
+      "step": 6380
+    },
+    {
+      "epoch": 1.1009648518263266,
+      "grad_norm": 11.335978829459279,
+      "learning_rate": 2.4958651402993735e-08,
+      "logits/chosen": -3.462235689163208,
+      "logits/rejected": -3.4436752796173096,
+      "logps/chosen": -1.923956274986267,
+      "logps/rejected": -2.2145538330078125,
+      "loss": 1.0463,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.923956274986267,
+      "rewards/margins": 0.2905975878238678,
+      "rewards/rejected": -2.2145538330078125,
+      "step": 6390
+    },
+    {
+      "epoch": 1.1026878015161956,
+      "grad_norm": 12.747207328983789,
+      "learning_rate": 2.4883472504529286e-08,
+      "logits/chosen": -3.41283917427063,
+      "logits/rejected": -3.400566577911377,
+      "logps/chosen": -1.9678151607513428,
+      "logps/rejected": -2.2396657466888428,
+      "loss": 1.0798,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.9678151607513428,
+      "rewards/margins": 0.27185067534446716,
+      "rewards/rejected": -2.2396657466888428,
+      "step": 6400
+    },
+    {
+      "epoch": 1.1026878015161956,
+      "eval_logits/chosen": -3.481964111328125,
+      "eval_logits/rejected": -3.4782474040985107,
+      "eval_logps/chosen": -1.8055204153060913,
+      "eval_logps/rejected": -1.9863353967666626,
+      "eval_loss": 1.0920108556747437,
+      "eval_rewards/accuracies": 0.624535322189331,
+      "eval_rewards/chosen": -1.8055204153060913,
+      "eval_rewards/margins": 0.1808149516582489,
+      "eval_rewards/rejected": -1.9863353967666626,
+      "eval_runtime": 155.7182,
+      "eval_samples_per_second": 27.64,
+      "eval_steps_per_second": 3.455,
+      "step": 6400
+    },
+    {
+      "epoch": 1.1044107512060648,
+      "grad_norm": 12.082872944068399,
+      "learning_rate": 2.4808294659831937e-08,
+      "logits/chosen": -3.481295347213745,
+      "logits/rejected": -3.4670042991638184,
+      "logps/chosen": -1.9438257217407227,
+      "logps/rejected": -2.2101173400878906,
+      "loss": 1.0425,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.9438257217407227,
+      "rewards/margins": 0.26629164814949036,
+      "rewards/rejected": -2.2101173400878906,
+      "step": 6410
+    },
+    {
+      "epoch": 1.1061337008959338,
+      "grad_norm": 13.299130125454987,
+      "learning_rate": 2.473311854874075e-08,
+      "logits/chosen": -3.441145420074463,
+      "logits/rejected": -3.433706760406494,
+      "logps/chosen": -1.996436357498169,
+      "logps/rejected": -2.2426958084106445,
+      "loss": 1.0883,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.996436357498169,
+      "rewards/margins": 0.24625949561595917,
+      "rewards/rejected": -2.2426958084106445,
+      "step": 6420
+    },
+    {
+      "epoch": 1.107856650585803,
+      "grad_norm": 10.304578533407732,
+      "learning_rate": 2.4657944851079078e-08,
+      "logits/chosen": -3.3995203971862793,
+      "logits/rejected": -3.3908233642578125,
+      "logps/chosen": -1.9019798040390015,
+      "logps/rejected": -2.101747989654541,
+      "loss": 1.119,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.9019798040390015,
+      "rewards/margins": 0.1997680962085724,
+      "rewards/rejected": -2.101747989654541,
+      "step": 6430
+    },
+    {
+      "epoch": 1.109579600275672,
+      "grad_norm": 11.828400045231056,
+      "learning_rate": 2.4582774246648447e-08,
+      "logits/chosen": -3.3840713500976562,
+      "logits/rejected": -3.3737869262695312,
+      "logps/chosen": -2.034101963043213,
+      "logps/rejected": -2.2422122955322266,
+      "loss": 1.1028,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -2.034101963043213,
+      "rewards/margins": 0.20811061561107635,
+      "rewards/rejected": -2.2422122955322266,
+      "step": 6440
+    },
+    {
+      "epoch": 1.111302549965541,
+      "grad_norm": 9.935999733788844,
+      "learning_rate": 2.4507607415222437e-08,
+      "logits/chosen": -3.393545150756836,
+      "logits/rejected": -3.375601291656494,
+      "logps/chosen": -1.9729492664337158,
+      "logps/rejected": -2.259352922439575,
+      "loss": 1.0465,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.9729492664337158,
+      "rewards/margins": 0.28640368580818176,
+      "rewards/rejected": -2.259352922439575,
+      "step": 6450
+    },
+    {
+      "epoch": 1.11302549965541,
+      "grad_norm": 13.063403404263378,
+      "learning_rate": 2.443244503654047e-08,
+      "logits/chosen": -3.392033100128174,
+      "logits/rejected": -3.392728328704834,
+      "logps/chosen": -1.9366604089736938,
+      "logps/rejected": -2.1797962188720703,
+      "loss": 1.0602,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.9366604089736938,
+      "rewards/margins": 0.24313604831695557,
+      "rewards/rejected": -2.1797962188720703,
+      "step": 6460
+    },
+    {
+      "epoch": 1.114748449345279,
+      "grad_norm": 11.74429446491521,
+      "learning_rate": 2.4357287790301757e-08,
+      "logits/chosen": -3.358668804168701,
+      "logits/rejected": -3.3473758697509766,
+      "logps/chosen": -1.9496071338653564,
+      "logps/rejected": -2.252514362335205,
+      "loss": 1.04,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.9496071338653564,
+      "rewards/margins": 0.3029070794582367,
+      "rewards/rejected": -2.252514362335205,
+      "step": 6470
+    },
+    {
+      "epoch": 1.1164713990351482,
+      "grad_norm": 11.953708997053024,
+      "learning_rate": 2.4282136356159026e-08,
+      "logits/chosen": -3.4172749519348145,
+      "logits/rejected": -3.3967297077178955,
+      "logps/chosen": -1.9061167240142822,
+      "logps/rejected": -2.2003939151763916,
+      "loss": 1.0646,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.9061167240142822,
+      "rewards/margins": 0.2942771911621094,
+      "rewards/rejected": -2.2003939151763916,
+      "step": 6480
+    },
+    {
+      "epoch": 1.1181943487250172,
+      "grad_norm": 10.96149412313629,
+      "learning_rate": 2.4206991413712514e-08,
+      "logits/chosen": -3.515418291091919,
+      "logits/rejected": -3.507902145385742,
+      "logps/chosen": -1.9523273706436157,
+      "logps/rejected": -2.181636095046997,
+      "loss": 1.0715,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.9523273706436157,
+      "rewards/margins": 0.22930900752544403,
+      "rewards/rejected": -2.181636095046997,
+      "step": 6490
+    },
+    {
+      "epoch": 1.1199172984148862,
+      "grad_norm": 11.791218630496866,
+      "learning_rate": 2.4131853642503697e-08,
+      "logits/chosen": -3.413747787475586,
+      "logits/rejected": -3.4076812267303467,
+      "logps/chosen": -2.0606017112731934,
+      "logps/rejected": -2.157837152481079,
+      "loss": 1.1707,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -2.0606017112731934,
+      "rewards/margins": 0.09723515808582306,
+      "rewards/rejected": -2.157837152481079,
+      "step": 6500
+    },
+    {
+      "epoch": 1.1199172984148862,
+      "eval_logits/chosen": -3.473191261291504,
+      "eval_logits/rejected": -3.4694576263427734,
+      "eval_logps/chosen": -1.8115531206130981,
+      "eval_logps/rejected": -1.9931327104568481,
+      "eval_loss": 1.0917991399765015,
+      "eval_rewards/accuracies": 0.625,
+      "eval_rewards/chosen": -1.8115531206130981,
+      "eval_rewards/margins": 0.18157950043678284,
+      "eval_rewards/rejected": -1.9931327104568481,
+      "eval_runtime": 155.4427,
+      "eval_samples_per_second": 27.689,
+      "eval_steps_per_second": 3.461,
+      "step": 6500
+    },
+    {
+      "epoch": 1.1216402481047554,
+      "grad_norm": 12.224571366681836,
+      "learning_rate": 2.4056723722009246e-08,
+      "logits/chosen": -3.438197612762451,
+      "logits/rejected": -3.41357684135437,
+      "logps/chosen": -1.918994665145874,
+      "logps/rejected": -2.245304584503174,
+      "loss": 1.0098,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.918994665145874,
+      "rewards/margins": 0.3263099789619446,
+      "rewards/rejected": -2.245304584503174,
+      "step": 6510
+    },
+    {
+      "epoch": 1.1233631977946243,
+      "grad_norm": 11.457126192920224,
+      "learning_rate": 2.3981602331634804e-08,
+      "logits/chosen": -3.393517017364502,
+      "logits/rejected": -3.3795249462127686,
+      "logps/chosen": -1.889651894569397,
+      "logps/rejected": -2.203306198120117,
+      "loss": 1.022,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.889651894569397,
+      "rewards/margins": 0.31365424394607544,
+      "rewards/rejected": -2.203306198120117,
+      "step": 6520
+    },
+    {
+      "epoch": 1.1250861474844935,
+      "grad_norm": 10.439194240922614,
+      "learning_rate": 2.3906490150708893e-08,
+      "logits/chosen": -3.3862648010253906,
+      "logits/rejected": -3.370769500732422,
+      "logps/chosen": -1.9234901666641235,
+      "logps/rejected": -2.2089579105377197,
+      "loss": 1.0419,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.9234901666641235,
+      "rewards/margins": 0.28546762466430664,
+      "rewards/rejected": -2.2089579105377197,
+      "step": 6530
+    },
+    {
+      "epoch": 1.1268090971743625,
+      "grad_norm": 12.036500179917471,
+      "learning_rate": 2.383138785847674e-08,
+      "logits/chosen": -3.423931837081909,
+      "logits/rejected": -3.4105498790740967,
+      "logps/chosen": -1.953293800354004,
+      "logps/rejected": -2.1603314876556396,
+      "loss": 1.0966,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.953293800354004,
+      "rewards/margins": 0.20703773200511932,
+      "rewards/rejected": -2.1603314876556396,
+      "step": 6540
+    },
+    {
+      "epoch": 1.1285320468642315,
+      "grad_norm": 12.44923538612757,
+      "learning_rate": 2.3756296134094176e-08,
+      "logits/chosen": -3.349327802658081,
+      "logits/rejected": -3.338196277618408,
+      "logps/chosen": -2.0173699855804443,
+      "logps/rejected": -2.3396806716918945,
+      "loss": 1.0346,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -2.0173699855804443,
+      "rewards/margins": 0.3223108649253845,
+      "rewards/rejected": -2.3396806716918945,
+      "step": 6550
+    },
+    {
+      "epoch": 1.1302549965541007,
+      "grad_norm": 10.655314010193836,
+      "learning_rate": 2.368121565662142e-08,
+      "logits/chosen": -3.4336647987365723,
+      "logits/rejected": -3.4113261699676514,
+      "logps/chosen": -1.9179922342300415,
+      "logps/rejected": -2.28322172164917,
+      "loss": 1.0063,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.9179922342300415,
+      "rewards/margins": 0.3652295470237732,
+      "rewards/rejected": -2.28322172164917,
+      "step": 6560
+    },
+    {
+      "epoch": 1.1319779462439696,
+      "grad_norm": 12.738528865603543,
+      "learning_rate": 2.3606147105017038e-08,
+      "logits/chosen": -3.4219493865966797,
+      "logits/rejected": -3.406679153442383,
+      "logps/chosen": -1.9982473850250244,
+      "logps/rejected": -2.3211865425109863,
+      "loss": 1.0231,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.9982473850250244,
+      "rewards/margins": 0.3229389488697052,
+      "rewards/rejected": -2.3211865425109863,
+      "step": 6570
+    },
+    {
+      "epoch": 1.1337008959338388,
+      "grad_norm": 9.404832853443324,
+      "learning_rate": 2.35310911581317e-08,
+      "logits/chosen": -3.429957151412964,
+      "logits/rejected": -3.411017656326294,
+      "logps/chosen": -1.90872323513031,
+      "logps/rejected": -2.178488254547119,
+      "loss": 1.071,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.90872323513031,
+      "rewards/margins": 0.26976481080055237,
+      "rewards/rejected": -2.178488254547119,
+      "step": 6580
+    },
+    {
+      "epoch": 1.1354238456237078,
+      "grad_norm": 9.975406580918412,
+      "learning_rate": 2.3456048494702132e-08,
+      "logits/chosen": -3.3972315788269043,
+      "logits/rejected": -3.385753631591797,
+      "logps/chosen": -1.9451984167099,
+      "logps/rejected": -2.2248411178588867,
+      "loss": 1.0422,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.9451984167099,
+      "rewards/margins": 0.2796427309513092,
+      "rewards/rejected": -2.2248411178588867,
+      "step": 6590
+    },
+    {
+      "epoch": 1.1371467953135768,
+      "grad_norm": 14.681109543394694,
+      "learning_rate": 2.3381019793344898e-08,
+      "logits/chosen": -3.456721782684326,
+      "logits/rejected": -3.447939395904541,
+      "logps/chosen": -2.0228888988494873,
+      "logps/rejected": -2.1658008098602295,
+      "loss": 1.1428,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -2.0228888988494873,
+      "rewards/margins": 0.142911896109581,
+      "rewards/rejected": -2.1658008098602295,
+      "step": 6600
+    },
+    {
+      "epoch": 1.1371467953135768,
+      "eval_logits/chosen": -3.464677095413208,
+      "eval_logits/rejected": -3.460919141769409,
+      "eval_logps/chosen": -1.8144869804382324,
+      "eval_logps/rejected": -1.9965152740478516,
+      "eval_loss": 1.0917773246765137,
+      "eval_rewards/accuracies": 0.6247676610946655,
+      "eval_rewards/chosen": -1.8144869804382324,
+      "eval_rewards/margins": 0.18202824890613556,
+      "eval_rewards/rejected": -1.9965152740478516,
+      "eval_runtime": 155.4677,
+      "eval_samples_per_second": 27.684,
+      "eval_steps_per_second": 3.461,
+      "step": 6600
+    },
+    {
+      "epoch": 1.138869745003446,
+      "grad_norm": 10.970876194506403,
+      "learning_rate": 2.330600573255034e-08,
+      "logits/chosen": -3.4280261993408203,
+      "logits/rejected": -3.4172167778015137,
+      "logps/chosen": -1.934708833694458,
+      "logps/rejected": -2.2180302143096924,
+      "loss": 1.0329,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.934708833694458,
+      "rewards/margins": 0.28332141041755676,
+      "rewards/rejected": -2.2180302143096924,
+      "step": 6610
+    },
+    {
+      "epoch": 1.140592694693315,
+      "grad_norm": 13.049800922984984,
+      "learning_rate": 2.3231006990676365e-08,
+      "logits/chosen": -3.4193642139434814,
+      "logits/rejected": -3.408034563064575,
+      "logps/chosen": -2.069882392883301,
+      "logps/rejected": -2.3080859184265137,
+      "loss": 1.0801,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -2.069882392883301,
+      "rewards/margins": 0.23820337653160095,
+      "rewards/rejected": -2.3080859184265137,
+      "step": 6620
+    },
+    {
+      "epoch": 1.1423156443831841,
+      "grad_norm": 10.686636246054626,
+      "learning_rate": 2.3156024245942392e-08,
+      "logits/chosen": -3.3981239795684814,
+      "logits/rejected": -3.3838086128234863,
+      "logps/chosen": -1.9325320720672607,
+      "logps/rejected": -2.135441541671753,
+      "loss": 1.0866,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.9325320720672607,
+      "rewards/margins": 0.20290973782539368,
+      "rewards/rejected": -2.135441541671753,
+      "step": 6630
+    },
+    {
+      "epoch": 1.144038594073053,
+      "grad_norm": 11.24936064685848,
+      "learning_rate": 2.3081058176423148e-08,
+      "logits/chosen": -3.429295063018799,
+      "logits/rejected": -3.4095797538757324,
+      "logps/chosen": -1.9664671421051025,
+      "logps/rejected": -2.356595516204834,
+      "loss": 0.9903,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.9664671421051025,
+      "rewards/margins": 0.3901287317276001,
+      "rewards/rejected": -2.356595516204834,
+      "step": 6640
+    },
+    {
+      "epoch": 1.145761543762922,
+      "grad_norm": 12.716329690658595,
+      "learning_rate": 2.3006109460042562e-08,
+      "logits/chosen": -3.4468204975128174,
+      "logits/rejected": -3.436032772064209,
+      "logps/chosen": -1.9885509014129639,
+      "logps/rejected": -2.316497325897217,
+      "loss": 1.0341,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.9885509014129639,
+      "rewards/margins": 0.32794636487960815,
+      "rewards/rejected": -2.316497325897217,
+      "step": 6650
+    },
+    {
+      "epoch": 1.1474844934527912,
+      "grad_norm": 11.039322495934172,
+      "learning_rate": 2.293117877456766e-08,
+      "logits/chosen": -3.4637978076934814,
+      "logits/rejected": -3.4526405334472656,
+      "logps/chosen": -1.876461386680603,
+      "logps/rejected": -2.223781108856201,
+      "loss": 1.0038,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.876461386680603,
+      "rewards/margins": 0.34731966257095337,
+      "rewards/rejected": -2.223781108856201,
+      "step": 6660
+    },
+    {
+      "epoch": 1.1492074431426602,
+      "grad_norm": 11.476260726500982,
+      "learning_rate": 2.2856266797602393e-08,
+      "logits/chosen": -3.4084198474884033,
+      "logits/rejected": -3.408863067626953,
+      "logps/chosen": -1.9839646816253662,
+      "logps/rejected": -2.2805380821228027,
+      "loss": 1.0317,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.9839646816253662,
+      "rewards/margins": 0.2965734601020813,
+      "rewards/rejected": -2.2805380821228027,
+      "step": 6670
+    },
+    {
+      "epoch": 1.1509303928325294,
+      "grad_norm": 11.688610575316675,
+      "learning_rate": 2.2781374206581543e-08,
+      "logits/chosen": -3.4148972034454346,
+      "logits/rejected": -3.3965370655059814,
+      "logps/chosen": -1.9790725708007812,
+      "logps/rejected": -2.350860118865967,
+      "loss": 1.0081,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.9790725708007812,
+      "rewards/margins": 0.37178757786750793,
+      "rewards/rejected": -2.350860118865967,
+      "step": 6680
+    },
+    {
+      "epoch": 1.1526533425223984,
+      "grad_norm": 15.20291620280058,
+      "learning_rate": 2.2706501678764558e-08,
+      "logits/chosen": -3.3994460105895996,
+      "logits/rejected": -3.3877291679382324,
+      "logps/chosen": -2.039080858230591,
+      "logps/rejected": -2.3276233673095703,
+      "loss": 1.0556,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -2.039080858230591,
+      "rewards/margins": 0.2885424494743347,
+      "rewards/rejected": -2.3276233673095703,
+      "step": 6690
+    },
+    {
+      "epoch": 1.1543762922122673,
+      "grad_norm": 12.76941769914266,
+      "learning_rate": 2.26316498912295e-08,
+      "logits/chosen": -3.4142813682556152,
+      "logits/rejected": -3.4071478843688965,
+      "logps/chosen": -1.9860906600952148,
+      "logps/rejected": -2.2994446754455566,
+      "loss": 1.0715,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.9860906600952148,
+      "rewards/margins": 0.3133540749549866,
+      "rewards/rejected": -2.2994446754455566,
+      "step": 6700
+    },
+    {
+      "epoch": 1.1543762922122673,
+      "eval_logits/chosen": -3.49177885055542,
+      "eval_logits/rejected": -3.4881672859191895,
+      "eval_logps/chosen": -1.815558910369873,
+      "eval_logps/rejected": -1.9987828731536865,
+      "eval_loss": 1.091330885887146,
+      "eval_rewards/accuracies": 0.6259293556213379,
+      "eval_rewards/chosen": -1.815558910369873,
+      "eval_rewards/margins": 0.18322399258613586,
+      "eval_rewards/rejected": -1.9987828731536865,
+      "eval_runtime": 155.4771,
+      "eval_samples_per_second": 27.683,
+      "eval_steps_per_second": 3.46,
+      "step": 6700
+    },
+    {
+      "epoch": 1.1560992419021365,
+      "grad_norm": 11.139093480891768,
+      "learning_rate": 2.2556819520866827e-08,
+      "logits/chosen": -3.409257411956787,
+      "logits/rejected": -3.3946621417999268,
+      "logps/chosen": -1.952646017074585,
+      "logps/rejected": -2.243669033050537,
+      "loss": 1.0643,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.952646017074585,
+      "rewards/margins": 0.2910230755805969,
+      "rewards/rejected": -2.243669033050537,
+      "step": 6710
+    },
+    {
+      "epoch": 1.1578221915920055,
+      "grad_norm": 12.172769197885591,
+      "learning_rate": 2.2482011244373356e-08,
+      "logits/chosen": -3.404743194580078,
+      "logits/rejected": -3.4008660316467285,
+      "logps/chosen": -1.9026057720184326,
+      "logps/rejected": -2.1675379276275635,
+      "loss": 1.0603,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.9026057720184326,
+      "rewards/margins": 0.2649320960044861,
+      "rewards/rejected": -2.1675379276275635,
+      "step": 6720
+    },
+    {
+      "epoch": 1.1595451412818747,
+      "grad_norm": 11.423985458290572,
+      "learning_rate": 2.2407225738246073e-08,
+      "logits/chosen": -3.3756790161132812,
+      "logits/rejected": -3.3669891357421875,
+      "logps/chosen": -1.9867572784423828,
+      "logps/rejected": -2.191256284713745,
+      "loss": 1.108,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.9867572784423828,
+      "rewards/margins": 0.20449912548065186,
+      "rewards/rejected": -2.191256284713745,
+      "step": 6730
+    },
+    {
+      "epoch": 1.1612680909717437,
+      "grad_norm": 12.34336598235776,
+      "learning_rate": 2.233246367877609e-08,
+      "logits/chosen": -3.4295153617858887,
+      "logits/rejected": -3.4251716136932373,
+      "logps/chosen": -1.990858793258667,
+      "logps/rejected": -2.192908763885498,
+      "loss": 1.105,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.990858793258667,
+      "rewards/margins": 0.20204992592334747,
+      "rewards/rejected": -2.192908763885498,
+      "step": 6740
+    },
+    {
+      "epoch": 1.1629910406616126,
+      "grad_norm": 11.719878943083298,
+      "learning_rate": 2.2257725742042437e-08,
+      "logits/chosen": -3.4420971870422363,
+      "logits/rejected": -3.4348385334014893,
+      "logps/chosen": -2.011589527130127,
+      "logps/rejected": -2.305131673812866,
+      "loss": 1.0535,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -2.011589527130127,
+      "rewards/margins": 0.29354196786880493,
+      "rewards/rejected": -2.305131673812866,
+      "step": 6750
+    },
+    {
+      "epoch": 1.1647139903514818,
+      "grad_norm": 13.306472256820598,
+      "learning_rate": 2.2183012603906064e-08,
+      "logits/chosen": -3.4041709899902344,
+      "logits/rejected": -3.389939069747925,
+      "logps/chosen": -1.9089692831039429,
+      "logps/rejected": -2.132821798324585,
+      "loss": 1.0857,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.9089692831039429,
+      "rewards/margins": 0.22385239601135254,
+      "rewards/rejected": -2.132821798324585,
+      "step": 6760
+    },
+    {
+      "epoch": 1.1664369400413508,
+      "grad_norm": 12.877230233245287,
+      "learning_rate": 2.2108324940003607e-08,
+      "logits/chosen": -3.416132688522339,
+      "logits/rejected": -3.4125523567199707,
+      "logps/chosen": -1.958125352859497,
+      "logps/rejected": -2.2240960597991943,
+      "loss": 1.0624,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.958125352859497,
+      "rewards/margins": 0.2659708261489868,
+      "rewards/rejected": -2.2240960597991943,
+      "step": 6770
+    },
+    {
+      "epoch": 1.1681598897312198,
+      "grad_norm": 13.241165596493136,
+      "learning_rate": 2.2033663425741377e-08,
+      "logits/chosen": -3.417736768722534,
+      "logits/rejected": -3.402555465698242,
+      "logps/chosen": -2.0529732704162598,
+      "logps/rejected": -2.249335289001465,
+      "loss": 1.1083,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -2.0529732704162598,
+      "rewards/margins": 0.19636209309101105,
+      "rewards/rejected": -2.249335289001465,
+      "step": 6780
+    },
+    {
+      "epoch": 1.169882839421089,
+      "grad_norm": 11.062007037777198,
+      "learning_rate": 2.1959028736289184e-08,
+      "logits/chosen": -3.4131131172180176,
+      "logits/rejected": -3.404632568359375,
+      "logps/chosen": -2.0231058597564697,
+      "logps/rejected": -2.3660521507263184,
+      "loss": 1.0256,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -2.0231058597564697,
+      "rewards/margins": 0.3429463505744934,
+      "rewards/rejected": -2.3660521507263184,
+      "step": 6790
+    },
+    {
+      "epoch": 1.171605789110958,
+      "grad_norm": 10.010218194316696,
+      "learning_rate": 2.1884421546574288e-08,
+      "logits/chosen": -3.3731789588928223,
+      "logits/rejected": -3.358083724975586,
+      "logps/chosen": -1.9506065845489502,
+      "logps/rejected": -2.232652425765991,
+      "loss": 1.0501,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.9506065845489502,
+      "rewards/margins": 0.2820459008216858,
+      "rewards/rejected": -2.232652425765991,
+      "step": 6800
+    },
+    {
+      "epoch": 1.171605789110958,
+      "eval_logits/chosen": -3.4778852462768555,
+      "eval_logits/rejected": -3.474215030670166,
+      "eval_logps/chosen": -1.8231767416000366,
+      "eval_logps/rejected": -2.0069351196289062,
+      "eval_loss": 1.0910567045211792,
+      "eval_rewards/accuracies": 0.6231412887573242,
+      "eval_rewards/chosen": -1.8231767416000366,
+      "eval_rewards/margins": 0.18375851213932037,
+      "eval_rewards/rejected": -2.0069351196289062,
+      "eval_runtime": 155.6732,
+      "eval_samples_per_second": 27.648,
+      "eval_steps_per_second": 3.456,
+      "step": 6800
+    },
+    {
+      "epoch": 1.173328738800827,
+      "grad_norm": 11.48968985379708,
+      "learning_rate": 2.180984253127523e-08,
+      "logits/chosen": -3.4102675914764404,
+      "logits/rejected": -3.4000308513641357,
+      "logps/chosen": -1.9845850467681885,
+      "logps/rejected": -2.1544082164764404,
+      "loss": 1.1147,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.9845850467681885,
+      "rewards/margins": 0.16982324421405792,
+      "rewards/rejected": -2.1544082164764404,
+      "step": 6810
+    },
+    {
+      "epoch": 1.175051688490696,
+      "grad_norm": 10.633313540056854,
+      "learning_rate": 2.173529236481581e-08,
+      "logits/chosen": -3.4592087268829346,
+      "logits/rejected": -3.4428722858428955,
+      "logps/chosen": -1.9789336919784546,
+      "logps/rejected": -2.346738338470459,
+      "loss": 1.0007,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.9789336919784546,
+      "rewards/margins": 0.3678043484687805,
+      "rewards/rejected": -2.346738338470459,
+      "step": 6820
+    },
+    {
+      "epoch": 1.176774638180565,
+      "grad_norm": 14.085108233133877,
+      "learning_rate": 2.1660771721358898e-08,
+      "logits/chosen": -3.458054304122925,
+      "logits/rejected": -3.452343702316284,
+      "logps/chosen": -2.0140271186828613,
+      "logps/rejected": -2.1826484203338623,
+      "loss": 1.1258,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -2.0140271186828613,
+      "rewards/margins": 0.16862158477306366,
+      "rewards/rejected": -2.1826484203338623,
+      "step": 6830
+    },
+    {
+      "epoch": 1.1784975878704342,
+      "grad_norm": 10.897652428122633,
+      "learning_rate": 2.1586281274800433e-08,
+      "logits/chosen": -3.445998430252075,
+      "logits/rejected": -3.4331583976745605,
+      "logps/chosen": -1.9494144916534424,
+      "logps/rejected": -2.191014528274536,
+      "loss": 1.0612,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.9494144916534424,
+      "rewards/margins": 0.24160000681877136,
+      "rewards/rejected": -2.191014528274536,
+      "step": 6840
+    },
+    {
+      "epoch": 1.1802205375603032,
+      "grad_norm": 11.793856914803671,
+      "learning_rate": 2.1511821698763248e-08,
+      "logits/chosen": -3.345694065093994,
+      "logits/rejected": -3.3309803009033203,
+      "logps/chosen": -1.9741350412368774,
+      "logps/rejected": -2.331859827041626,
+      "loss": 1.0066,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.9741350412368774,
+      "rewards/margins": 0.35772472620010376,
+      "rewards/rejected": -2.331859827041626,
+      "step": 6850
+    },
+    {
+      "epoch": 1.1819434872501722,
+      "grad_norm": 11.997001839059115,
+      "learning_rate": 2.143739366659102e-08,
+      "logits/chosen": -3.4134151935577393,
+      "logits/rejected": -3.3935799598693848,
+      "logps/chosen": -1.9874414205551147,
+      "logps/rejected": -2.2140369415283203,
+      "loss": 1.0769,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.9874414205551147,
+      "rewards/margins": 0.2265954464673996,
+      "rewards/rejected": -2.2140369415283203,
+      "step": 6860
+    },
+    {
+      "epoch": 1.1836664369400414,
+      "grad_norm": 12.447615169553789,
+      "learning_rate": 2.1362997851342184e-08,
+      "logits/chosen": -3.3783531188964844,
+      "logits/rejected": -3.3724663257598877,
+      "logps/chosen": -2.0840086936950684,
+      "logps/rejected": -2.267148971557617,
+      "loss": 1.1232,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -2.0840086936950684,
+      "rewards/margins": 0.18314030766487122,
+      "rewards/rejected": -2.267148971557617,
+      "step": 6870
+    },
+    {
+      "epoch": 1.1853893866299103,
+      "grad_norm": 11.714270042995526,
+      "learning_rate": 2.1288634925783816e-08,
+      "logits/chosen": -3.444370985031128,
+      "logits/rejected": -3.427804946899414,
+      "logps/chosen": -1.9677927494049072,
+      "logps/rejected": -2.25700044631958,
+      "loss": 1.0582,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.9677927494049072,
+      "rewards/margins": 0.28920769691467285,
+      "rewards/rejected": -2.25700044631958,
+      "step": 6880
+    },
+    {
+      "epoch": 1.1871123363197795,
+      "grad_norm": 11.122620830899038,
+      "learning_rate": 2.1214305562385588e-08,
+      "logits/chosen": -3.386046886444092,
+      "logits/rejected": -3.3719165325164795,
+      "logps/chosen": -1.8777778148651123,
+      "logps/rejected": -2.2574100494384766,
+      "loss": 1.0077,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.8777778148651123,
+      "rewards/margins": 0.37963250279426575,
+      "rewards/rejected": -2.2574100494384766,
+      "step": 6890
+    },
+    {
+      "epoch": 1.1888352860096485,
+      "grad_norm": 12.703088216309423,
+      "learning_rate": 2.1140010433313643e-08,
+      "logits/chosen": -3.4314465522766113,
+      "logits/rejected": -3.420292377471924,
+      "logps/chosen": -1.9657869338989258,
+      "logps/rejected": -2.228525400161743,
+      "loss": 1.0595,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.9657869338989258,
+      "rewards/margins": 0.2627386152744293,
+      "rewards/rejected": -2.228525400161743,
+      "step": 6900
+    },
+    {
+      "epoch": 1.1888352860096485,
+      "eval_logits/chosen": -3.4640936851501465,
+      "eval_logits/rejected": -3.4603748321533203,
+      "eval_logps/chosen": -1.8266420364379883,
+      "eval_logps/rejected": -2.0106890201568604,
+      "eval_loss": 1.0910621881484985,
+      "eval_rewards/accuracies": 0.6252323389053345,
+      "eval_rewards/chosen": -1.8266420364379883,
+      "eval_rewards/margins": 0.18404707312583923,
+      "eval_rewards/rejected": -2.0106890201568604,
+      "eval_runtime": 155.453,
+      "eval_samples_per_second": 27.687,
+      "eval_steps_per_second": 3.461,
+      "step": 6900
+    },
+    {
+      "epoch": 1.1905582356995175,
+      "grad_norm": 14.824716678126752,
+      "learning_rate": 2.106575021042457e-08,
+      "logits/chosen": -3.4385673999786377,
+      "logits/rejected": -3.427150249481201,
+      "logps/chosen": -1.9893693923950195,
+      "logps/rejected": -2.291886329650879,
+      "loss": 1.0348,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.9893693923950195,
+      "rewards/margins": 0.3025168776512146,
+      "rewards/rejected": -2.291886329650879,
+      "step": 6910
+    },
+    {
+      "epoch": 1.1922811853893867,
+      "grad_norm": 16.57704822796873,
+      "learning_rate": 2.099152556525926e-08,
+      "logits/chosen": -3.475630283355713,
+      "logits/rejected": -3.4601573944091797,
+      "logps/chosen": -2.007235288619995,
+      "logps/rejected": -2.2298707962036133,
+      "loss": 1.0899,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -2.007235288619995,
+      "rewards/margins": 0.22263555228710175,
+      "rewards/rejected": -2.2298707962036133,
+      "step": 6920
+    },
+    {
+      "epoch": 1.1940041350792556,
+      "grad_norm": 12.72243612974444,
+      "learning_rate": 2.0917337169036925e-08,
+      "logits/chosen": -3.368088483810425,
+      "logits/rejected": -3.356076717376709,
+      "logps/chosen": -1.935720443725586,
+      "logps/rejected": -2.2330641746520996,
+      "loss": 1.043,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.935720443725586,
+      "rewards/margins": 0.29734379053115845,
+      "rewards/rejected": -2.2330641746520996,
+      "step": 6930
+    },
+    {
+      "epoch": 1.1957270847691248,
+      "grad_norm": 13.069383124489308,
+      "learning_rate": 2.0843185692648913e-08,
+      "logits/chosen": -3.355874538421631,
+      "logits/rejected": -3.3308615684509277,
+      "logps/chosen": -2.0228724479675293,
+      "logps/rejected": -2.35180401802063,
+      "loss": 1.0236,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -2.0228724479675293,
+      "rewards/margins": 0.3289314806461334,
+      "rewards/rejected": -2.35180401802063,
+      "step": 6940
+    },
+    {
+      "epoch": 1.1974500344589938,
+      "grad_norm": 10.896475014842688,
+      "learning_rate": 2.076907180665276e-08,
+      "logits/chosen": -3.4026687145233154,
+      "logits/rejected": -3.393219470977783,
+      "logps/chosen": -1.9413831233978271,
+      "logps/rejected": -2.2236905097961426,
+      "loss": 1.0613,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.9413831233978271,
+      "rewards/margins": 0.2823072075843811,
+      "rewards/rejected": -2.2236905097961426,
+      "step": 6950
+    },
+    {
+      "epoch": 1.1991729841488628,
+      "grad_norm": 13.635288013051321,
+      "learning_rate": 2.0694996181266027e-08,
+      "logits/chosen": -3.4929661750793457,
+      "logits/rejected": -3.4715209007263184,
+      "logps/chosen": -2.075517416000366,
+      "logps/rejected": -2.318410873413086,
+      "loss": 1.0762,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -2.075517416000366,
+      "rewards/margins": 0.24289333820343018,
+      "rewards/rejected": -2.318410873413086,
+      "step": 6960
+    },
+    {
+      "epoch": 1.200895933838732,
+      "grad_norm": 10.716692287890377,
+      "learning_rate": 2.0620959486360313e-08,
+      "logits/chosen": -3.4451401233673096,
+      "logits/rejected": -3.427570343017578,
+      "logps/chosen": -1.9249626398086548,
+      "logps/rejected": -2.1775269508361816,
+      "loss": 1.0667,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.9249626398086548,
+      "rewards/margins": 0.2525642514228821,
+      "rewards/rejected": -2.1775269508361816,
+      "step": 6970
+    },
+    {
+      "epoch": 1.202618883528601,
+      "grad_norm": 10.660797995432288,
+      "learning_rate": 2.0546962391455128e-08,
+      "logits/chosen": -3.3799118995666504,
+      "logits/rejected": -3.3648600578308105,
+      "logps/chosen": -1.9633506536483765,
+      "logps/rejected": -2.2720298767089844,
+      "loss": 1.0282,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.9633506536483765,
+      "rewards/margins": 0.30867907404899597,
+      "rewards/rejected": -2.2720298767089844,
+      "step": 6980
+    },
+    {
+      "epoch": 1.20434183321847,
+      "grad_norm": 11.09958765202879,
+      "learning_rate": 2.0473005565711924e-08,
+      "logits/chosen": -3.3331313133239746,
+      "logits/rejected": -3.327349901199341,
+      "logps/chosen": -2.082742691040039,
+      "logps/rejected": -2.2335104942321777,
+      "loss": 1.1303,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -2.082742691040039,
+      "rewards/margins": 0.15076801180839539,
+      "rewards/rejected": -2.2335104942321777,
+      "step": 6990
+    },
+    {
+      "epoch": 1.206064782908339,
+      "grad_norm": 13.47233496346886,
+      "learning_rate": 2.039908967792795e-08,
+      "logits/chosen": -3.514160633087158,
+      "logits/rejected": -3.5003600120544434,
+      "logps/chosen": -2.091055154800415,
+      "logps/rejected": -2.393519163131714,
+      "loss": 1.0657,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -2.091055154800415,
+      "rewards/margins": 0.3024642765522003,
+      "rewards/rejected": -2.393519163131714,
+      "step": 7000
+    },
+    {
+      "epoch": 1.206064782908339,
+      "eval_logits/chosen": -3.471799373626709,
+      "eval_logits/rejected": -3.4681344032287598,
+      "eval_logps/chosen": -1.8323529958724976,
+      "eval_logps/rejected": -2.017329454421997,
+      "eval_loss": 1.0906610488891602,
+      "eval_rewards/accuracies": 0.6243029832839966,
+      "eval_rewards/chosen": -1.8323529958724976,
+      "eval_rewards/margins": 0.1849764883518219,
+      "eval_rewards/rejected": -2.017329454421997,
+      "eval_runtime": 155.744,
+      "eval_samples_per_second": 27.635,
+      "eval_steps_per_second": 3.454,
+      "step": 7000
+    },
+    {
+      "epoch": 1.207787732598208,
+      "grad_norm": 11.555406271101281,
+      "learning_rate": 2.0325215396530286e-08,
+      "logits/chosen": -3.402296543121338,
+      "logits/rejected": -3.387709140777588,
+      "logps/chosen": -2.0441126823425293,
+      "logps/rejected": -2.2946605682373047,
+      "loss": 1.0773,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -2.0441126823425293,
+      "rewards/margins": 0.25054794549942017,
+      "rewards/rejected": -2.2946605682373047,
+      "step": 7010
+    },
+    {
+      "epoch": 1.2095106822880772,
+      "grad_norm": 10.652216783564976,
+      "learning_rate": 2.025138338956974e-08,
+      "logits/chosen": -3.404336929321289,
+      "logits/rejected": -3.3904311656951904,
+      "logps/chosen": -1.9957082271575928,
+      "logps/rejected": -2.36806321144104,
+      "loss": 1.0204,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.9957082271575928,
+      "rewards/margins": 0.37235480546951294,
+      "rewards/rejected": -2.36806321144104,
+      "step": 7020
+    },
+    {
+      "epoch": 1.2112336319779462,
+      "grad_norm": 10.48417948750337,
+      "learning_rate": 2.0177594324714838e-08,
+      "logits/chosen": -3.4127960205078125,
+      "logits/rejected": -3.4057040214538574,
+      "logps/chosen": -1.9837955236434937,
+      "logps/rejected": -2.2497267723083496,
+      "loss": 1.0614,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.9837955236434937,
+      "rewards/margins": 0.2659311890602112,
+      "rewards/rejected": -2.2497267723083496,
+      "step": 7030
+    },
+    {
+      "epoch": 1.2129565816678154,
+      "grad_norm": 11.083591667847761,
+      "learning_rate": 2.0103848869245765e-08,
+      "logits/chosen": -3.3733811378479004,
+      "logits/rejected": -3.362372636795044,
+      "logps/chosen": -1.9914352893829346,
+      "logps/rejected": -2.281447172164917,
+      "loss": 1.0423,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.9914352893829346,
+      "rewards/margins": 0.290012001991272,
+      "rewards/rejected": -2.281447172164917,
+      "step": 7040
+    },
+    {
+      "epoch": 1.2146795313576844,
+      "grad_norm": 12.399394729053128,
+      "learning_rate": 2.0030147690048372e-08,
+      "logits/chosen": -3.3652491569519043,
+      "logits/rejected": -3.35286021232605,
+      "logps/chosen": -1.9643980264663696,
+      "logps/rejected": -2.2737364768981934,
+      "loss": 1.035,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.9643980264663696,
+      "rewards/margins": 0.3093385398387909,
+      "rewards/rejected": -2.2737364768981934,
+      "step": 7050
+    },
+    {
+      "epoch": 1.2164024810475533,
+      "grad_norm": 13.866428164193943,
+      "learning_rate": 1.995649145360809e-08,
+      "logits/chosen": -3.4365272521972656,
+      "logits/rejected": -3.428710460662842,
+      "logps/chosen": -2.0994956493377686,
+      "logps/rejected": -2.3501229286193848,
+      "loss": 1.0929,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -2.0994956493377686,
+      "rewards/margins": 0.25062718987464905,
+      "rewards/rejected": -2.3501229286193848,
+      "step": 7060
+    },
+    {
+      "epoch": 1.2181254307374225,
+      "grad_norm": 11.238441701383064,
+      "learning_rate": 1.988288082600392e-08,
+      "logits/chosen": -3.4179129600524902,
+      "logits/rejected": -3.4017555713653564,
+      "logps/chosen": -1.9622795581817627,
+      "logps/rejected": -2.2774081230163574,
+      "loss": 1.0734,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.9622795581817627,
+      "rewards/margins": 0.3151286244392395,
+      "rewards/rejected": -2.2774081230163574,
+      "step": 7070
+    },
+    {
+      "epoch": 1.2198483804272915,
+      "grad_norm": 11.087495523296862,
+      "learning_rate": 1.980931647290246e-08,
+      "logits/chosen": -3.445127487182617,
+      "logits/rejected": -3.4299156665802,
+      "logps/chosen": -2.002880334854126,
+      "logps/rejected": -2.385158061981201,
+      "loss": 0.9924,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -2.002880334854126,
+      "rewards/margins": 0.38227784633636475,
+      "rewards/rejected": -2.385158061981201,
+      "step": 7080
+    },
+    {
+      "epoch": 1.2215713301171607,
+      "grad_norm": 15.07375286467558,
+      "learning_rate": 1.97357990595518e-08,
+      "logits/chosen": -3.469676971435547,
+      "logits/rejected": -3.4642257690429688,
+      "logps/chosen": -2.1233890056610107,
+      "logps/rejected": -2.3908755779266357,
+      "loss": 1.1018,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -2.1233890056610107,
+      "rewards/margins": 0.2674865126609802,
+      "rewards/rejected": -2.3908755779266357,
+      "step": 7090
+    },
+    {
+      "epoch": 1.2232942798070296,
+      "grad_norm": 10.08672185839451,
+      "learning_rate": 1.9662329250775585e-08,
+      "logits/chosen": -3.386719226837158,
+      "logits/rejected": -3.376624345779419,
+      "logps/chosen": -2.0070395469665527,
+      "logps/rejected": -2.2200205326080322,
+      "loss": 1.0894,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -2.0070395469665527,
+      "rewards/margins": 0.2129809409379959,
+      "rewards/rejected": -2.2200205326080322,
+      "step": 7100
+    },
+    {
+      "epoch": 1.2232942798070296,
+      "eval_logits/chosen": -3.475728750228882,
+      "eval_logits/rejected": -3.472092866897583,
+      "eval_logps/chosen": -1.8311244249343872,
+      "eval_logps/rejected": -2.01617169380188,
+      "eval_loss": 1.0907784700393677,
+      "eval_rewards/accuracies": 0.6240706443786621,
+      "eval_rewards/chosen": -1.8311244249343872,
+      "eval_rewards/margins": 0.18504735827445984,
+      "eval_rewards/rejected": -2.01617169380188,
+      "eval_runtime": 155.7108,
+      "eval_samples_per_second": 27.641,
+      "eval_steps_per_second": 3.455,
+      "step": 7100
+    },
+    {
+      "epoch": 1.2250172294968986,
+      "grad_norm": 10.101841079861957,
+      "learning_rate": 1.9588907710966942e-08,
+      "logits/chosen": -3.414616346359253,
+      "logits/rejected": -3.394655227661133,
+      "logps/chosen": -1.9432121515274048,
+      "logps/rejected": -2.2663979530334473,
+      "loss": 1.0157,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.9432121515274048,
+      "rewards/margins": 0.32318592071533203,
+      "rewards/rejected": -2.2663979530334473,
+      "step": 7110
+    },
+    {
+      "epoch": 1.2267401791867678,
+      "grad_norm": 12.557076341238842,
+      "learning_rate": 1.951553510408252e-08,
+      "logits/chosen": -3.419938325881958,
+      "logits/rejected": -3.397604465484619,
+      "logps/chosen": -2.060234308242798,
+      "logps/rejected": -2.3409759998321533,
+      "loss": 1.0654,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -2.060234308242798,
+      "rewards/margins": 0.2807416319847107,
+      "rewards/rejected": -2.3409759998321533,
+      "step": 7120
+    },
+    {
+      "epoch": 1.2284631288766368,
+      "grad_norm": 10.106532022600787,
+      "learning_rate": 1.9442212093636433e-08,
+      "logits/chosen": -3.3377959728240967,
+      "logits/rejected": -3.322880983352661,
+      "logps/chosen": -1.943436861038208,
+      "logps/rejected": -2.203794479370117,
+      "loss": 1.0583,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.943436861038208,
+      "rewards/margins": 0.26035767793655396,
+      "rewards/rejected": -2.203794479370117,
+      "step": 7130
+    },
+    {
+      "epoch": 1.230186078566506,
+      "grad_norm": 11.373838916929863,
+      "learning_rate": 1.936893934269433e-08,
+      "logits/chosen": -3.4089043140411377,
+      "logits/rejected": -3.4067273139953613,
+      "logps/chosen": -1.9778871536254883,
+      "logps/rejected": -2.167294979095459,
+      "loss": 1.1165,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.9778871536254883,
+      "rewards/margins": 0.18940754234790802,
+      "rewards/rejected": -2.167294979095459,
+      "step": 7140
+    },
+    {
+      "epoch": 1.231909028256375,
+      "grad_norm": 13.766043385967174,
+      "learning_rate": 1.9295717513867323e-08,
+      "logits/chosen": -3.4566409587860107,
+      "logits/rejected": -3.443812847137451,
+      "logps/chosen": -2.0945372581481934,
+      "logps/rejected": -2.287062883377075,
+      "loss": 1.1116,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -2.0945372581481934,
+      "rewards/margins": 0.1925256848335266,
+      "rewards/rejected": -2.287062883377075,
+      "step": 7150
+    },
+    {
+      "epoch": 1.233631977946244,
+      "grad_norm": 11.19152979693063,
+      "learning_rate": 1.922254726930607e-08,
+      "logits/chosen": -3.400963306427002,
+      "logits/rejected": -3.3808701038360596,
+      "logps/chosen": -1.9656105041503906,
+      "logps/rejected": -2.3819825649261475,
+      "loss": 0.9838,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.9656105041503906,
+      "rewards/margins": 0.4163718819618225,
+      "rewards/rejected": -2.3819825649261475,
+      "step": 7160
+    },
+    {
+      "epoch": 1.235354927636113,
+      "grad_norm": 11.855079267928454,
+      "learning_rate": 1.9149429270694706e-08,
+      "logits/chosen": -3.365699291229248,
+      "logits/rejected": -3.353494167327881,
+      "logps/chosen": -2.074960231781006,
+      "logps/rejected": -2.3256211280822754,
+      "loss": 1.078,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -2.074960231781006,
+      "rewards/margins": 0.25066089630126953,
+      "rewards/rejected": -2.3256211280822754,
+      "step": 7170
+    },
+    {
+      "epoch": 1.237077877325982,
+      "grad_norm": 13.462238229669442,
+      "learning_rate": 1.9076364179244935e-08,
+      "logits/chosen": -3.445920944213867,
+      "logits/rejected": -3.4382290840148926,
+      "logps/chosen": -2.03007173538208,
+      "logps/rejected": -2.25898814201355,
+      "loss": 1.0802,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -2.03007173538208,
+      "rewards/margins": 0.2289164811372757,
+      "rewards/rejected": -2.25898814201355,
+      "step": 7180
+    },
+    {
+      "epoch": 1.2388008270158513,
+      "grad_norm": 14.054445162399643,
+      "learning_rate": 1.9003352655689992e-08,
+      "logits/chosen": -3.3723347187042236,
+      "logits/rejected": -3.359126329421997,
+      "logps/chosen": -2.0436012744903564,
+      "logps/rejected": -2.3185787200927734,
+      "loss": 1.0694,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -2.0436012744903564,
+      "rewards/margins": 0.27497756481170654,
+      "rewards/rejected": -2.3185787200927734,
+      "step": 7190
+    },
+    {
+      "epoch": 1.2405237767057202,
+      "grad_norm": 12.566550672478431,
+      "learning_rate": 1.8930395360278723e-08,
+      "logits/chosen": -3.3667023181915283,
+      "logits/rejected": -3.3580856323242188,
+      "logps/chosen": -1.9698820114135742,
+      "logps/rejected": -2.2830333709716797,
+      "loss": 1.0263,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.9698820114135742,
+      "rewards/margins": 0.31315141916275024,
+      "rewards/rejected": -2.2830333709716797,
+      "step": 7200
+    },
+    {
+      "epoch": 1.2405237767057202,
+      "eval_logits/chosen": -3.4559898376464844,
+      "eval_logits/rejected": -3.452270984649658,
+      "eval_logps/chosen": -1.8363481760025024,
+      "eval_logps/rejected": -2.022144079208374,
+      "eval_loss": 1.0904544591903687,
+      "eval_rewards/accuracies": 0.6247676610946655,
+      "eval_rewards/chosen": -1.8363481760025024,
+      "eval_rewards/margins": 0.18579573929309845,
+      "eval_rewards/rejected": -2.022144079208374,
+      "eval_runtime": 155.6056,
+      "eval_samples_per_second": 27.66,
+      "eval_steps_per_second": 3.457,
+      "step": 7200
+    },
+    {
+      "epoch": 1.2422467263955892,
+      "grad_norm": 15.021935980619334,
+      "learning_rate": 1.885749295276955e-08,
+      "logits/chosen": -3.4415950775146484,
+      "logits/rejected": -3.428610324859619,
+      "logps/chosen": -2.0366597175598145,
+      "logps/rejected": -2.2378270626068115,
+      "loss": 1.1081,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -2.0366597175598145,
+      "rewards/margins": 0.20116767287254333,
+      "rewards/rejected": -2.2378270626068115,
+      "step": 7210
+    },
+    {
+      "epoch": 1.2439696760854584,
+      "grad_norm": 12.096895063503064,
+      "learning_rate": 1.878464609242457e-08,
+      "logits/chosen": -3.3599350452423096,
+      "logits/rejected": -3.34165620803833,
+      "logps/chosen": -2.0816941261291504,
+      "logps/rejected": -2.3739800453186035,
+      "loss": 1.0575,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -2.0816941261291504,
+      "rewards/margins": 0.2922857105731964,
+      "rewards/rejected": -2.3739800453186035,
+      "step": 7220
+    },
+    {
+      "epoch": 1.2456926257753274,
+      "grad_norm": 11.307427465951182,
+      "learning_rate": 1.8711855438003542e-08,
+      "logits/chosen": -3.3892478942871094,
+      "logits/rejected": -3.378993511199951,
+      "logps/chosen": -2.0585381984710693,
+      "logps/rejected": -2.2968831062316895,
+      "loss": 1.0924,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -2.0585381984710693,
+      "rewards/margins": 0.2383447140455246,
+      "rewards/rejected": -2.2968831062316895,
+      "step": 7230
+    },
+    {
+      "epoch": 1.2474155754651963,
+      "grad_norm": 12.78433806749662,
+      "learning_rate": 1.8639121647757975e-08,
+      "logits/chosen": -3.3986752033233643,
+      "logits/rejected": -3.3920435905456543,
+      "logps/chosen": -2.1599035263061523,
+      "logps/rejected": -2.2857723236083984,
+      "loss": 1.1474,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -2.1599035263061523,
+      "rewards/margins": 0.12586870789527893,
+      "rewards/rejected": -2.2857723236083984,
+      "step": 7240
+    },
+    {
+      "epoch": 1.2491385251550655,
+      "grad_norm": 13.553846798175478,
+      "learning_rate": 1.8566445379425115e-08,
+      "logits/chosen": -3.4410316944122314,
+      "logits/rejected": -3.4242382049560547,
+      "logps/chosen": -2.0173120498657227,
+      "logps/rejected": -2.328035354614258,
+      "loss": 1.0222,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -2.0173120498657227,
+      "rewards/margins": 0.3107234835624695,
+      "rewards/rejected": -2.328035354614258,
+      "step": 7250
+    },
+    {
+      "epoch": 1.2508614748449345,
+      "grad_norm": 11.42510795591192,
+      "learning_rate": 1.849382729022207e-08,
+      "logits/chosen": -3.428539752960205,
+      "logits/rejected": -3.411975383758545,
+      "logps/chosen": -1.9945770502090454,
+      "logps/rejected": -2.315516233444214,
+      "loss": 1.0335,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.9945770502090454,
+      "rewards/margins": 0.32093897461891174,
+      "rewards/rejected": -2.315516233444214,
+      "step": 7260
+    },
+    {
+      "epoch": 1.2525844245348035,
+      "grad_norm": 13.197903252705819,
+      "learning_rate": 1.8421268036839798e-08,
+      "logits/chosen": -3.439929485321045,
+      "logits/rejected": -3.424172878265381,
+      "logps/chosen": -2.041116237640381,
+      "logps/rejected": -2.322969436645508,
+      "loss": 1.0712,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -2.041116237640381,
+      "rewards/margins": 0.2818533778190613,
+      "rewards/rejected": -2.322969436645508,
+      "step": 7270
+    },
+    {
+      "epoch": 1.2543073742246726,
+      "grad_norm": 12.174912410883843,
+      "learning_rate": 1.834876827543721e-08,
+      "logits/chosen": -3.4464309215545654,
+      "logits/rejected": -3.4286842346191406,
+      "logps/chosen": -2.011495351791382,
+      "logps/rejected": -2.376075267791748,
+      "loss": 1.009,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -2.011495351791382,
+      "rewards/margins": 0.3645797073841095,
+      "rewards/rejected": -2.376075267791748,
+      "step": 7280
+    },
+    {
+      "epoch": 1.2560303239145416,
+      "grad_norm": 11.933555355344582,
+      "learning_rate": 1.827632866163525e-08,
+      "logits/chosen": -3.324185848236084,
+      "logits/rejected": -3.317171573638916,
+      "logps/chosen": -2.080921173095703,
+      "logps/rejected": -2.2889628410339355,
+      "loss": 1.1028,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -2.080921173095703,
+      "rewards/margins": 0.2080419957637787,
+      "rewards/rejected": -2.2889628410339355,
+      "step": 7290
+    },
+    {
+      "epoch": 1.2577532736044108,
+      "grad_norm": 13.139981660585914,
+      "learning_rate": 1.8203949850510903e-08,
+      "logits/chosen": -3.2991180419921875,
+      "logits/rejected": -3.2894935607910156,
+      "logps/chosen": -1.9979912042617798,
+      "logps/rejected": -2.274498462677002,
+      "loss": 1.0575,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.9979912042617798,
+      "rewards/margins": 0.2765074372291565,
+      "rewards/rejected": -2.274498462677002,
+      "step": 7300
+    },
+    {
+      "epoch": 1.2577532736044108,
+      "eval_logits/chosen": -3.4567437171936035,
+      "eval_logits/rejected": -3.453038215637207,
+      "eval_logps/chosen": -1.84248948097229,
+      "eval_logps/rejected": -2.0289347171783447,
+      "eval_loss": 1.0902963876724243,
+      "eval_rewards/accuracies": 0.6243029832839966,
+      "eval_rewards/chosen": -1.84248948097229,
+      "eval_rewards/margins": 0.1864451915025711,
+      "eval_rewards/rejected": -2.0289347171783447,
+      "eval_runtime": 155.5158,
+      "eval_samples_per_second": 27.676,
+      "eval_steps_per_second": 3.459,
+      "step": 7300
+    },
+    {
+      "epoch": 1.2594762232942798,
+      "grad_norm": 11.688336566428697,
+      "learning_rate": 1.8131632496591348e-08,
+      "logits/chosen": -3.421450138092041,
+      "logits/rejected": -3.4082305431365967,
+      "logps/chosen": -2.073664665222168,
+      "logps/rejected": -2.3233070373535156,
+      "loss": 1.0792,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -2.073664665222168,
+      "rewards/margins": 0.24964237213134766,
+      "rewards/rejected": -2.3233070373535156,
+      "step": 7310
+    },
+    {
+      "epoch": 1.2611991729841487,
+      "grad_norm": 10.628978353232347,
+      "learning_rate": 1.8059377253847973e-08,
+      "logits/chosen": -3.43058443069458,
+      "logits/rejected": -3.4165706634521484,
+      "logps/chosen": -2.011075496673584,
+      "logps/rejected": -2.2387638092041016,
+      "loss": 1.0932,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -2.011075496673584,
+      "rewards/margins": 0.227688267827034,
+      "rewards/rejected": -2.2387638092041016,
+      "step": 7320
+    },
+    {
+      "epoch": 1.262922122674018,
+      "grad_norm": 12.652410799204025,
+      "learning_rate": 1.798718477569051e-08,
+      "logits/chosen": -3.3940834999084473,
+      "logits/rejected": -3.3794875144958496,
+      "logps/chosen": -1.9532549381256104,
+      "logps/rejected": -2.32130765914917,
+      "loss": 1.0048,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.9532549381256104,
+      "rewards/margins": 0.36805295944213867,
+      "rewards/rejected": -2.32130765914917,
+      "step": 7330
+    },
+    {
+      "epoch": 1.264645072363887,
+      "grad_norm": 11.175519085954091,
+      "learning_rate": 1.791505571496109e-08,
+      "logits/chosen": -3.4181129932403564,
+      "logits/rejected": -3.4039390087127686,
+      "logps/chosen": -1.9738986492156982,
+      "logps/rejected": -2.2823987007141113,
+      "loss": 1.0301,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.9738986492156982,
+      "rewards/margins": 0.3084999918937683,
+      "rewards/rejected": -2.2823987007141113,
+      "step": 7340
+    },
+    {
+      "epoch": 1.266368022053756,
+      "grad_norm": 11.194575620272454,
+      "learning_rate": 1.7842990723928375e-08,
+      "logits/chosen": -3.442927598953247,
+      "logits/rejected": -3.42742657661438,
+      "logps/chosen": -1.955824851989746,
+      "logps/rejected": -2.2152419090270996,
+      "loss": 1.0569,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.955824851989746,
+      "rewards/margins": 0.2594171166419983,
+      "rewards/rejected": -2.2152419090270996,
+      "step": 7350
+    },
+    {
+      "epoch": 1.268090971743625,
+      "grad_norm": 9.95718672742114,
+      "learning_rate": 1.7770990454281608e-08,
+      "logits/chosen": -3.3960044384002686,
+      "logits/rejected": -3.383613109588623,
+      "logps/chosen": -2.0251574516296387,
+      "logps/rejected": -2.2762949466705322,
+      "loss": 1.0728,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -2.0251574516296387,
+      "rewards/margins": 0.2511375844478607,
+      "rewards/rejected": -2.2762949466705322,
+      "step": 7360
+    },
+    {
+      "epoch": 1.269813921433494,
+      "grad_norm": 11.671859902864446,
+      "learning_rate": 1.7699055557124793e-08,
+      "logits/chosen": -3.326345920562744,
+      "logits/rejected": -3.3126749992370605,
+      "logps/chosen": -2.023937463760376,
+      "logps/rejected": -2.327054977416992,
+      "loss": 1.0497,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -2.023937463760376,
+      "rewards/margins": 0.30311763286590576,
+      "rewards/rejected": -2.327054977416992,
+      "step": 7370
+    },
+    {
+      "epoch": 1.2715368711233632,
+      "grad_norm": 11.568969325266785,
+      "learning_rate": 1.7627186682970725e-08,
+      "logits/chosen": -3.381850481033325,
+      "logits/rejected": -3.371791362762451,
+      "logps/chosen": -2.031034231185913,
+      "logps/rejected": -2.343127727508545,
+      "loss": 1.0438,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -2.031034231185913,
+      "rewards/margins": 0.3120933473110199,
+      "rewards/rejected": -2.343127727508545,
+      "step": 7380
+    },
+    {
+      "epoch": 1.2732598208132322,
+      "grad_norm": 15.717793615472472,
+      "learning_rate": 1.755538448173518e-08,
+      "logits/chosen": -3.364393711090088,
+      "logits/rejected": -3.3527169227600098,
+      "logps/chosen": -2.0175838470458984,
+      "logps/rejected": -2.3025097846984863,
+      "loss": 1.0565,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -2.0175838470458984,
+      "rewards/margins": 0.2849257290363312,
+      "rewards/rejected": -2.3025097846984863,
+      "step": 7390
+    },
+    {
+      "epoch": 1.2749827705031014,
+      "grad_norm": 12.415925244997085,
+      "learning_rate": 1.7483649602730987e-08,
+      "logits/chosen": -3.3715178966522217,
+      "logits/rejected": -3.3511548042297363,
+      "logps/chosen": -2.085451602935791,
+      "logps/rejected": -2.4355430603027344,
+      "loss": 1.0439,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -2.085451602935791,
+      "rewards/margins": 0.3500916659832001,
+      "rewards/rejected": -2.4355430603027344,
+      "step": 7400
+    },
+    {
+      "epoch": 1.2749827705031014,
+      "eval_logits/chosen": -3.4655940532684326,
+      "eval_logits/rejected": -3.46195125579834,
+      "eval_logps/chosen": -1.847452998161316,
+      "eval_logps/rejected": -2.0349009037017822,
+      "eval_loss": 1.089822769165039,
+      "eval_rewards/accuracies": 0.6236059665679932,
+      "eval_rewards/chosen": -1.847452998161316,
+      "eval_rewards/margins": 0.18744820356369019,
+      "eval_rewards/rejected": -2.0349009037017822,
+      "eval_runtime": 155.4653,
+      "eval_samples_per_second": 27.685,
+      "eval_steps_per_second": 3.461,
+      "step": 7400
+    },
+    {
+      "epoch": 1.2767057201929704,
+      "grad_norm": 14.983298806442155,
+      "learning_rate": 1.741198269466219e-08,
+      "logits/chosen": -3.3490099906921387,
+      "logits/rejected": -3.3324408531188965,
+      "logps/chosen": -1.975225806236267,
+      "logps/rejected": -2.279466390609741,
+      "loss": 1.0438,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.975225806236267,
+      "rewards/margins": 0.30424046516418457,
+      "rewards/rejected": -2.279466390609741,
+      "step": 7410
+    },
+    {
+      "epoch": 1.2784286698828393,
+      "grad_norm": 11.029093349498345,
+      "learning_rate": 1.7340384405618133e-08,
+      "logits/chosen": -3.3143889904022217,
+      "logits/rejected": -3.2999815940856934,
+      "logps/chosen": -1.920106291770935,
+      "logps/rejected": -2.228076219558716,
+      "loss": 1.0344,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.920106291770935,
+      "rewards/margins": 0.30796995759010315,
+      "rewards/rejected": -2.228076219558716,
+      "step": 7420
+    },
+    {
+      "epoch": 1.2801516195727085,
+      "grad_norm": 12.208958163038682,
+      "learning_rate": 1.7268855383067683e-08,
+      "logits/chosen": -3.3549251556396484,
+      "logits/rejected": -3.337815523147583,
+      "logps/chosen": -2.1092162132263184,
+      "logps/rejected": -2.4394733905792236,
+      "loss": 1.027,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -2.1092162132263184,
+      "rewards/margins": 0.33025720715522766,
+      "rewards/rejected": -2.4394733905792236,
+      "step": 7430
+    },
+    {
+      "epoch": 1.2818745692625775,
+      "grad_norm": 14.740158471302783,
+      "learning_rate": 1.7197396273853275e-08,
+      "logits/chosen": -3.4213085174560547,
+      "logits/rejected": -3.4069762229919434,
+      "logps/chosen": -2.1035516262054443,
+      "logps/rejected": -2.4722466468811035,
+      "loss": 1.0382,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -2.1035516262054443,
+      "rewards/margins": 0.3686951994895935,
+      "rewards/rejected": -2.4722466468811035,
+      "step": 7440
+    },
+    {
+      "epoch": 1.2835975189524467,
+      "grad_norm": 12.854227455979249,
+      "learning_rate": 1.7126007724185164e-08,
+      "logits/chosen": -3.444411039352417,
+      "logits/rejected": -3.431365966796875,
+      "logps/chosen": -2.0490634441375732,
+      "logps/rejected": -2.333268642425537,
+      "loss": 1.0547,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -2.0490634441375732,
+      "rewards/margins": 0.28420501947402954,
+      "rewards/rejected": -2.333268642425537,
+      "step": 7450
+    },
+    {
+      "epoch": 1.2853204686423156,
+      "grad_norm": 13.306979214239679,
+      "learning_rate": 1.705469037963548e-08,
+      "logits/chosen": -3.332761287689209,
+      "logits/rejected": -3.334984302520752,
+      "logps/chosen": -2.072204828262329,
+      "logps/rejected": -2.260439157485962,
+      "loss": 1.1201,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -2.072204828262329,
+      "rewards/margins": 0.1882343292236328,
+      "rewards/rejected": -2.260439157485962,
+      "step": 7460
+    },
+    {
+      "epoch": 1.2870434183321846,
+      "grad_norm": 11.756790225962856,
+      "learning_rate": 1.698344488513247e-08,
+      "logits/chosen": -3.387481689453125,
+      "logits/rejected": -3.3776440620422363,
+      "logps/chosen": -2.0246710777282715,
+      "logps/rejected": -2.179156541824341,
+      "loss": 1.1283,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -2.0246710777282715,
+      "rewards/margins": 0.15448543429374695,
+      "rewards/rejected": -2.179156541824341,
+      "step": 7470
+    },
+    {
+      "epoch": 1.2887663680220538,
+      "grad_norm": 10.701186262013573,
+      "learning_rate": 1.691227188495461e-08,
+      "logits/chosen": -3.3690123558044434,
+      "logits/rejected": -3.353325605392456,
+      "logps/chosen": -2.003532648086548,
+      "logps/rejected": -2.205436944961548,
+      "loss": 1.1195,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -2.003532648086548,
+      "rewards/margins": 0.20190434157848358,
+      "rewards/rejected": -2.205436944961548,
+      "step": 7480
+    },
+    {
+      "epoch": 1.2904893177119228,
+      "grad_norm": 11.251701291536458,
+      "learning_rate": 1.684117202272485e-08,
+      "logits/chosen": -3.3602306842803955,
+      "logits/rejected": -3.3525567054748535,
+      "logps/chosen": -2.0144968032836914,
+      "logps/rejected": -2.304309844970703,
+      "loss": 1.0324,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -2.0144968032836914,
+      "rewards/margins": 0.28981316089630127,
+      "rewards/rejected": -2.304309844970703,
+      "step": 7490
+    },
+    {
+      "epoch": 1.292212267401792,
+      "grad_norm": 10.165006694306847,
+      "learning_rate": 1.6770145941404697e-08,
+      "logits/chosen": -3.36047625541687,
+      "logits/rejected": -3.3479857444763184,
+      "logps/chosen": -1.9401159286499023,
+      "logps/rejected": -2.25445818901062,
+      "loss": 1.0479,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.9401159286499023,
+      "rewards/margins": 0.3143422305583954,
+      "rewards/rejected": -2.25445818901062,
+      "step": 7500
+    },
+    {
+      "epoch": 1.292212267401792,
+      "eval_logits/chosen": -3.4558935165405273,
+      "eval_logits/rejected": -3.452216625213623,
+      "eval_logps/chosen": -1.8506419658660889,
+      "eval_logps/rejected": -2.0381667613983154,
+      "eval_loss": 1.0897728204727173,
+      "eval_rewards/accuracies": 0.6247676610946655,
+      "eval_rewards/chosen": -1.8506419658660889,
+      "eval_rewards/margins": 0.18752481043338776,
+      "eval_rewards/rejected": -2.0381667613983154,
+      "eval_runtime": 155.3453,
+      "eval_samples_per_second": 27.706,
+      "eval_steps_per_second": 3.463,
+      "step": 7500
+    },
+    {
+      "epoch": 1.293935217091661,
+      "grad_norm": 12.927242233434502,
+      "learning_rate": 1.669919428328847e-08,
+      "logits/chosen": -3.3886566162109375,
+      "logits/rejected": -3.368140697479248,
+      "logps/chosen": -2.008953809738159,
+      "logps/rejected": -2.2524211406707764,
+      "loss": 1.0685,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -2.008953809738159,
+      "rewards/margins": 0.24346742033958435,
+      "rewards/rejected": -2.2524211406707764,
+      "step": 7510
+    },
+    {
+      "epoch": 1.29565816678153,
+      "grad_norm": 12.345860512094035,
+      "learning_rate": 1.66283176899975e-08,
+      "logits/chosen": -3.3704612255096436,
+      "logits/rejected": -3.359811305999756,
+      "logps/chosen": -1.9886213541030884,
+      "logps/rejected": -2.330554962158203,
+      "loss": 1.026,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.9886213541030884,
+      "rewards/margins": 0.3419334590435028,
+      "rewards/rejected": -2.330554962158203,
+      "step": 7520
+    },
+    {
+      "epoch": 1.297381116471399,
+      "grad_norm": 10.57424605037142,
+      "learning_rate": 1.6557516802474246e-08,
+      "logits/chosen": -3.338059186935425,
+      "logits/rejected": -3.3367362022399902,
+      "logps/chosen": -2.029294013977051,
+      "logps/rejected": -2.2100818157196045,
+      "loss": 1.1101,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -2.029294013977051,
+      "rewards/margins": 0.1807878464460373,
+      "rewards/rejected": -2.2100818157196045,
+      "step": 7530
+    },
+    {
+      "epoch": 1.299104066161268,
+      "grad_norm": 10.325180780041435,
+      "learning_rate": 1.648679226097662e-08,
+      "logits/chosen": -3.4370803833007812,
+      "logits/rejected": -3.433239459991455,
+      "logps/chosen": -1.9779422283172607,
+      "logps/rejected": -2.2113795280456543,
+      "loss": 1.0822,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.9779422283172607,
+      "rewards/margins": 0.23343737423419952,
+      "rewards/rejected": -2.2113795280456543,
+      "step": 7540
+    },
+    {
+      "epoch": 1.3008270158511372,
+      "grad_norm": 13.52188188826102,
+      "learning_rate": 1.641614470507207e-08,
+      "logits/chosen": -3.3910441398620605,
+      "logits/rejected": -3.3798294067382812,
+      "logps/chosen": -2.000918388366699,
+      "logps/rejected": -2.2934141159057617,
+      "loss": 1.0337,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -2.000918388366699,
+      "rewards/margins": 0.2924959063529968,
+      "rewards/rejected": -2.2934141159057617,
+      "step": 7550
+    },
+    {
+      "epoch": 1.3025499655410062,
+      "grad_norm": 14.056619723199976,
+      "learning_rate": 1.6345574773631897e-08,
+      "logits/chosen": -3.427295207977295,
+      "logits/rejected": -3.4164741039276123,
+      "logps/chosen": -1.9288158416748047,
+      "logps/rejected": -2.407188653945923,
+      "loss": 0.966,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.9288158416748047,
+      "rewards/margins": 0.4783729016780853,
+      "rewards/rejected": -2.407188653945923,
+      "step": 7560
+    },
+    {
+      "epoch": 1.3042729152308752,
+      "grad_norm": 13.714116999788336,
+      "learning_rate": 1.627508310482541e-08,
+      "logits/chosen": -3.432959794998169,
+      "logits/rejected": -3.4226584434509277,
+      "logps/chosen": -2.1240766048431396,
+      "logps/rejected": -2.2854795455932617,
+      "loss": 1.1264,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -2.1240766048431396,
+      "rewards/margins": 0.16140279173851013,
+      "rewards/rejected": -2.2854795455932617,
+      "step": 7570
+    },
+    {
+      "epoch": 1.3059958649207444,
+      "grad_norm": 13.03117444023929,
+      "learning_rate": 1.6204670336114223e-08,
+      "logits/chosen": -3.3991408348083496,
+      "logits/rejected": -3.387526750564575,
+      "logps/chosen": -1.9581336975097656,
+      "logps/rejected": -2.197786569595337,
+      "loss": 1.0743,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.9581336975097656,
+      "rewards/margins": 0.23965275287628174,
+      "rewards/rejected": -2.197786569595337,
+      "step": 7580
+    },
+    {
+      "epoch": 1.3077188146106133,
+      "grad_norm": 15.185829064146443,
+      "learning_rate": 1.6134337104246395e-08,
+      "logits/chosen": -3.4231231212615967,
+      "logits/rejected": -3.4002633094787598,
+      "logps/chosen": -2.0717878341674805,
+      "logps/rejected": -2.425459384918213,
+      "loss": 1.0182,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -2.0717878341674805,
+      "rewards/margins": 0.3536716103553772,
+      "rewards/rejected": -2.425459384918213,
+      "step": 7590
+    },
+    {
+      "epoch": 1.3094417643004825,
+      "grad_norm": 12.400605509854776,
+      "learning_rate": 1.6064084045250787e-08,
+      "logits/chosen": -3.4041073322296143,
+      "logits/rejected": -3.3882572650909424,
+      "logps/chosen": -2.050949811935425,
+      "logps/rejected": -2.3748888969421387,
+      "loss": 1.0345,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -2.050949811935425,
+      "rewards/margins": 0.32393884658813477,
+      "rewards/rejected": -2.3748888969421387,
+      "step": 7600
+    },
+    {
+      "epoch": 1.3094417643004825,
+      "eval_logits/chosen": -3.459791421890259,
+      "eval_logits/rejected": -3.456153392791748,
+      "eval_logps/chosen": -1.8523318767547607,
+      "eval_logps/rejected": -2.040158271789551,
+      "eval_loss": 1.0897589921951294,
+      "eval_rewards/accuracies": 0.6238383054733276,
+      "eval_rewards/chosen": -1.8523318767547607,
+      "eval_rewards/margins": 0.18782643973827362,
+      "eval_rewards/rejected": -2.040158271789551,
+      "eval_runtime": 155.7625,
+      "eval_samples_per_second": 27.632,
+      "eval_steps_per_second": 3.454,
+      "step": 7600
+    },
+    {
+      "epoch": 1.3111647139903515,
+      "grad_norm": 9.516411328503247,
+      "learning_rate": 1.5993911794431198e-08,
+      "logits/chosen": -3.3750083446502686,
+      "logits/rejected": -3.359097957611084,
+      "logps/chosen": -2.0369513034820557,
+      "logps/rejected": -2.3835208415985107,
+      "loss": 1.018,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -2.0369513034820557,
+      "rewards/margins": 0.34656962752342224,
+      "rewards/rejected": -2.3835208415985107,
+      "step": 7610
+    },
+    {
+      "epoch": 1.3128876636802205,
+      "grad_norm": 12.09389879477969,
+      "learning_rate": 1.59238209863607e-08,
+      "logits/chosen": -3.396771192550659,
+      "logits/rejected": -3.3825621604919434,
+      "logps/chosen": -2.0092689990997314,
+      "logps/rejected": -2.210419178009033,
+      "loss": 1.1054,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -2.0092689990997314,
+      "rewards/margins": 0.20115017890930176,
+      "rewards/rejected": -2.210419178009033,
+      "step": 7620
+    },
+    {
+      "epoch": 1.3146106133700897,
+      "grad_norm": 11.103097139783335,
+      "learning_rate": 1.5853812254875877e-08,
+      "logits/chosen": -3.360229015350342,
+      "logits/rejected": -3.364773988723755,
+      "logps/chosen": -2.0901896953582764,
+      "logps/rejected": -2.169556140899658,
+      "loss": 1.1864,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -2.0901896953582764,
+      "rewards/margins": 0.07936622947454453,
+      "rewards/rejected": -2.169556140899658,
+      "step": 7630
+    },
+    {
+      "epoch": 1.3163335630599586,
+      "grad_norm": 14.177774720912685,
+      "learning_rate": 1.5783886233071076e-08,
+      "logits/chosen": -3.337369203567505,
+      "logits/rejected": -3.325772762298584,
+      "logps/chosen": -2.021383762359619,
+      "logps/rejected": -2.233210325241089,
+      "loss": 1.0911,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -2.021383762359619,
+      "rewards/margins": 0.2118268758058548,
+      "rewards/rejected": -2.233210325241089,
+      "step": 7640
+    },
+    {
+      "epoch": 1.3180565127498278,
+      "grad_norm": 11.940445988756826,
+      "learning_rate": 1.5714043553292683e-08,
+      "logits/chosen": -3.426755428314209,
+      "logits/rejected": -3.412158250808716,
+      "logps/chosen": -2.1203107833862305,
+      "logps/rejected": -2.4554195404052734,
+      "loss": 1.0302,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -2.1203107833862305,
+      "rewards/margins": 0.33510875701904297,
+      "rewards/rejected": -2.4554195404052734,
+      "step": 7650
+    },
+    {
+      "epoch": 1.3197794624396968,
+      "grad_norm": 11.489762607185208,
+      "learning_rate": 1.564428484713345e-08,
+      "logits/chosen": -3.404176712036133,
+      "logits/rejected": -3.3825104236602783,
+      "logps/chosen": -1.9825506210327148,
+      "logps/rejected": -2.3892266750335693,
+      "loss": 0.9893,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.9825506210327148,
+      "rewards/margins": 0.4066759943962097,
+      "rewards/rejected": -2.3892266750335693,
+      "step": 7660
+    },
+    {
+      "epoch": 1.3215024121295658,
+      "grad_norm": 13.087131319644127,
+      "learning_rate": 1.5574610745426703e-08,
+      "logits/chosen": -3.3712191581726074,
+      "logits/rejected": -3.355543851852417,
+      "logps/chosen": -2.0651016235351562,
+      "logps/rejected": -2.333069324493408,
+      "loss": 1.0704,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -2.0651016235351562,
+      "rewards/margins": 0.26796749234199524,
+      "rewards/rejected": -2.333069324493408,
+      "step": 7670
+    },
+    {
+      "epoch": 1.323225361819435,
+      "grad_norm": 11.412249819158044,
+      "learning_rate": 1.550502187824073e-08,
+      "logits/chosen": -3.4189200401306152,
+      "logits/rejected": -3.408628463745117,
+      "logps/chosen": -2.0064682960510254,
+      "logps/rejected": -2.2163379192352295,
+      "loss": 1.0934,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -2.0064682960510254,
+      "rewards/margins": 0.2098696231842041,
+      "rewards/rejected": -2.2163379192352295,
+      "step": 7680
+    },
+    {
+      "epoch": 1.324948311509304,
+      "grad_norm": 14.179312468830092,
+      "learning_rate": 1.543551887487301e-08,
+      "logits/chosen": -3.462543487548828,
+      "logits/rejected": -3.439073085784912,
+      "logps/chosen": -1.9205774068832397,
+      "logps/rejected": -2.188371181488037,
+      "loss": 1.0494,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.9205774068832397,
+      "rewards/margins": 0.2677935063838959,
+      "rewards/rejected": -2.188371181488037,
+      "step": 7690
+    },
+    {
+      "epoch": 1.3266712611991731,
+      "grad_norm": 12.237808468423477,
+      "learning_rate": 1.536610236384455e-08,
+      "logits/chosen": -3.384690523147583,
+      "logits/rejected": -3.372149705886841,
+      "logps/chosen": -1.9653714895248413,
+      "logps/rejected": -2.29309344291687,
+      "loss": 1.0292,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.9653714895248413,
+      "rewards/margins": 0.32772213220596313,
+      "rewards/rejected": -2.29309344291687,
+      "step": 7700
+    },
+    {
+      "epoch": 1.3266712611991731,
+      "eval_logits/chosen": -3.4526891708374023,
+      "eval_logits/rejected": -3.4490208625793457,
+      "eval_logps/chosen": -1.8566126823425293,
+      "eval_logps/rejected": -2.0451242923736572,
+      "eval_loss": 1.089496374130249,
+      "eval_rewards/accuracies": 0.6243029832839966,
+      "eval_rewards/chosen": -1.8566126823425293,
+      "eval_rewards/margins": 0.1885116696357727,
+      "eval_rewards/rejected": -2.0451242923736572,
+      "eval_runtime": 155.5806,
+      "eval_samples_per_second": 27.664,
+      "eval_steps_per_second": 3.458,
+      "step": 7700
+    },
+    {
+      "epoch": 1.328394210889042,
+      "grad_norm": 13.170768690097356,
+      "learning_rate": 1.5296772972894213e-08,
+      "logits/chosen": -3.4126648902893066,
+      "logits/rejected": -3.4054999351501465,
+      "logps/chosen": -2.1215553283691406,
+      "logps/rejected": -2.3539912700653076,
+      "loss": 1.1109,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -2.1215553283691406,
+      "rewards/margins": 0.23243579268455505,
+      "rewards/rejected": -2.3539912700653076,
+      "step": 7710
+    },
+    {
+      "epoch": 1.330117160578911,
+      "grad_norm": 12.214716302751475,
+      "learning_rate": 1.5227531328972994e-08,
+      "logits/chosen": -3.390167713165283,
+      "logits/rejected": -3.3718864917755127,
+      "logps/chosen": -2.094923734664917,
+      "logps/rejected": -2.347651243209839,
+      "loss": 1.0647,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -2.094923734664917,
+      "rewards/margins": 0.25272756814956665,
+      "rewards/rejected": -2.347651243209839,
+      "step": 7720
+    },
+    {
+      "epoch": 1.33184011026878,
+      "grad_norm": 11.014935539428267,
+      "learning_rate": 1.5158378058238442e-08,
+      "logits/chosen": -3.378035306930542,
+      "logits/rejected": -3.3674354553222656,
+      "logps/chosen": -2.0667848587036133,
+      "logps/rejected": -2.2910842895507812,
+      "loss": 1.0808,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -2.0667848587036133,
+      "rewards/margins": 0.2242996245622635,
+      "rewards/rejected": -2.2910842895507812,
+      "step": 7730
+    },
+    {
+      "epoch": 1.3335630599586492,
+      "grad_norm": 12.352892729135384,
+      "learning_rate": 1.5089313786048885e-08,
+      "logits/chosen": -3.365837812423706,
+      "logits/rejected": -3.3571364879608154,
+      "logps/chosen": -2.012409210205078,
+      "logps/rejected": -2.3463380336761475,
+      "loss": 1.0261,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -2.012409210205078,
+      "rewards/margins": 0.3339290916919708,
+      "rewards/rejected": -2.3463380336761475,
+      "step": 7740
+    },
+    {
+      "epoch": 1.3352860096485184,
+      "grad_norm": 11.554220085876954,
+      "learning_rate": 1.5020339136957876e-08,
+      "logits/chosen": -3.363211154937744,
+      "logits/rejected": -3.3451881408691406,
+      "logps/chosen": -2.070110559463501,
+      "logps/rejected": -2.4833667278289795,
+      "loss": 1.0041,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -2.070110559463501,
+      "rewards/margins": 0.4132562577724457,
+      "rewards/rejected": -2.4833667278289795,
+      "step": 7750
+    },
+    {
+      "epoch": 1.3370089593383874,
+      "grad_norm": 12.930911510285881,
+      "learning_rate": 1.4951454734708456e-08,
+      "logits/chosen": -3.3254027366638184,
+      "logits/rejected": -3.3142306804656982,
+      "logps/chosen": -1.9629294872283936,
+      "logps/rejected": -2.2675070762634277,
+      "loss": 1.0467,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.9629294872283936,
+      "rewards/margins": 0.30457746982574463,
+      "rewards/rejected": -2.2675070762634277,
+      "step": 7760
+    },
+    {
+      "epoch": 1.3387319090282563,
+      "grad_norm": 11.167738552181161,
+      "learning_rate": 1.4882661202227597e-08,
+      "logits/chosen": -3.3489997386932373,
+      "logits/rejected": -3.3371243476867676,
+      "logps/chosen": -1.968117117881775,
+      "logps/rejected": -2.2978737354278564,
+      "loss": 1.0011,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.968117117881775,
+      "rewards/margins": 0.3297564387321472,
+      "rewards/rejected": -2.2978737354278564,
+      "step": 7770
+    },
+    {
+      "epoch": 1.3404548587181253,
+      "grad_norm": 14.738434879798906,
+      "learning_rate": 1.4813959161620502e-08,
+      "logits/chosen": -3.4426791667938232,
+      "logits/rejected": -3.43316650390625,
+      "logps/chosen": -2.0997540950775146,
+      "logps/rejected": -2.3920369148254395,
+      "loss": 1.054,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -2.0997540950775146,
+      "rewards/margins": 0.29228293895721436,
+      "rewards/rejected": -2.3920369148254395,
+      "step": 7780
+    },
+    {
+      "epoch": 1.3421778084079945,
+      "grad_norm": 13.9267779028636,
+      "learning_rate": 1.4745349234165017e-08,
+      "logits/chosen": -3.393700122833252,
+      "logits/rejected": -3.385270595550537,
+      "logps/chosen": -2.0611891746520996,
+      "logps/rejected": -2.326726198196411,
+      "loss": 1.0571,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -2.0611891746520996,
+      "rewards/margins": 0.2655371427536011,
+      "rewards/rejected": -2.326726198196411,
+      "step": 7790
+    },
+    {
+      "epoch": 1.3439007580978635,
+      "grad_norm": 12.855215070214832,
+      "learning_rate": 1.4676832040305984e-08,
+      "logits/chosen": -3.418776750564575,
+      "logits/rejected": -3.415955066680908,
+      "logps/chosen": -2.0785863399505615,
+      "logps/rejected": -2.3359203338623047,
+      "loss": 1.0667,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -2.0785863399505615,
+      "rewards/margins": 0.2573338449001312,
+      "rewards/rejected": -2.3359203338623047,
+      "step": 7800
+    },
+    {
+      "epoch": 1.3439007580978635,
+      "eval_logits/chosen": -3.44140362739563,
+      "eval_logits/rejected": -3.4376933574676514,
+      "eval_logps/chosen": -1.8601454496383667,
+      "eval_logps/rejected": -2.048933744430542,
+      "eval_loss": 1.0895626544952393,
+      "eval_rewards/accuracies": 0.6243029832839966,
+      "eval_rewards/chosen": -1.8601454496383667,
+      "eval_rewards/margins": 0.1887885183095932,
+      "eval_rewards/rejected": -2.048933744430542,
+      "eval_runtime": 155.6091,
+      "eval_samples_per_second": 27.659,
+      "eval_steps_per_second": 3.457,
+      "step": 7800
+    },
+    {
+      "epoch": 1.3456237077877327,
+      "grad_norm": 12.889882231454884,
+      "learning_rate": 1.4608408199649686e-08,
+      "logits/chosen": -3.430255174636841,
+      "logits/rejected": -3.416360855102539,
+      "logps/chosen": -2.055924892425537,
+      "logps/rejected": -2.345905303955078,
+      "loss": 1.0763,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -2.055924892425537,
+      "rewards/margins": 0.28998011350631714,
+      "rewards/rejected": -2.345905303955078,
+      "step": 7810
+    },
+    {
+      "epoch": 1.3473466574776016,
+      "grad_norm": 12.659786921433474,
+      "learning_rate": 1.4540078330958166e-08,
+      "logits/chosen": -3.4072914123535156,
+      "logits/rejected": -3.391702175140381,
+      "logps/chosen": -1.9663101434707642,
+      "logps/rejected": -2.332919120788574,
+      "loss": 1.0414,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.9663101434707642,
+      "rewards/margins": 0.3666090965270996,
+      "rewards/rejected": -2.332919120788574,
+      "step": 7820
+    },
+    {
+      "epoch": 1.3490696071674706,
+      "grad_norm": 13.851236044115696,
+      "learning_rate": 1.4471843052143696e-08,
+      "logits/chosen": -3.3566060066223145,
+      "logits/rejected": -3.3519511222839355,
+      "logps/chosen": -2.099036693572998,
+      "logps/rejected": -2.3890480995178223,
+      "loss": 1.0565,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -2.099036693572998,
+      "rewards/margins": 0.29001155495643616,
+      "rewards/rejected": -2.3890480995178223,
+      "step": 7830
+    },
+    {
+      "epoch": 1.3507925568573398,
+      "grad_norm": 11.89029049010653,
+      "learning_rate": 1.4403702980263149e-08,
+      "logits/chosen": -3.3504226207733154,
+      "logits/rejected": -3.338012218475342,
+      "logps/chosen": -2.0554652214050293,
+      "logps/rejected": -2.3409929275512695,
+      "loss": 1.0519,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -2.0554652214050293,
+      "rewards/margins": 0.28552764654159546,
+      "rewards/rejected": -2.3409929275512695,
+      "step": 7840
+    },
+    {
+      "epoch": 1.3525155065472088,
+      "grad_norm": 12.469836065712395,
+      "learning_rate": 1.4335658731512452e-08,
+      "logits/chosen": -3.347590923309326,
+      "logits/rejected": -3.331817626953125,
+      "logps/chosen": -2.1141467094421387,
+      "logps/rejected": -2.2607851028442383,
+      "loss": 1.1523,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -2.1141467094421387,
+      "rewards/margins": 0.14663848280906677,
+      "rewards/rejected": -2.2607851028442383,
+      "step": 7850
+    },
+    {
+      "epoch": 1.354238456237078,
+      "grad_norm": 9.035061775995686,
+      "learning_rate": 1.4267710921220974e-08,
+      "logits/chosen": -3.356928586959839,
+      "logits/rejected": -3.3394832611083984,
+      "logps/chosen": -1.9854180812835693,
+      "logps/rejected": -2.3981335163116455,
+      "loss": 0.9689,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.9854180812835693,
+      "rewards/margins": 0.4127153754234314,
+      "rewards/rejected": -2.3981335163116455,
+      "step": 7860
+    },
+    {
+      "epoch": 1.355961405926947,
+      "grad_norm": 10.49579102799946,
+      "learning_rate": 1.4199860163846007e-08,
+      "logits/chosen": -3.382218599319458,
+      "logits/rejected": -3.3709475994110107,
+      "logps/chosen": -2.0988097190856934,
+      "logps/rejected": -2.3707680702209473,
+      "loss": 1.0705,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -2.0988097190856934,
+      "rewards/margins": 0.2719583809375763,
+      "rewards/rejected": -2.3707680702209473,
+      "step": 7870
+    },
+    {
+      "epoch": 1.3576843556168159,
+      "grad_norm": 12.107074987065694,
+      "learning_rate": 1.4132107072967165e-08,
+      "logits/chosen": -3.4137184619903564,
+      "logits/rejected": -3.40520977973938,
+      "logps/chosen": -2.0751662254333496,
+      "logps/rejected": -2.352374315261841,
+      "loss": 1.0592,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -2.0751662254333496,
+      "rewards/margins": 0.2772078514099121,
+      "rewards/rejected": -2.352374315261841,
+      "step": 7880
+    },
+    {
+      "epoch": 1.359407305306685,
+      "grad_norm": 13.667180881201768,
+      "learning_rate": 1.406445226128088e-08,
+      "logits/chosen": -3.3760993480682373,
+      "logits/rejected": -3.3663570880889893,
+      "logps/chosen": -2.0257906913757324,
+      "logps/rejected": -2.3393502235412598,
+      "loss": 1.0451,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -2.0257906913757324,
+      "rewards/margins": 0.31355950236320496,
+      "rewards/rejected": -2.3393502235412598,
+      "step": 7890
+    },
+    {
+      "epoch": 1.361130254996554,
+      "grad_norm": 11.775393940672888,
+      "learning_rate": 1.3996896340594791e-08,
+      "logits/chosen": -3.3611137866973877,
+      "logits/rejected": -3.359464645385742,
+      "logps/chosen": -2.0664868354797363,
+      "logps/rejected": -2.3155415058135986,
+      "loss": 1.0894,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -2.0664868354797363,
+      "rewards/margins": 0.249054953455925,
+      "rewards/rejected": -2.3155415058135986,
+      "step": 7900
+    },
+    {
+      "epoch": 1.361130254996554,
+      "eval_logits/chosen": -3.4538474082946777,
+      "eval_logits/rejected": -3.450190782546997,
+      "eval_logps/chosen": -1.8628642559051514,
+      "eval_logps/rejected": -2.0521364212036133,
+      "eval_loss": 1.0893720388412476,
+      "eval_rewards/accuracies": 0.6233736276626587,
+      "eval_rewards/chosen": -1.8628642559051514,
+      "eval_rewards/margins": 0.18927228450775146,
+      "eval_rewards/rejected": -2.0521364212036133,
+      "eval_runtime": 155.7113,
+      "eval_samples_per_second": 27.641,
+      "eval_steps_per_second": 3.455,
+      "step": 7900
+    },
+    {
+      "epoch": 1.3628532046864232,
+      "grad_norm": 12.231570642449816,
+      "learning_rate": 1.3929439921822333e-08,
+      "logits/chosen": -3.3726305961608887,
+      "logits/rejected": -3.3611557483673096,
+      "logps/chosen": -2.1079633235931396,
+      "logps/rejected": -2.315833330154419,
+      "loss": 1.1217,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -2.1079633235931396,
+      "rewards/margins": 0.20786967873573303,
+      "rewards/rejected": -2.315833330154419,
+      "step": 7910
+    },
+    {
+      "epoch": 1.3645761543762922,
+      "grad_norm": 15.388795752971403,
+      "learning_rate": 1.3862083614977067e-08,
+      "logits/chosen": -3.3755671977996826,
+      "logits/rejected": -3.362419843673706,
+      "logps/chosen": -2.0668320655822754,
+      "logps/rejected": -2.2979512214660645,
+      "loss": 1.0791,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -2.0668320655822754,
+      "rewards/margins": 0.2311190664768219,
+      "rewards/rejected": -2.2979512214660645,
+      "step": 7920
+    },
+    {
+      "epoch": 1.3662991040661612,
+      "grad_norm": 13.298291567700012,
+      "learning_rate": 1.3794828029167265e-08,
+      "logits/chosen": -3.402794361114502,
+      "logits/rejected": -3.3873488903045654,
+      "logps/chosen": -2.0839126110076904,
+      "logps/rejected": -2.3090410232543945,
+      "loss": 1.0873,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -2.0839126110076904,
+      "rewards/margins": 0.22512862086296082,
+      "rewards/rejected": -2.3090410232543945,
+      "step": 7930
+    },
+    {
+      "epoch": 1.3680220537560304,
+      "grad_norm": 13.671231479944101,
+      "learning_rate": 1.3727673772590375e-08,
+      "logits/chosen": -3.3569817543029785,
+      "logits/rejected": -3.347126007080078,
+      "logps/chosen": -2.1264572143554688,
+      "logps/rejected": -2.3750417232513428,
+      "loss": 1.0931,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -2.1264572143554688,
+      "rewards/margins": 0.24858446419239044,
+      "rewards/rejected": -2.3750417232513428,
+      "step": 7940
+    },
+    {
+      "epoch": 1.3697450034458993,
+      "grad_norm": 12.33508045194854,
+      "learning_rate": 1.3660621452527505e-08,
+      "logits/chosen": -3.32171630859375,
+      "logits/rejected": -3.3172764778137207,
+      "logps/chosen": -1.9911562204360962,
+      "logps/rejected": -2.3459177017211914,
+      "loss": 1.0178,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.9911562204360962,
+      "rewards/margins": 0.3547613024711609,
+      "rewards/rejected": -2.3459177017211914,
+      "step": 7950
+    },
+    {
+      "epoch": 1.3714679531357685,
+      "grad_norm": 12.235563160949052,
+      "learning_rate": 1.3593671675337953e-08,
+      "logits/chosen": -3.329850673675537,
+      "logits/rejected": -3.315883159637451,
+      "logps/chosen": -2.0762460231781006,
+      "logps/rejected": -2.394834041595459,
+      "loss": 1.0373,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -2.0762460231781006,
+      "rewards/margins": 0.3185880780220032,
+      "rewards/rejected": -2.394834041595459,
+      "step": 7960
+    },
+    {
+      "epoch": 1.3731909028256375,
+      "grad_norm": 13.602614478333411,
+      "learning_rate": 1.3526825046453705e-08,
+      "logits/chosen": -3.3907337188720703,
+      "logits/rejected": -3.3737196922302246,
+      "logps/chosen": -2.092822551727295,
+      "logps/rejected": -2.3950016498565674,
+      "loss": 1.0401,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -2.092822551727295,
+      "rewards/margins": 0.3021793067455292,
+      "rewards/rejected": -2.3950016498565674,
+      "step": 7970
+    },
+    {
+      "epoch": 1.3749138525155065,
+      "grad_norm": 13.407188452368832,
+      "learning_rate": 1.3460082170373988e-08,
+      "logits/chosen": -3.417320966720581,
+      "logits/rejected": -3.409909725189209,
+      "logps/chosen": -2.088026523590088,
+      "logps/rejected": -2.3272359371185303,
+      "loss": 1.0861,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -2.088026523590088,
+      "rewards/margins": 0.23920938372612,
+      "rewards/rejected": -2.3272359371185303,
+      "step": 7980
+    },
+    {
+      "epoch": 1.3766368022053757,
+      "grad_norm": 11.627480363764686,
+      "learning_rate": 1.339344365065973e-08,
+      "logits/chosen": -3.4348349571228027,
+      "logits/rejected": -3.4293010234832764,
+      "logps/chosen": -2.0565643310546875,
+      "logps/rejected": -2.3151767253875732,
+      "loss": 1.0763,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -2.0565643310546875,
+      "rewards/margins": 0.25861239433288574,
+      "rewards/rejected": -2.3151767253875732,
+      "step": 7990
+    },
+    {
+      "epoch": 1.3783597518952446,
+      "grad_norm": 13.852143793595387,
+      "learning_rate": 1.3326910089928244e-08,
+      "logits/chosen": -3.3484294414520264,
+      "logits/rejected": -3.3451263904571533,
+      "logps/chosen": -2.07145094871521,
+      "logps/rejected": -2.243041753768921,
+      "loss": 1.1202,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -2.07145094871521,
+      "rewards/margins": 0.17159108817577362,
+      "rewards/rejected": -2.243041753768921,
+      "step": 8000
+    },
+    {
+      "epoch": 1.3783597518952446,
+      "eval_logits/chosen": -3.4375693798065186,
+      "eval_logits/rejected": -3.433849573135376,
+      "eval_logps/chosen": -1.8667365312576294,
+      "eval_logps/rejected": -2.056304931640625,
+      "eval_loss": 1.0892691612243652,
+      "eval_rewards/accuracies": 0.6247676610946655,
+      "eval_rewards/chosen": -1.8667365312576294,
+      "eval_rewards/margins": 0.1895684450864792,
+      "eval_rewards/rejected": -2.056304931640625,
+      "eval_runtime": 155.6606,
+      "eval_samples_per_second": 27.65,
+      "eval_steps_per_second": 3.456,
+      "step": 8000
+    },
+    {
+      "epoch": 1.3800827015851138,
+      "grad_norm": 12.161380514787465,
+      "learning_rate": 1.3260482089847603e-08,
+      "logits/chosen": -3.3650143146514893,
+      "logits/rejected": -3.353980541229248,
+      "logps/chosen": -2.0687601566314697,
+      "logps/rejected": -2.2701828479766846,
+      "loss": 1.1104,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -2.0687601566314697,
+      "rewards/margins": 0.2014225721359253,
+      "rewards/rejected": -2.2701828479766846,
+      "step": 8010
+    },
+    {
+      "epoch": 1.3818056512749828,
+      "grad_norm": 13.121351061080412,
+      "learning_rate": 1.3194160251131364e-08,
+      "logits/chosen": -3.374310255050659,
+      "logits/rejected": -3.354067325592041,
+      "logps/chosen": -2.130094528198242,
+      "logps/rejected": -2.4190194606781006,
+      "loss": 1.059,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -2.130094528198242,
+      "rewards/margins": 0.2889249324798584,
+      "rewards/rejected": -2.4190194606781006,
+      "step": 8020
+    },
+    {
+      "epoch": 1.3835286009648518,
+      "grad_norm": 12.49018308827461,
+      "learning_rate": 1.3127945173532989e-08,
+      "logits/chosen": -3.3796706199645996,
+      "logits/rejected": -3.3725497722625732,
+      "logps/chosen": -2.053818941116333,
+      "logps/rejected": -2.3904242515563965,
+      "loss": 1.0339,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -2.053818941116333,
+      "rewards/margins": 0.33660537004470825,
+      "rewards/rejected": -2.3904242515563965,
+      "step": 8030
+    },
+    {
+      "epoch": 1.385251550654721,
+      "grad_norm": 11.25717770092257,
+      "learning_rate": 1.3061837455840539e-08,
+      "logits/chosen": -3.34409761428833,
+      "logits/rejected": -3.3287513256073,
+      "logps/chosen": -2.038478136062622,
+      "logps/rejected": -2.3026316165924072,
+      "loss": 1.0616,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -2.038478136062622,
+      "rewards/margins": 0.26415371894836426,
+      "rewards/rejected": -2.3026316165924072,
+      "step": 8040
+    },
+    {
+      "epoch": 1.38697450034459,
+      "grad_norm": 9.6938370535091,
+      "learning_rate": 1.2995837695871186e-08,
+      "logits/chosen": -3.393660068511963,
+      "logits/rejected": -3.3845458030700684,
+      "logps/chosen": -1.9736438989639282,
+      "logps/rejected": -2.2918851375579834,
+      "loss": 1.025,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.9736438989639282,
+      "rewards/margins": 0.31824153661727905,
+      "rewards/rejected": -2.2918851375579834,
+      "step": 8050
+    },
+    {
+      "epoch": 1.388697450034459,
+      "grad_norm": 12.101062858448415,
+      "learning_rate": 1.2929946490465854e-08,
+      "logits/chosen": -3.4165940284729004,
+      "logits/rejected": -3.3951199054718018,
+      "logps/chosen": -2.002747058868408,
+      "logps/rejected": -2.3990652561187744,
+      "loss": 0.9978,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -2.002747058868408,
+      "rewards/margins": 0.39631831645965576,
+      "rewards/rejected": -2.3990652561187744,
+      "step": 8060
+    },
+    {
+      "epoch": 1.390420399724328,
+      "grad_norm": 12.243496993270785,
+      "learning_rate": 1.2864164435483777e-08,
+      "logits/chosen": -3.362217664718628,
+      "logits/rejected": -3.346694231033325,
+      "logps/chosen": -2.065006732940674,
+      "logps/rejected": -2.3752176761627197,
+      "loss": 1.0672,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -2.065006732940674,
+      "rewards/margins": 0.3102104067802429,
+      "rewards/rejected": -2.3752176761627197,
+      "step": 8070
+    },
+    {
+      "epoch": 1.392143349414197,
+      "grad_norm": 14.67436309775157,
+      "learning_rate": 1.2798492125797144e-08,
+      "logits/chosen": -3.3524696826934814,
+      "logits/rejected": -3.349421262741089,
+      "logps/chosen": -2.0053021907806396,
+      "logps/rejected": -2.281238317489624,
+      "loss": 1.0503,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -2.0053021907806396,
+      "rewards/margins": 0.2759363055229187,
+      "rewards/rejected": -2.281238317489624,
+      "step": 8080
+    },
+    {
+      "epoch": 1.3938662991040662,
+      "grad_norm": 11.621101974165496,
+      "learning_rate": 1.273293015528571e-08,
+      "logits/chosen": -3.3479506969451904,
+      "logits/rejected": -3.332428455352783,
+      "logps/chosen": -2.0262722969055176,
+      "logps/rejected": -2.397275686264038,
+      "loss": 1.0013,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -2.0262722969055176,
+      "rewards/margins": 0.37100356817245483,
+      "rewards/rejected": -2.397275686264038,
+      "step": 8090
+    },
+    {
+      "epoch": 1.3955892487939352,
+      "grad_norm": 15.546931466624486,
+      "learning_rate": 1.2667479116831437e-08,
+      "logits/chosen": -3.370763063430786,
+      "logits/rejected": -3.3667426109313965,
+      "logps/chosen": -2.0073976516723633,
+      "logps/rejected": -2.251396894454956,
+      "loss": 1.0709,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -2.0073976516723633,
+      "rewards/margins": 0.2439991682767868,
+      "rewards/rejected": -2.251396894454956,
+      "step": 8100
+    },
+    {
+      "epoch": 1.3955892487939352,
+      "eval_logits/chosen": -3.4319071769714355,
+      "eval_logits/rejected": -3.4281728267669678,
+      "eval_logps/chosen": -1.8691883087158203,
+      "eval_logps/rejected": -2.059539318084717,
+      "eval_loss": 1.0889434814453125,
+      "eval_rewards/accuracies": 0.6243029832839966,
+      "eval_rewards/chosen": -1.8691883087158203,
+      "eval_rewards/margins": 0.19035089015960693,
+      "eval_rewards/rejected": -2.059539318084717,
+      "eval_runtime": 155.8805,
+      "eval_samples_per_second": 27.611,
+      "eval_steps_per_second": 3.451,
+      "step": 8100
+    },
+    {
+      "epoch": 1.3973121984838044,
+      "grad_norm": 11.735104438467467,
+      "learning_rate": 1.2602139602313067e-08,
+      "logits/chosen": -3.3940796852111816,
+      "logits/rejected": -3.374265193939209,
+      "logps/chosen": -2.065246105194092,
+      "logps/rejected": -2.367819309234619,
+      "loss": 1.0326,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -2.065246105194092,
+      "rewards/margins": 0.30257314443588257,
+      "rewards/rejected": -2.367819309234619,
+      "step": 8110
+    },
+    {
+      "epoch": 1.3990351481736734,
+      "grad_norm": 12.05031156078595,
+      "learning_rate": 1.2536912202600907e-08,
+      "logits/chosen": -3.351496458053589,
+      "logits/rejected": -3.3434557914733887,
+      "logps/chosen": -2.017407178878784,
+      "logps/rejected": -2.25342059135437,
+      "loss": 1.0961,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -2.017407178878784,
+      "rewards/margins": 0.23601368069648743,
+      "rewards/rejected": -2.25342059135437,
+      "step": 8120
+    },
+    {
+      "epoch": 1.4007580978635423,
+      "grad_norm": 12.559952153133823,
+      "learning_rate": 1.2471797507551324e-08,
+      "logits/chosen": -3.364426374435425,
+      "logits/rejected": -3.3550734519958496,
+      "logps/chosen": -2.0090737342834473,
+      "logps/rejected": -2.2542872428894043,
+      "loss": 1.0746,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -2.0090737342834473,
+      "rewards/margins": 0.24521362781524658,
+      "rewards/rejected": -2.2542872428894043,
+      "step": 8130
+    },
+    {
+      "epoch": 1.4024810475534115,
+      "grad_norm": 12.666309613740193,
+      "learning_rate": 1.2406796106001527e-08,
+      "logits/chosen": -3.3460984230041504,
+      "logits/rejected": -3.3320090770721436,
+      "logps/chosen": -2.035346746444702,
+      "logps/rejected": -2.3807053565979004,
+      "loss": 1.0299,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -2.035346746444702,
+      "rewards/margins": 0.3453586995601654,
+      "rewards/rejected": -2.3807053565979004,
+      "step": 8140
+    },
+    {
+      "epoch": 1.4042039972432805,
+      "grad_norm": 13.323755185469777,
+      "learning_rate": 1.2341908585764196e-08,
+      "logits/chosen": -3.3948769569396973,
+      "logits/rejected": -3.38409161567688,
+      "logps/chosen": -2.051933765411377,
+      "logps/rejected": -2.381584644317627,
+      "loss": 1.0359,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -2.051933765411377,
+      "rewards/margins": 0.32965072989463806,
+      "rewards/rejected": -2.381584644317627,
+      "step": 8150
+    },
+    {
+      "epoch": 1.4059269469331497,
+      "grad_norm": 11.878510666831797,
+      "learning_rate": 1.2277135533622174e-08,
+      "logits/chosen": -3.3630568981170654,
+      "logits/rejected": -3.3532302379608154,
+      "logps/chosen": -2.1431140899658203,
+      "logps/rejected": -2.40252423286438,
+      "loss": 1.0614,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -2.1431140899658203,
+      "rewards/margins": 0.25940996408462524,
+      "rewards/rejected": -2.40252423286438,
+      "step": 8160
+    },
+    {
+      "epoch": 1.4076498966230186,
+      "grad_norm": 16.290921293894286,
+      "learning_rate": 1.2212477535323157e-08,
+      "logits/chosen": -3.388526201248169,
+      "logits/rejected": -3.3769912719726562,
+      "logps/chosen": -2.1270620822906494,
+      "logps/rejected": -2.20481014251709,
+      "loss": 1.189,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -2.1270620822906494,
+      "rewards/margins": 0.077748142182827,
+      "rewards/rejected": -2.20481014251709,
+      "step": 8170
+    },
+    {
+      "epoch": 1.4093728463128876,
+      "grad_norm": 16.274010686393957,
+      "learning_rate": 1.2147935175574404e-08,
+      "logits/chosen": -3.380419969558716,
+      "logits/rejected": -3.3695578575134277,
+      "logps/chosen": -2.068319797515869,
+      "logps/rejected": -2.3133316040039062,
+      "loss": 1.0817,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -2.068319797515869,
+      "rewards/margins": 0.24501188099384308,
+      "rewards/rejected": -2.3133316040039062,
+      "step": 8180
+    },
+    {
+      "epoch": 1.4110957960027566,
+      "grad_norm": 11.686106236981065,
+      "learning_rate": 1.208350903803745e-08,
+      "logits/chosen": -3.3727073669433594,
+      "logits/rejected": -3.3623054027557373,
+      "logps/chosen": -2.0284981727600098,
+      "logps/rejected": -2.278683662414551,
+      "loss": 1.0728,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -2.0284981727600098,
+      "rewards/margins": 0.2501857578754425,
+      "rewards/rejected": -2.278683662414551,
+      "step": 8190
+    },
+    {
+      "epoch": 1.4128187456926258,
+      "grad_norm": 11.472734297723765,
+      "learning_rate": 1.2019199705322794e-08,
+      "logits/chosen": -3.3855674266815186,
+      "logits/rejected": -3.3670318126678467,
+      "logps/chosen": -2.0394604206085205,
+      "logps/rejected": -2.449223279953003,
+      "loss": 0.9842,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -2.0394604206085205,
+      "rewards/margins": 0.4097629189491272,
+      "rewards/rejected": -2.449223279953003,
+      "step": 8200
+    },
+    {
+      "epoch": 1.4128187456926258,
+      "eval_logits/chosen": -3.442497730255127,
+      "eval_logits/rejected": -3.438816547393799,
+      "eval_logps/chosen": -1.8731565475463867,
+      "eval_logps/rejected": -2.064129590988159,
+      "eval_loss": 1.0887478590011597,
+      "eval_rewards/accuracies": 0.622444212436676,
+      "eval_rewards/chosen": -1.8731565475463867,
+      "eval_rewards/margins": 0.19097290933132172,
+      "eval_rewards/rejected": -2.064129590988159,
+      "eval_runtime": 155.8806,
+      "eval_samples_per_second": 27.611,
+      "eval_steps_per_second": 3.451,
+      "step": 8200
+    },
+    {
+      "epoch": 1.414541695382495,
+      "grad_norm": 11.219408334828461,
+      "learning_rate": 1.1955007758984717e-08,
+      "logits/chosen": -3.307590961456299,
+      "logits/rejected": -3.2985050678253174,
+      "logps/chosen": -2.0896096229553223,
+      "logps/rejected": -2.4162516593933105,
+      "loss": 1.0505,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -2.0896096229553223,
+      "rewards/margins": 0.3266420364379883,
+      "rewards/rejected": -2.4162516593933105,
+      "step": 8210
+    },
+    {
+      "epoch": 1.416264645072364,
+      "grad_norm": 10.997806335813893,
+      "learning_rate": 1.1890933779515897e-08,
+      "logits/chosen": -3.3932647705078125,
+      "logits/rejected": -3.3785786628723145,
+      "logps/chosen": -2.0589725971221924,
+      "logps/rejected": -2.3382725715637207,
+      "loss": 1.06,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -2.0589725971221924,
+      "rewards/margins": 0.2793000638484955,
+      "rewards/rejected": -2.3382725715637207,
+      "step": 8220
+    },
+    {
+      "epoch": 1.417987594762233,
+      "grad_norm": 12.044421805858212,
+      "learning_rate": 1.1826978346342301e-08,
+      "logits/chosen": -3.363684892654419,
+      "logits/rejected": -3.350675582885742,
+      "logps/chosen": -2.0559122562408447,
+      "logps/rejected": -2.331371545791626,
+      "loss": 1.0641,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -2.0559122562408447,
+      "rewards/margins": 0.275459349155426,
+      "rewards/rejected": -2.331371545791626,
+      "step": 8230
+    },
+    {
+      "epoch": 1.4197105444521019,
+      "grad_norm": 13.002099488259834,
+      "learning_rate": 1.1763142037817806e-08,
+      "logits/chosen": -3.4280059337615967,
+      "logits/rejected": -3.4126789569854736,
+      "logps/chosen": -2.0701115131378174,
+      "logps/rejected": -2.3126039505004883,
+      "loss": 1.0715,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -2.0701115131378174,
+      "rewards/margins": 0.24249252676963806,
+      "rewards/rejected": -2.3126039505004883,
+      "step": 8240
+    },
+    {
+      "epoch": 1.421433494141971,
+      "grad_norm": 11.219435575870083,
+      "learning_rate": 1.169942543121908e-08,
+      "logits/chosen": -3.373795986175537,
+      "logits/rejected": -3.3604769706726074,
+      "logps/chosen": -2.035872220993042,
+      "logps/rejected": -2.3567287921905518,
+      "loss": 1.0324,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -2.035872220993042,
+      "rewards/margins": 0.3208564519882202,
+      "rewards/rejected": -2.3567287921905518,
+      "step": 8250
+    },
+    {
+      "epoch": 1.42315644383184,
+      "grad_norm": 12.827987928019912,
+      "learning_rate": 1.1635829102740293e-08,
+      "logits/chosen": -3.4371085166931152,
+      "logits/rejected": -3.4266650676727295,
+      "logps/chosen": -2.00901460647583,
+      "logps/rejected": -2.2803454399108887,
+      "loss": 1.0509,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -2.00901460647583,
+      "rewards/margins": 0.27133092284202576,
+      "rewards/rejected": -2.2803454399108887,
+      "step": 8260
+    },
+    {
+      "epoch": 1.4248793935217092,
+      "grad_norm": 12.613495945892753,
+      "learning_rate": 1.1572353627487949e-08,
+      "logits/chosen": -3.4183337688446045,
+      "logits/rejected": -3.4136581420898438,
+      "logps/chosen": -2.078277587890625,
+      "logps/rejected": -2.295870065689087,
+      "loss": 1.1087,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -2.078277587890625,
+      "rewards/margins": 0.21759271621704102,
+      "rewards/rejected": -2.295870065689087,
+      "step": 8270
+    },
+    {
+      "epoch": 1.4266023432115782,
+      "grad_norm": 11.734665691181565,
+      "learning_rate": 1.1508999579475653e-08,
+      "logits/chosen": -3.3644909858703613,
+      "logits/rejected": -3.3588931560516357,
+      "logps/chosen": -2.132209300994873,
+      "logps/rejected": -2.412349224090576,
+      "loss": 1.0611,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -2.132209300994873,
+      "rewards/margins": 0.28013983368873596,
+      "rewards/rejected": -2.412349224090576,
+      "step": 8280
+    },
+    {
+      "epoch": 1.4283252929014472,
+      "grad_norm": 11.015188110822697,
+      "learning_rate": 1.1445767531618943e-08,
+      "logits/chosen": -3.3591442108154297,
+      "logits/rejected": -3.334130048751831,
+      "logps/chosen": -2.053840398788452,
+      "logps/rejected": -2.3265538215637207,
+      "loss": 1.0541,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -2.053840398788452,
+      "rewards/margins": 0.27271372079849243,
+      "rewards/rejected": -2.3265538215637207,
+      "step": 8290
+    },
+    {
+      "epoch": 1.4300482425913164,
+      "grad_norm": 11.346533655650259,
+      "learning_rate": 1.1382658055730096e-08,
+      "logits/chosen": -3.4501845836639404,
+      "logits/rejected": -3.4383513927459717,
+      "logps/chosen": -2.1134629249572754,
+      "logps/rejected": -2.359807252883911,
+      "loss": 1.0825,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -2.1134629249572754,
+      "rewards/margins": 0.24634405970573425,
+      "rewards/rejected": -2.359807252883911,
+      "step": 8300
+    },
+    {
+      "epoch": 1.4300482425913164,
+      "eval_logits/chosen": -3.448821783065796,
+      "eval_logits/rejected": -3.4451801776885986,
+      "eval_logps/chosen": -1.8771284818649292,
+      "eval_logps/rejected": -2.068108081817627,
+      "eval_loss": 1.0887627601623535,
+      "eval_rewards/accuracies": 0.6243029832839966,
+      "eval_rewards/chosen": -1.8771284818649292,
+      "eval_rewards/margins": 0.19097968935966492,
+      "eval_rewards/rejected": -2.068108081817627,
+      "eval_runtime": 155.7524,
+      "eval_samples_per_second": 27.634,
+      "eval_steps_per_second": 3.454,
+      "step": 8300
+    },
+    {
+      "epoch": 1.4317711922811853,
+      "grad_norm": 13.574360569414766,
+      "learning_rate": 1.1319671722512957e-08,
+      "logits/chosen": -3.3108773231506348,
+      "logits/rejected": -3.294097423553467,
+      "logps/chosen": -2.054011821746826,
+      "logps/rejected": -2.3131766319274902,
+      "loss": 1.0657,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -2.054011821746826,
+      "rewards/margins": 0.25916463136672974,
+      "rewards/rejected": -2.3131766319274902,
+      "step": 8310
+    },
+    {
+      "epoch": 1.4334941419710545,
+      "grad_norm": 10.957933555542908,
+      "learning_rate": 1.1256809101557793e-08,
+      "logits/chosen": -3.3721423149108887,
+      "logits/rejected": -3.362767457962036,
+      "logps/chosen": -2.008420705795288,
+      "logps/rejected": -2.3438870906829834,
+      "loss": 1.0191,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -2.008420705795288,
+      "rewards/margins": 0.33546629548072815,
+      "rewards/rejected": -2.3438870906829834,
+      "step": 8320
+    },
+    {
+      "epoch": 1.4352170916609235,
+      "grad_norm": 14.594233216407654,
+      "learning_rate": 1.1194070761336133e-08,
+      "logits/chosen": -3.3672614097595215,
+      "logits/rejected": -3.362370729446411,
+      "logps/chosen": -2.081789255142212,
+      "logps/rejected": -2.4404497146606445,
+      "loss": 1.021,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -2.081789255142212,
+      "rewards/margins": 0.3586602210998535,
+      "rewards/rejected": -2.4404497146606445,
+      "step": 8330
+    },
+    {
+      "epoch": 1.4369400413507925,
+      "grad_norm": 11.91339853457774,
+      "learning_rate": 1.11314572691956e-08,
+      "logits/chosen": -3.4118428230285645,
+      "logits/rejected": -3.4033894538879395,
+      "logps/chosen": -2.113788366317749,
+      "logps/rejected": -2.313765525817871,
+      "loss": 1.1092,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -2.113788366317749,
+      "rewards/margins": 0.19997724890708923,
+      "rewards/rejected": -2.313765525817871,
+      "step": 8340
+    },
+    {
+      "epoch": 1.4386629910406616,
+      "grad_norm": 13.247957996119705,
+      "learning_rate": 1.106896919135483e-08,
+      "logits/chosen": -3.317502498626709,
+      "logits/rejected": -3.3086349964141846,
+      "logps/chosen": -2.1254451274871826,
+      "logps/rejected": -2.3736376762390137,
+      "loss": 1.0795,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -2.1254451274871826,
+      "rewards/margins": 0.2481924593448639,
+      "rewards/rejected": -2.3736376762390137,
+      "step": 8350
+    },
+    {
+      "epoch": 1.4403859407305306,
+      "grad_norm": 12.508500289289557,
+      "learning_rate": 1.1006607092898326e-08,
+      "logits/chosen": -3.317509174346924,
+      "logits/rejected": -3.2985329627990723,
+      "logps/chosen": -2.0001673698425293,
+      "logps/rejected": -2.381077289581299,
+      "loss": 1.0059,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -2.0001673698425293,
+      "rewards/margins": 0.3809099793434143,
+      "rewards/rejected": -2.381077289581299,
+      "step": 8360
+    },
+    {
+      "epoch": 1.4421088904203998,
+      "grad_norm": 12.565554940102212,
+      "learning_rate": 1.0944371537771346e-08,
+      "logits/chosen": -3.3506176471710205,
+      "logits/rejected": -3.3409454822540283,
+      "logps/chosen": -2.0602593421936035,
+      "logps/rejected": -2.415473461151123,
+      "loss": 1.0426,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -2.0602593421936035,
+      "rewards/margins": 0.35521399974823,
+      "rewards/rejected": -2.415473461151123,
+      "step": 8370
+    },
+    {
+      "epoch": 1.4438318401102688,
+      "grad_norm": 11.162932935910286,
+      "learning_rate": 1.0882263088774809e-08,
+      "logits/chosen": -3.4354941844940186,
+      "logits/rejected": -3.4261772632598877,
+      "logps/chosen": -2.039003610610962,
+      "logps/rejected": -2.3921115398406982,
+      "loss": 1.021,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -2.039003610610962,
+      "rewards/margins": 0.3531082570552826,
+      "rewards/rejected": -2.3921115398406982,
+      "step": 8380
+    },
+    {
+      "epoch": 1.4455547898001377,
+      "grad_norm": 13.273737425835273,
+      "learning_rate": 1.0820282307560197e-08,
+      "logits/chosen": -3.3984875679016113,
+      "logits/rejected": -3.382511615753174,
+      "logps/chosen": -2.070009469985962,
+      "logps/rejected": -2.4886841773986816,
+      "loss": 0.9686,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -2.070009469985962,
+      "rewards/margins": 0.4186745584011078,
+      "rewards/rejected": -2.4886841773986816,
+      "step": 8390
+    },
+    {
+      "epoch": 1.447277739490007,
+      "grad_norm": 11.515157036014347,
+      "learning_rate": 1.075842975462449e-08,
+      "logits/chosen": -3.382897138595581,
+      "logits/rejected": -3.3711276054382324,
+      "logps/chosen": -1.9906479120254517,
+      "logps/rejected": -2.278052806854248,
+      "loss": 1.0353,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.9906479120254517,
+      "rewards/margins": 0.2874049246311188,
+      "rewards/rejected": -2.278052806854248,
+      "step": 8400
+    },
+    {
+      "epoch": 1.447277739490007,
+      "eval_logits/chosen": -3.4438095092773438,
+      "eval_logits/rejected": -3.440155029296875,
+      "eval_logps/chosen": -1.8814116716384888,
+      "eval_logps/rejected": -2.072943925857544,
+      "eval_loss": 1.088468074798584,
+      "eval_rewards/accuracies": 0.6247676610946655,
+      "eval_rewards/chosen": -1.8814116716384888,
+      "eval_rewards/margins": 0.1915321946144104,
+      "eval_rewards/rejected": -2.072943925857544,
+      "eval_runtime": 155.7774,
+      "eval_samples_per_second": 27.629,
+      "eval_steps_per_second": 3.454,
+      "step": 8400
+    },
+    {
+      "epoch": 1.449000689179876,
+      "grad_norm": 12.427640624105056,
+      "learning_rate": 1.0696705989305086e-08,
+      "logits/chosen": -3.351097822189331,
+      "logits/rejected": -3.3312830924987793,
+      "logps/chosen": -2.146123170852661,
+      "logps/rejected": -2.5502870082855225,
+      "loss": 1.0031,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -2.146123170852661,
+      "rewards/margins": 0.40416401624679565,
+      "rewards/rejected": -2.5502870082855225,
+      "step": 8410
+    },
+    {
+      "epoch": 1.450723638869745,
+      "grad_norm": 11.408712146986648,
+      "learning_rate": 1.0635111569774754e-08,
+      "logits/chosen": -3.291322708129883,
+      "logits/rejected": -3.284122943878174,
+      "logps/chosen": -2.062080144882202,
+      "logps/rejected": -2.305058717727661,
+      "loss": 1.0869,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -2.062080144882202,
+      "rewards/margins": 0.2429783046245575,
+      "rewards/rejected": -2.305058717727661,
+      "step": 8420
+    },
+    {
+      "epoch": 1.452446588559614,
+      "grad_norm": 10.30151229471276,
+      "learning_rate": 1.0573647053036552e-08,
+      "logits/chosen": -3.369002103805542,
+      "logits/rejected": -3.3611788749694824,
+      "logps/chosen": -2.0615086555480957,
+      "logps/rejected": -2.3830339908599854,
+      "loss": 1.0479,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -2.0615086555480957,
+      "rewards/margins": 0.32152530550956726,
+      "rewards/rejected": -2.3830339908599854,
+      "step": 8430
+    },
+    {
+      "epoch": 1.454169538249483,
+      "grad_norm": 13.924041188277473,
+      "learning_rate": 1.0512312994918865e-08,
+      "logits/chosen": -3.387824535369873,
+      "logits/rejected": -3.3778159618377686,
+      "logps/chosen": -2.106511354446411,
+      "logps/rejected": -2.3884243965148926,
+      "loss": 1.0597,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -2.106511354446411,
+      "rewards/margins": 0.28191301226615906,
+      "rewards/rejected": -2.3884243965148926,
+      "step": 8440
+    },
+    {
+      "epoch": 1.4558924879393522,
+      "grad_norm": 11.721977207727877,
+      "learning_rate": 1.0451109950070276e-08,
+      "logits/chosen": -3.322500705718994,
+      "logits/rejected": -3.318852663040161,
+      "logps/chosen": -1.9902114868164062,
+      "logps/rejected": -2.294934034347534,
+      "loss": 1.0525,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.9902114868164062,
+      "rewards/margins": 0.3047229051589966,
+      "rewards/rejected": -2.294934034347534,
+      "step": 8450
+    },
+    {
+      "epoch": 1.4576154376292212,
+      "grad_norm": 11.959113273919465,
+      "learning_rate": 1.039003847195466e-08,
+      "logits/chosen": -3.3849074840545654,
+      "logits/rejected": -3.370879650115967,
+      "logps/chosen": -2.038325071334839,
+      "logps/rejected": -2.464848041534424,
+      "loss": 0.9648,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -2.038325071334839,
+      "rewards/margins": 0.4265229105949402,
+      "rewards/rejected": -2.464848041534424,
+      "step": 8460
+    },
+    {
+      "epoch": 1.4593383873190904,
+      "grad_norm": 15.200052004637696,
+      "learning_rate": 1.0329099112846071e-08,
+      "logits/chosen": -3.3641295433044434,
+      "logits/rejected": -3.3485450744628906,
+      "logps/chosen": -2.1571261882781982,
+      "logps/rejected": -2.4499876499176025,
+      "loss": 1.0625,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -2.1571261882781982,
+      "rewards/margins": 0.2928614020347595,
+      "rewards/rejected": -2.4499876499176025,
+      "step": 8470
+    },
+    {
+      "epoch": 1.4610613370089593,
+      "grad_norm": 12.569584530206996,
+      "learning_rate": 1.0268292423823838e-08,
+      "logits/chosen": -3.3729751110076904,
+      "logits/rejected": -3.3575923442840576,
+      "logps/chosen": -2.061161756515503,
+      "logps/rejected": -2.372161388397217,
+      "loss": 1.051,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -2.061161756515503,
+      "rewards/margins": 0.3109992444515228,
+      "rewards/rejected": -2.372161388397217,
+      "step": 8480
+    },
+    {
+      "epoch": 1.4627842866988283,
+      "grad_norm": 11.575036578693853,
+      "learning_rate": 1.020761895476753e-08,
+      "logits/chosen": -3.3922972679138184,
+      "logits/rejected": -3.3881709575653076,
+      "logps/chosen": -2.1499686241149902,
+      "logps/rejected": -2.3961117267608643,
+      "loss": 1.0903,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -2.1499686241149902,
+      "rewards/margins": 0.24614329636096954,
+      "rewards/rejected": -2.3961117267608643,
+      "step": 8490
+    },
+    {
+      "epoch": 1.4645072363886975,
+      "grad_norm": 11.144482424488238,
+      "learning_rate": 1.0147079254352e-08,
+      "logits/chosen": -3.321580171585083,
+      "logits/rejected": -3.3132076263427734,
+      "logps/chosen": -2.0944581031799316,
+      "logps/rejected": -2.400571823120117,
+      "loss": 1.0484,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -2.0944581031799316,
+      "rewards/margins": 0.3061137795448303,
+      "rewards/rejected": -2.400571823120117,
+      "step": 8500
+    },
+    {
+      "epoch": 1.4645072363886975,
+      "eval_logits/chosen": -3.4414682388305664,
+      "eval_logits/rejected": -3.437810182571411,
+      "eval_logps/chosen": -1.8808684349060059,
+      "eval_logps/rejected": -2.0725314617156982,
+      "eval_loss": 1.088517665863037,
+      "eval_rewards/accuracies": 0.6233736276626587,
+      "eval_rewards/chosen": -1.8808684349060059,
+      "eval_rewards/margins": 0.19166330993175507,
+      "eval_rewards/rejected": -2.0725314617156982,
+      "eval_runtime": 155.6792,
+      "eval_samples_per_second": 27.647,
+      "eval_steps_per_second": 3.456,
+      "step": 8500
+    },
+    {
+      "epoch": 1.4662301860785665,
+      "grad_norm": 14.492165279248525,
+      "learning_rate": 1.008667387004242e-08,
+      "logits/chosen": -3.3637588024139404,
+      "logits/rejected": -3.344637632369995,
+      "logps/chosen": -2.099735975265503,
+      "logps/rejected": -2.444492816925049,
+      "loss": 1.0562,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -2.099735975265503,
+      "rewards/margins": 0.34475669264793396,
+      "rewards/rejected": -2.444492816925049,
+      "step": 8510
+    },
+    {
+      "epoch": 1.4679531357684357,
+      "grad_norm": 11.685358767543136,
+      "learning_rate": 1.0026403348089329e-08,
+      "logits/chosen": -3.357090473175049,
+      "logits/rejected": -3.337810516357422,
+      "logps/chosen": -2.0437188148498535,
+      "logps/rejected": -2.4609029293060303,
+      "loss": 0.9944,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -2.0437188148498535,
+      "rewards/margins": 0.4171839654445648,
+      "rewards/rejected": -2.4609029293060303,
+      "step": 8520
+    },
+    {
+      "epoch": 1.4696760854583046,
+      "grad_norm": 13.969339851753631,
+      "learning_rate": 9.9662682335237e-09,
+      "logits/chosen": -3.3490490913391113,
+      "logits/rejected": -3.3403236865997314,
+      "logps/chosen": -2.106513500213623,
+      "logps/rejected": -2.3427011966705322,
+      "loss": 1.0893,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -2.106513500213623,
+      "rewards/margins": 0.23618781566619873,
+      "rewards/rejected": -2.3427011966705322,
+      "step": 8530
+    },
+    {
+      "epoch": 1.4713990351481736,
+      "grad_norm": 12.032000907408166,
+      "learning_rate": 9.906269070152004e-09,
+      "logits/chosen": -3.4222006797790527,
+      "logits/rejected": -3.414266586303711,
+      "logps/chosen": -2.0552127361297607,
+      "logps/rejected": -2.3353943824768066,
+      "loss": 1.0452,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -2.0552127361297607,
+      "rewards/margins": 0.2801814377307892,
+      "rewards/rejected": -2.3353943824768066,
+      "step": 8540
+    },
+    {
+      "epoch": 1.4731219848380428,
+      "grad_norm": 12.840297571329993,
+      "learning_rate": 9.846406400551307e-09,
+      "logits/chosen": -3.3838303089141846,
+      "logits/rejected": -3.3799242973327637,
+      "logps/chosen": -2.081017017364502,
+      "logps/rejected": -2.320730447769165,
+      "loss": 1.0926,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -2.081017017364502,
+      "rewards/margins": 0.2397136688232422,
+      "rewards/rejected": -2.320730447769165,
+      "step": 8550
+    },
+    {
+      "epoch": 1.4748449345279118,
+      "grad_norm": 11.87122056652336,
+      "learning_rate": 9.786680766064318e-09,
+      "logits/chosen": -3.4370319843292236,
+      "logits/rejected": -3.4256629943847656,
+      "logps/chosen": -2.046665906906128,
+      "logps/rejected": -2.3277406692504883,
+      "loss": 1.059,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -2.046665906906128,
+      "rewards/margins": 0.2810743451118469,
+      "rewards/rejected": -2.3277406692504883,
+      "step": 8560
+    },
+    {
+      "epoch": 1.476567884217781,
+      "grad_norm": 13.311647303438166,
+      "learning_rate": 9.727092706794554e-09,
+      "logits/chosen": -3.3504669666290283,
+      "logits/rejected": -3.3371729850769043,
+      "logps/chosen": -2.081444025039673,
+      "logps/rejected": -2.40982723236084,
+      "loss": 1.0305,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -2.081444025039673,
+      "rewards/margins": 0.3283833861351013,
+      "rewards/rejected": -2.40982723236084,
+      "step": 8570
+    },
+    {
+      "epoch": 1.47829083390765,
+      "grad_norm": 13.210670054247007,
+      "learning_rate": 9.667642761601433e-09,
+      "logits/chosen": -3.4010658264160156,
+      "logits/rejected": -3.389624834060669,
+      "logps/chosen": -1.9877710342407227,
+      "logps/rejected": -2.3411388397216797,
+      "loss": 1.0102,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.9877710342407227,
+      "rewards/margins": 0.35336780548095703,
+      "rewards/rejected": -2.3411388397216797,
+      "step": 8580
+    },
+    {
+      "epoch": 1.480013783597519,
+      "grad_norm": 13.909907033806679,
+      "learning_rate": 9.608331468095376e-09,
+      "logits/chosen": -3.4140541553497314,
+      "logits/rejected": -3.399421215057373,
+      "logps/chosen": -2.0175297260284424,
+      "logps/rejected": -2.3428289890289307,
+      "loss": 1.022,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -2.0175297260284424,
+      "rewards/margins": 0.32529929280281067,
+      "rewards/rejected": -2.3428289890289307,
+      "step": 8590
+    },
+    {
+      "epoch": 1.481736733287388,
+      "grad_norm": 11.462907941240882,
+      "learning_rate": 9.549159362632986e-09,
+      "logits/chosen": -3.3536458015441895,
+      "logits/rejected": -3.337233304977417,
+      "logps/chosen": -2.0623345375061035,
+      "logps/rejected": -2.3542869091033936,
+      "loss": 1.0415,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -2.0623345375061035,
+      "rewards/margins": 0.29195231199264526,
+      "rewards/rejected": -2.3542869091033936,
+      "step": 8600
+    },
+    {
+      "epoch": 1.481736733287388,
+      "eval_logits/chosen": -3.447143793106079,
+      "eval_logits/rejected": -3.443513870239258,
+      "eval_logps/chosen": -1.8834823369979858,
+      "eval_logps/rejected": -2.07529354095459,
+      "eval_loss": 1.0885599851608276,
+      "eval_rewards/accuracies": 0.6238383054733276,
+      "eval_rewards/chosen": -1.8834823369979858,
+      "eval_rewards/margins": 0.19181111454963684,
+      "eval_rewards/rejected": -2.07529354095459,
+      "eval_runtime": 155.7999,
+      "eval_samples_per_second": 27.625,
+      "eval_steps_per_second": 3.453,
+      "step": 8600
+    },
+    {
+      "epoch": 1.483459682977257,
+      "grad_norm": 13.383291682637786,
+      "learning_rate": 9.490126980312165e-09,
+      "logits/chosen": -3.3678641319274902,
+      "logits/rejected": -3.3549304008483887,
+      "logps/chosen": -2.064180850982666,
+      "logps/rejected": -2.3028016090393066,
+      "loss": 1.0952,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -2.064180850982666,
+      "rewards/margins": 0.23862037062644958,
+      "rewards/rejected": -2.3028016090393066,
+      "step": 8610
+    },
+    {
+      "epoch": 1.4851826326671262,
+      "grad_norm": 12.631719169373927,
+      "learning_rate": 9.43123485496729e-09,
+      "logits/chosen": -3.322261095046997,
+      "logits/rejected": -3.3115105628967285,
+      "logps/chosen": -2.114708423614502,
+      "logps/rejected": -2.353480815887451,
+      "loss": 1.0823,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -2.114708423614502,
+      "rewards/margins": 0.23877234756946564,
+      "rewards/rejected": -2.353480815887451,
+      "step": 8620
+    },
+    {
+      "epoch": 1.4869055823569952,
+      "grad_norm": 13.940561216617615,
+      "learning_rate": 9.372483519164398e-09,
+      "logits/chosen": -3.306807279586792,
+      "logits/rejected": -3.2987430095672607,
+      "logps/chosen": -2.0118460655212402,
+      "logps/rejected": -2.2760584354400635,
+      "loss": 1.0592,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -2.0118460655212402,
+      "rewards/margins": 0.26421231031417847,
+      "rewards/rejected": -2.2760584354400635,
+      "step": 8630
+    },
+    {
+      "epoch": 1.4886285320468642,
+      "grad_norm": 12.021623957753814,
+      "learning_rate": 9.313873504196313e-09,
+      "logits/chosen": -3.3927574157714844,
+      "logits/rejected": -3.3777518272399902,
+      "logps/chosen": -2.041154146194458,
+      "logps/rejected": -2.365455150604248,
+      "loss": 1.026,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -2.041154146194458,
+      "rewards/margins": 0.3243010640144348,
+      "rewards/rejected": -2.365455150604248,
+      "step": 8640
+    },
+    {
+      "epoch": 1.4903514817367332,
+      "grad_norm": 13.065327227427161,
+      "learning_rate": 9.255405340077949e-09,
+      "logits/chosen": -3.344301700592041,
+      "logits/rejected": -3.3337531089782715,
+      "logps/chosen": -2.0510551929473877,
+      "logps/rejected": -2.361353874206543,
+      "loss": 1.0423,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -2.0510551929473877,
+      "rewards/margins": 0.31029850244522095,
+      "rewards/rejected": -2.361353874206543,
+      "step": 8650
+    },
+    {
+      "epoch": 1.4920744314266023,
+      "grad_norm": 13.583118055542416,
+      "learning_rate": 9.197079555541378e-09,
+      "logits/chosen": -3.350508451461792,
+      "logits/rejected": -3.343189239501953,
+      "logps/chosen": -2.0512712001800537,
+      "logps/rejected": -2.332828998565674,
+      "loss": 1.0749,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -2.0512712001800537,
+      "rewards/margins": 0.28155824542045593,
+      "rewards/rejected": -2.332828998565674,
+      "step": 8660
+    },
+    {
+      "epoch": 1.4937973811164715,
+      "grad_norm": 11.828195347052262,
+      "learning_rate": 9.138896678031201e-09,
+      "logits/chosen": -3.4106898307800293,
+      "logits/rejected": -3.4005649089813232,
+      "logps/chosen": -2.1228442192077637,
+      "logps/rejected": -2.4533400535583496,
+      "loss": 1.0275,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -2.1228442192077637,
+      "rewards/margins": 0.33049583435058594,
+      "rewards/rejected": -2.4533400535583496,
+      "step": 8670
+    },
+    {
+      "epoch": 1.4955203308063405,
+      "grad_norm": 12.670453978083687,
+      "learning_rate": 9.080857233699624e-09,
+      "logits/chosen": -3.385005235671997,
+      "logits/rejected": -3.37992525100708,
+      "logps/chosen": -2.086282253265381,
+      "logps/rejected": -2.3239052295684814,
+      "loss": 1.0715,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -2.086282253265381,
+      "rewards/margins": 0.23762281239032745,
+      "rewards/rejected": -2.3239052295684814,
+      "step": 8680
+    },
+    {
+      "epoch": 1.4972432804962095,
+      "grad_norm": 12.205447471179841,
+      "learning_rate": 9.022961747401842e-09,
+      "logits/chosen": -3.3941776752471924,
+      "logits/rejected": -3.3808815479278564,
+      "logps/chosen": -2.0709939002990723,
+      "logps/rejected": -2.2435169219970703,
+      "loss": 1.1211,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -2.0709939002990723,
+      "rewards/margins": 0.1725229024887085,
+      "rewards/rejected": -2.2435169219970703,
+      "step": 8690
+    },
+    {
+      "epoch": 1.4989662301860784,
+      "grad_norm": 11.591816068275724,
+      "learning_rate": 8.96521074269117e-09,
+      "logits/chosen": -3.389867067337036,
+      "logits/rejected": -3.372288465499878,
+      "logps/chosen": -2.062361478805542,
+      "logps/rejected": -2.3587543964385986,
+      "loss": 1.0403,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -2.062361478805542,
+      "rewards/margins": 0.2963927984237671,
+      "rewards/rejected": -2.3587543964385986,
+      "step": 8700
+    },
+    {
+      "epoch": 1.4989662301860784,
+      "eval_logits/chosen": -3.4437458515167236,
+      "eval_logits/rejected": -3.440110683441162,
+      "eval_logps/chosen": -1.8862613439559937,
+      "eval_logps/rejected": -2.0782883167266846,
+      "eval_loss": 1.0885708332061768,
+      "eval_rewards/accuracies": 0.622444212436676,
+      "eval_rewards/chosen": -1.8862613439559937,
+      "eval_rewards/margins": 0.19202715158462524,
+      "eval_rewards/rejected": -2.0782883167266846,
+      "eval_runtime": 155.6191,
+      "eval_samples_per_second": 27.657,
+      "eval_steps_per_second": 3.457,
+      "step": 8700
+    },
+    {
+      "epoch": 1.5006891798759476,
+      "grad_norm": 11.926856477450745,
+      "learning_rate": 8.907604741814404e-09,
+      "logits/chosen": -3.3837451934814453,
+      "logits/rejected": -3.376244306564331,
+      "logps/chosen": -2.0221524238586426,
+      "logps/rejected": -2.257502555847168,
+      "loss": 1.0882,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -2.0221524238586426,
+      "rewards/margins": 0.2353503257036209,
+      "rewards/rejected": -2.257502555847168,
+      "step": 8710
+    },
+    {
+      "epoch": 1.5024121295658168,
+      "grad_norm": 11.460975310536078,
+      "learning_rate": 8.850144265707039e-09,
+      "logits/chosen": -3.369791030883789,
+      "logits/rejected": -3.3578715324401855,
+      "logps/chosen": -2.015796422958374,
+      "logps/rejected": -2.286170482635498,
+      "loss": 1.048,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -2.015796422958374,
+      "rewards/margins": 0.2703741788864136,
+      "rewards/rejected": -2.286170482635498,
+      "step": 8720
+    },
+    {
+      "epoch": 1.5041350792556858,
+      "grad_norm": 13.919113440725326,
+      "learning_rate": 8.792829833988588e-09,
+      "logits/chosen": -3.3745932579040527,
+      "logits/rejected": -3.359283447265625,
+      "logps/chosen": -2.005582094192505,
+      "logps/rejected": -2.3495402336120605,
+      "loss": 1.0216,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -2.005582094192505,
+      "rewards/margins": 0.34395843744277954,
+      "rewards/rejected": -2.3495402336120605,
+      "step": 8730
+    },
+    {
+      "epoch": 1.5058580289455548,
+      "grad_norm": 12.054573942887444,
+      "learning_rate": 8.73566196495787e-09,
+      "logits/chosen": -3.3506622314453125,
+      "logits/rejected": -3.3443145751953125,
+      "logps/chosen": -2.064584255218506,
+      "logps/rejected": -2.3671560287475586,
+      "loss": 1.054,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -2.064584255218506,
+      "rewards/margins": 0.3025716543197632,
+      "rewards/rejected": -2.3671560287475586,
+      "step": 8740
+    },
+    {
+      "epoch": 1.5075809786354237,
+      "grad_norm": 14.813471721159436,
+      "learning_rate": 8.678641175588324e-09,
+      "logits/chosen": -3.3783950805664062,
+      "logits/rejected": -3.3631675243377686,
+      "logps/chosen": -2.055903434753418,
+      "logps/rejected": -2.4392037391662598,
+      "loss": 1.0233,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -2.055903434753418,
+      "rewards/margins": 0.3833003044128418,
+      "rewards/rejected": -2.4392037391662598,
+      "step": 8750
+    },
+    {
+      "epoch": 1.509303928325293,
+      "grad_norm": 11.917662917019184,
+      "learning_rate": 8.621767981523351e-09,
+      "logits/chosen": -3.3482279777526855,
+      "logits/rejected": -3.340461015701294,
+      "logps/chosen": -2.0279829502105713,
+      "logps/rejected": -2.397745132446289,
+      "loss": 1.0106,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -2.0279829502105713,
+      "rewards/margins": 0.36976227164268494,
+      "rewards/rejected": -2.397745132446289,
+      "step": 8760
+    },
+    {
+      "epoch": 1.5110268780151621,
+      "grad_norm": 12.609897542303694,
+      "learning_rate": 8.565042897071607e-09,
+      "logits/chosen": -3.3789985179901123,
+      "logits/rejected": -3.3636512756347656,
+      "logps/chosen": -2.07240629196167,
+      "logps/rejected": -2.35217547416687,
+      "loss": 1.0572,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -2.07240629196167,
+      "rewards/margins": 0.27976903319358826,
+      "rewards/rejected": -2.35217547416687,
+      "step": 8770
+    },
+    {
+      "epoch": 1.512749827705031,
+      "grad_norm": 12.846754563728828,
+      "learning_rate": 8.508466435202402e-09,
+      "logits/chosen": -3.388455867767334,
+      "logits/rejected": -3.388914108276367,
+      "logps/chosen": -2.1042046546936035,
+      "logps/rejected": -2.3863015174865723,
+      "loss": 1.0778,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -2.1042046546936035,
+      "rewards/margins": 0.2820969820022583,
+      "rewards/rejected": -2.3863015174865723,
+      "step": 8780
+    },
+    {
+      "epoch": 1.5144727773949,
+      "grad_norm": 12.401211924489658,
+      "learning_rate": 8.452039107541043e-09,
+      "logits/chosen": -3.4049675464630127,
+      "logits/rejected": -3.39037823677063,
+      "logps/chosen": -2.089129686355591,
+      "logps/rejected": -2.415071487426758,
+      "loss": 1.0409,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -2.089129686355591,
+      "rewards/margins": 0.3259415924549103,
+      "rewards/rejected": -2.415071487426758,
+      "step": 8790
+    },
+    {
+      "epoch": 1.516195727084769,
+      "grad_norm": 12.29363910844857,
+      "learning_rate": 8.395761424364193e-09,
+      "logits/chosen": -3.328817844390869,
+      "logits/rejected": -3.311171054840088,
+      "logps/chosen": -2.05114483833313,
+      "logps/rejected": -2.445539951324463,
+      "loss": 1.0025,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -2.05114483833313,
+      "rewards/margins": 0.3943954408168793,
+      "rewards/rejected": -2.445539951324463,
+      "step": 8800
+    },
+    {
+      "epoch": 1.516195727084769,
+      "eval_logits/chosen": -3.4457342624664307,
+      "eval_logits/rejected": -3.4421215057373047,
+      "eval_logps/chosen": -1.8873032331466675,
+      "eval_logps/rejected": -2.0799124240875244,
+      "eval_loss": 1.088286280632019,
+      "eval_rewards/accuracies": 0.622444212436676,
+      "eval_rewards/chosen": -1.8873032331466675,
+      "eval_rewards/margins": 0.19260933995246887,
+      "eval_rewards/rejected": -2.0799124240875244,
+      "eval_runtime": 155.4378,
+      "eval_samples_per_second": 27.69,
+      "eval_steps_per_second": 3.461,
+      "step": 8800
+    },
+    {
+      "epoch": 1.5179186767746382,
+      "grad_norm": 13.44218292556638,
+      "learning_rate": 8.33963389459528e-09,
+      "logits/chosen": -3.425590991973877,
+      "logits/rejected": -3.4143176078796387,
+      "logps/chosen": -2.016841411590576,
+      "logps/rejected": -2.3242132663726807,
+      "loss": 1.0277,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -2.016841411590576,
+      "rewards/margins": 0.3073716461658478,
+      "rewards/rejected": -2.3242132663726807,
+      "step": 8810
+    },
+    {
+      "epoch": 1.5196416264645074,
+      "grad_norm": 13.944008021642928,
+      "learning_rate": 8.283657025799872e-09,
+      "logits/chosen": -3.376124620437622,
+      "logits/rejected": -3.3664581775665283,
+      "logps/chosen": -2.0818355083465576,
+      "logps/rejected": -2.3338093757629395,
+      "loss": 1.0674,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -2.0818355083465576,
+      "rewards/margins": 0.25197383761405945,
+      "rewards/rejected": -2.3338093757629395,
+      "step": 8820
+    },
+    {
+      "epoch": 1.5213645761543764,
+      "grad_norm": 14.394661304470757,
+      "learning_rate": 8.227831324181108e-09,
+      "logits/chosen": -3.2969086170196533,
+      "logits/rejected": -3.286653995513916,
+      "logps/chosen": -2.0826172828674316,
+      "logps/rejected": -2.341648578643799,
+      "loss": 1.0741,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -2.0826172828674316,
+      "rewards/margins": 0.2590309977531433,
+      "rewards/rejected": -2.341648578643799,
+      "step": 8830
+    },
+    {
+      "epoch": 1.5230875258442453,
+      "grad_norm": 13.388171647812339,
+      "learning_rate": 8.172157294575107e-09,
+      "logits/chosen": -3.3235716819763184,
+      "logits/rejected": -3.317469835281372,
+      "logps/chosen": -2.0576159954071045,
+      "logps/rejected": -2.3772473335266113,
+      "loss": 1.0295,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -2.0576159954071045,
+      "rewards/margins": 0.31963127851486206,
+      "rewards/rejected": -2.3772473335266113,
+      "step": 8840
+    },
+    {
+      "epoch": 1.5248104755341143,
+      "grad_norm": 11.542894126167196,
+      "learning_rate": 8.116635440446401e-09,
+      "logits/chosen": -3.435932159423828,
+      "logits/rejected": -3.4260661602020264,
+      "logps/chosen": -1.9646486043930054,
+      "logps/rejected": -2.279749631881714,
+      "loss": 1.0255,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.9646486043930054,
+      "rewards/margins": 0.3151010274887085,
+      "rewards/rejected": -2.279749631881714,
+      "step": 8850
+    },
+    {
+      "epoch": 1.5265334252239835,
+      "grad_norm": 13.400951624725051,
+      "learning_rate": 8.061266263883404e-09,
+      "logits/chosen": -3.3747596740722656,
+      "logits/rejected": -3.362881898880005,
+      "logps/chosen": -1.9950700998306274,
+      "logps/rejected": -2.2927870750427246,
+      "loss": 1.0256,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.9950700998306274,
+      "rewards/margins": 0.29771697521209717,
+      "rewards/rejected": -2.2927870750427246,
+      "step": 8860
+    },
+    {
+      "epoch": 1.5282563749138525,
+      "grad_norm": 10.883571324762109,
+      "learning_rate": 8.006050265593815e-09,
+      "logits/chosen": -3.4585800170898438,
+      "logits/rejected": -3.4408462047576904,
+      "logps/chosen": -2.026123046875,
+      "logps/rejected": -2.336472749710083,
+      "loss": 1.0472,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -2.026123046875,
+      "rewards/margins": 0.31034982204437256,
+      "rewards/rejected": -2.336472749710083,
+      "step": 8870
+    },
+    {
+      "epoch": 1.5299793246037217,
+      "grad_norm": 12.445986988761613,
+      "learning_rate": 7.950987944900191e-09,
+      "logits/chosen": -3.3398525714874268,
+      "logits/rejected": -3.3261284828186035,
+      "logps/chosen": -2.1273796558380127,
+      "logps/rejected": -2.4149792194366455,
+      "loss": 1.0632,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -2.1273796558380127,
+      "rewards/margins": 0.2875993251800537,
+      "rewards/rejected": -2.4149792194366455,
+      "step": 8880
+    },
+    {
+      "epoch": 1.5317022742935906,
+      "grad_norm": 13.571317954966464,
+      "learning_rate": 7.896079799735308e-09,
+      "logits/chosen": -3.3715312480926514,
+      "logits/rejected": -3.35785174369812,
+      "logps/chosen": -1.986513376235962,
+      "logps/rejected": -2.2499465942382812,
+      "loss": 1.0464,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.986513376235962,
+      "rewards/margins": 0.26343339681625366,
+      "rewards/rejected": -2.2499465942382812,
+      "step": 8890
+    },
+    {
+      "epoch": 1.5334252239834596,
+      "grad_norm": 11.745107179475879,
+      "learning_rate": 7.841326326637782e-09,
+      "logits/chosen": -3.3846287727355957,
+      "logits/rejected": -3.368257522583008,
+      "logps/chosen": -2.049900531768799,
+      "logps/rejected": -2.392817497253418,
+      "loss": 1.0338,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -2.049900531768799,
+      "rewards/margins": 0.3429167568683624,
+      "rewards/rejected": -2.392817497253418,
+      "step": 8900
+    },
+    {
+      "epoch": 1.5334252239834596,
+      "eval_logits/chosen": -3.426377058029175,
+      "eval_logits/rejected": -3.4226746559143066,
+      "eval_logps/chosen": -1.8921267986297607,
+      "eval_logps/rejected": -2.0851564407348633,
+      "eval_loss": 1.0880974531173706,
+      "eval_rewards/accuracies": 0.6238383054733276,
+      "eval_rewards/chosen": -1.8921267986297607,
+      "eval_rewards/margins": 0.1930299550294876,
+      "eval_rewards/rejected": -2.0851564407348633,
+      "eval_runtime": 155.4389,
+      "eval_samples_per_second": 27.689,
+      "eval_steps_per_second": 3.461,
+      "step": 8900
+    },
+    {
+      "epoch": 1.5351481736733288,
+      "grad_norm": 11.137429536400901,
+      "learning_rate": 7.786728020747463e-09,
+      "logits/chosen": -3.3572723865509033,
+      "logits/rejected": -3.3494229316711426,
+      "logps/chosen": -2.0908114910125732,
+      "logps/rejected": -2.379889726638794,
+      "loss": 1.0936,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -2.0908114910125732,
+      "rewards/margins": 0.28907784819602966,
+      "rewards/rejected": -2.379889726638794,
+      "step": 8910
+    },
+    {
+      "epoch": 1.5368711233631978,
+      "grad_norm": 11.805767626452418,
+      "learning_rate": 7.732285375801039e-09,
+      "logits/chosen": -3.417253017425537,
+      "logits/rejected": -3.39918851852417,
+      "logps/chosen": -2.111953020095825,
+      "logps/rejected": -2.5041909217834473,
+      "loss": 0.997,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -2.111953020095825,
+      "rewards/margins": 0.39223822951316833,
+      "rewards/rejected": -2.5041909217834473,
+      "step": 8920
+    },
+    {
+      "epoch": 1.538594073053067,
+      "grad_norm": 11.731919201329212,
+      "learning_rate": 7.677998884127543e-09,
+      "logits/chosen": -3.3902320861816406,
+      "logits/rejected": -3.3727962970733643,
+      "logps/chosen": -2.035983085632324,
+      "logps/rejected": -2.3461151123046875,
+      "loss": 1.0429,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -2.035983085632324,
+      "rewards/margins": 0.3101321756839752,
+      "rewards/rejected": -2.3461151123046875,
+      "step": 8930
+    },
+    {
+      "epoch": 1.540317022742936,
+      "grad_norm": 12.624216609272953,
+      "learning_rate": 7.623869036643902e-09,
+      "logits/chosen": -3.376899242401123,
+      "logits/rejected": -3.3681328296661377,
+      "logps/chosen": -2.082692861557007,
+      "logps/rejected": -2.411139965057373,
+      "loss": 1.035,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -2.082692861557007,
+      "rewards/margins": 0.3284473121166229,
+      "rewards/rejected": -2.411139965057373,
+      "step": 8940
+    },
+    {
+      "epoch": 1.5420399724328049,
+      "grad_norm": 14.325306077359958,
+      "learning_rate": 7.569896322850488e-09,
+      "logits/chosen": -3.320495128631592,
+      "logits/rejected": -3.319319486618042,
+      "logps/chosen": -2.0104849338531494,
+      "logps/rejected": -2.253514051437378,
+      "loss": 1.0701,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -2.0104849338531494,
+      "rewards/margins": 0.24302887916564941,
+      "rewards/rejected": -2.253514051437378,
+      "step": 8950
+    },
+    {
+      "epoch": 1.5437629221226739,
+      "grad_norm": 11.381814766128594,
+      "learning_rate": 7.516081230826716e-09,
+      "logits/chosen": -3.391714096069336,
+      "logits/rejected": -3.3796870708465576,
+      "logps/chosen": -2.1190884113311768,
+      "logps/rejected": -2.4361178874969482,
+      "loss": 1.0357,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -2.1190884113311768,
+      "rewards/margins": 0.3170296549797058,
+      "rewards/rejected": -2.4361178874969482,
+      "step": 8960
+    },
+    {
+      "epoch": 1.545485871812543,
+      "grad_norm": 13.521644242194466,
+      "learning_rate": 7.462424247226607e-09,
+      "logits/chosen": -3.356555938720703,
+      "logits/rejected": -3.344083070755005,
+      "logps/chosen": -2.073683977127075,
+      "logps/rejected": -2.2901034355163574,
+      "loss": 1.0978,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -2.073683977127075,
+      "rewards/margins": 0.2164197862148285,
+      "rewards/rejected": -2.2901034355163574,
+      "step": 8970
+    },
+    {
+      "epoch": 1.5472088215024122,
+      "grad_norm": 13.104735978319853,
+      "learning_rate": 7.408925857274373e-09,
+      "logits/chosen": -3.4099457263946533,
+      "logits/rejected": -3.39355731010437,
+      "logps/chosen": -2.1199917793273926,
+      "logps/rejected": -2.426830291748047,
+      "loss": 1.0497,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -2.1199917793273926,
+      "rewards/margins": 0.3068386912345886,
+      "rewards/rejected": -2.426830291748047,
+      "step": 8980
+    },
+    {
+      "epoch": 1.5489317711922812,
+      "grad_norm": 11.276772983132943,
+      "learning_rate": 7.355586544760109e-09,
+      "logits/chosen": -3.364435911178589,
+      "logits/rejected": -3.354950428009033,
+      "logps/chosen": -2.0160648822784424,
+      "logps/rejected": -2.296431064605713,
+      "loss": 1.0402,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -2.0160648822784424,
+      "rewards/margins": 0.28036636114120483,
+      "rewards/rejected": -2.296431064605713,
+      "step": 8990
+    },
+    {
+      "epoch": 1.5506547208821502,
+      "grad_norm": 13.568439300229306,
+      "learning_rate": 7.302406792035298e-09,
+      "logits/chosen": -3.374277114868164,
+      "logits/rejected": -3.357239246368408,
+      "logps/chosen": -2.12677001953125,
+      "logps/rejected": -2.4157729148864746,
+      "loss": 1.0588,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -2.12677001953125,
+      "rewards/margins": 0.2890029549598694,
+      "rewards/rejected": -2.4157729148864746,
+      "step": 9000
+    },
+    {
+      "epoch": 1.5506547208821502,
+      "eval_logits/chosen": -3.4384145736694336,
+      "eval_logits/rejected": -3.434771776199341,
+      "eval_logps/chosen": -1.8938052654266357,
+      "eval_logps/rejected": -2.086888313293457,
+      "eval_loss": 1.0881671905517578,
+      "eval_rewards/accuracies": 0.6222118735313416,
+      "eval_rewards/chosen": -1.8938052654266357,
+      "eval_rewards/margins": 0.19308315217494965,
+      "eval_rewards/rejected": -2.086888313293457,
+      "eval_runtime": 155.5149,
+      "eval_samples_per_second": 27.676,
+      "eval_steps_per_second": 3.459,
+      "step": 9000
+    },
+    {
+      "epoch": 1.5523776705720191,
+      "grad_norm": 13.611217255185297,
+      "learning_rate": 7.249387080008551e-09,
+      "logits/chosen": -3.3640530109405518,
+      "logits/rejected": -3.351393222808838,
+      "logps/chosen": -2.053029775619507,
+      "logps/rejected": -2.3402316570281982,
+      "loss": 1.053,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -2.053029775619507,
+      "rewards/margins": 0.287201851606369,
+      "rewards/rejected": -2.3402316570281982,
+      "step": 9010
+    },
+    {
+      "epoch": 1.5541006202618883,
+      "grad_norm": 11.903204833135884,
+      "learning_rate": 7.196527888141199e-09,
+      "logits/chosen": -3.3305068016052246,
+      "logits/rejected": -3.321561336517334,
+      "logps/chosen": -2.115711212158203,
+      "logps/rejected": -2.466613292694092,
+      "loss": 1.0275,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -2.115711212158203,
+      "rewards/margins": 0.3509019911289215,
+      "rewards/rejected": -2.466613292694092,
+      "step": 9020
+    },
+    {
+      "epoch": 1.5558235699517575,
+      "grad_norm": 12.81975904020415,
+      "learning_rate": 7.14382969444299e-09,
+      "logits/chosen": -3.3375403881073,
+      "logits/rejected": -3.3394436836242676,
+      "logps/chosen": -2.1069138050079346,
+      "logps/rejected": -2.300320863723755,
+      "loss": 1.1133,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -2.1069138050079346,
+      "rewards/margins": 0.19340716302394867,
+      "rewards/rejected": -2.300320863723755,
+      "step": 9030
+    },
+    {
+      "epoch": 1.5575465196416265,
+      "grad_norm": 11.807917664183828,
+      "learning_rate": 7.091292975467744e-09,
+      "logits/chosen": -3.344649076461792,
+      "logits/rejected": -3.3325748443603516,
+      "logps/chosen": -1.991320252418518,
+      "logps/rejected": -2.2588090896606445,
+      "loss": 1.0604,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.991320252418518,
+      "rewards/margins": 0.26748889684677124,
+      "rewards/rejected": -2.2588090896606445,
+      "step": 9040
+    },
+    {
+      "epoch": 1.5592694693314955,
+      "grad_norm": 13.24658737351648,
+      "learning_rate": 7.038918206309061e-09,
+      "logits/chosen": -3.363126277923584,
+      "logits/rejected": -3.35394024848938,
+      "logps/chosen": -2.15848970413208,
+      "logps/rejected": -2.450592041015625,
+      "loss": 1.072,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -2.15848970413208,
+      "rewards/margins": 0.2921021580696106,
+      "rewards/rejected": -2.450592041015625,
+      "step": 9050
+    },
+    {
+      "epoch": 1.5609924190213644,
+      "grad_norm": 13.157519138053033,
+      "learning_rate": 6.986705860596004e-09,
+      "logits/chosen": -3.3757052421569824,
+      "logits/rejected": -3.3626301288604736,
+      "logps/chosen": -2.086751937866211,
+      "logps/rejected": -2.3911185264587402,
+      "loss": 1.0391,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -2.086751937866211,
+      "rewards/margins": 0.3043665289878845,
+      "rewards/rejected": -2.3911185264587402,
+      "step": 9060
+    },
+    {
+      "epoch": 1.5627153687112336,
+      "grad_norm": 13.38913204233695,
+      "learning_rate": 6.934656410488848e-09,
+      "logits/chosen": -3.3583309650421143,
+      "logits/rejected": -3.348170518875122,
+      "logps/chosen": -2.0671863555908203,
+      "logps/rejected": -2.401096820831299,
+      "loss": 1.0253,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -2.0671863555908203,
+      "rewards/margins": 0.33391067385673523,
+      "rewards/rejected": -2.401096820831299,
+      "step": 9070
+    },
+    {
+      "epoch": 1.5644383184011028,
+      "grad_norm": 11.498170275672445,
+      "learning_rate": 6.882770326674752e-09,
+      "logits/chosen": -3.354210376739502,
+      "logits/rejected": -3.353961944580078,
+      "logps/chosen": -2.0234580039978027,
+      "logps/rejected": -2.320186138153076,
+      "loss": 1.0488,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -2.0234580039978027,
+      "rewards/margins": 0.2967282831668854,
+      "rewards/rejected": -2.320186138153076,
+      "step": 9080
+    },
+    {
+      "epoch": 1.5661612680909718,
+      "grad_norm": 12.186187916307258,
+      "learning_rate": 6.831048078363602e-09,
+      "logits/chosen": -3.3424885272979736,
+      "logits/rejected": -3.3251564502716064,
+      "logps/chosen": -2.0821683406829834,
+      "logps/rejected": -2.3564114570617676,
+      "loss": 1.0553,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -2.0821683406829834,
+      "rewards/margins": 0.27424296736717224,
+      "rewards/rejected": -2.3564114570617676,
+      "step": 9090
+    },
+    {
+      "epoch": 1.5678842177808407,
+      "grad_norm": 15.044632455626337,
+      "learning_rate": 6.779490133283638e-09,
+      "logits/chosen": -3.381788730621338,
+      "logits/rejected": -3.367598056793213,
+      "logps/chosen": -2.1106629371643066,
+      "logps/rejected": -2.327448606491089,
+      "loss": 1.0998,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -2.1106629371643066,
+      "rewards/margins": 0.2167859524488449,
+      "rewards/rejected": -2.327448606491089,
+      "step": 9100
+    },
+    {
+      "epoch": 1.5678842177808407,
+      "eval_logits/chosen": -3.439126491546631,
+      "eval_logits/rejected": -3.4354970455169678,
+      "eval_logps/chosen": -1.8946640491485596,
+      "eval_logps/rejected": -2.0878422260284424,
+      "eval_loss": 1.0880857706069946,
+      "eval_rewards/accuracies": 0.6233736276626587,
+      "eval_rewards/chosen": -1.8946640491485596,
+      "eval_rewards/margins": 0.19317813217639923,
+      "eval_rewards/rejected": -2.0878422260284424,
+      "eval_runtime": 155.6124,
+      "eval_samples_per_second": 27.658,
+      "eval_steps_per_second": 3.457,
+      "step": 9100
+    },
+    {
+      "epoch": 1.5696071674707097,
+      "grad_norm": 11.182314714704285,
+      "learning_rate": 6.72809695767736e-09,
+      "logits/chosen": -3.3898589611053467,
+      "logits/rejected": -3.3753795623779297,
+      "logps/chosen": -2.0450425148010254,
+      "logps/rejected": -2.4069406986236572,
+      "loss": 1.0212,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -2.0450425148010254,
+      "rewards/margins": 0.3618980348110199,
+      "rewards/rejected": -2.4069406986236572,
+      "step": 9110
+    },
+    {
+      "epoch": 1.571330117160579,
+      "grad_norm": 11.819522265654228,
+      "learning_rate": 6.676869016297179e-09,
+      "logits/chosen": -3.3956246376037598,
+      "logits/rejected": -3.376795530319214,
+      "logps/chosen": -2.042591094970703,
+      "logps/rejected": -2.3806662559509277,
+      "loss": 1.0151,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -2.042591094970703,
+      "rewards/margins": 0.33807528018951416,
+      "rewards/rejected": -2.3806662559509277,
+      "step": 9120
+    },
+    {
+      "epoch": 1.573053066850448,
+      "grad_norm": 12.361716807079427,
+      "learning_rate": 6.625806772401346e-09,
+      "logits/chosen": -3.3429179191589355,
+      "logits/rejected": -3.330155611038208,
+      "logps/chosen": -2.050503969192505,
+      "logps/rejected": -2.3110005855560303,
+      "loss": 1.066,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -2.050503969192505,
+      "rewards/margins": 0.26049700379371643,
+      "rewards/rejected": -2.3110005855560303,
+      "step": 9130
+    },
+    {
+      "epoch": 1.574776016540317,
+      "grad_norm": 11.19018233130226,
+      "learning_rate": 6.574910687749641e-09,
+      "logits/chosen": -3.388246536254883,
+      "logits/rejected": -3.3602161407470703,
+      "logps/chosen": -1.9514808654785156,
+      "logps/rejected": -2.3538901805877686,
+      "loss": 0.9836,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.9514808654785156,
+      "rewards/margins": 0.40240979194641113,
+      "rewards/rejected": -2.3538901805877686,
+      "step": 9140
+    },
+    {
+      "epoch": 1.576498966230186,
+      "grad_norm": 14.494096775414832,
+      "learning_rate": 6.524181222599282e-09,
+      "logits/chosen": -3.379873275756836,
+      "logits/rejected": -3.365915298461914,
+      "logps/chosen": -2.1099801063537598,
+      "logps/rejected": -2.5097954273223877,
+      "loss": 1.0447,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -2.1099801063537598,
+      "rewards/margins": 0.3998152017593384,
+      "rewards/rejected": -2.5097954273223877,
+      "step": 9150
+    },
+    {
+      "epoch": 1.578221915920055,
+      "grad_norm": 16.179943670022443,
+      "learning_rate": 6.473618835700731e-09,
+      "logits/chosen": -3.388232469558716,
+      "logits/rejected": -3.3854362964630127,
+      "logps/chosen": -2.095226287841797,
+      "logps/rejected": -2.3754029273986816,
+      "loss": 1.0581,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -2.095226287841797,
+      "rewards/margins": 0.2801765501499176,
+      "rewards/rejected": -2.3754029273986816,
+      "step": 9160
+    },
+    {
+      "epoch": 1.5799448656099242,
+      "grad_norm": 11.556306438112518,
+      "learning_rate": 6.4232239842935434e-09,
+      "logits/chosen": -3.3819668292999268,
+      "logits/rejected": -3.3660895824432373,
+      "logps/chosen": -2.1780097484588623,
+      "logps/rejected": -2.4718034267425537,
+      "loss": 1.0726,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -2.1780097484588623,
+      "rewards/margins": 0.29379358887672424,
+      "rewards/rejected": -2.4718034267425537,
+      "step": 9170
+    },
+    {
+      "epoch": 1.5816678152997934,
+      "grad_norm": 11.859984881267636,
+      "learning_rate": 6.372997124102245e-09,
+      "logits/chosen": -3.3773109912872314,
+      "logits/rejected": -3.365447521209717,
+      "logps/chosen": -2.09277081489563,
+      "logps/rejected": -2.410689115524292,
+      "loss": 1.0384,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -2.09277081489563,
+      "rewards/margins": 0.31791865825653076,
+      "rewards/rejected": -2.410689115524292,
+      "step": 9180
+    },
+    {
+      "epoch": 1.5833907649896624,
+      "grad_norm": 12.355976624689347,
+      "learning_rate": 6.3229387093321955e-09,
+      "logits/chosen": -3.443082094192505,
+      "logits/rejected": -3.438563823699951,
+      "logps/chosen": -2.0745761394500732,
+      "logps/rejected": -2.3370003700256348,
+      "loss": 1.0705,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -2.0745761394500732,
+      "rewards/margins": 0.26242440938949585,
+      "rewards/rejected": -2.3370003700256348,
+      "step": 9190
+    },
+    {
+      "epoch": 1.5851137146795313,
+      "grad_norm": 13.422648263744122,
+      "learning_rate": 6.273049192665503e-09,
+      "logits/chosen": -3.3851516246795654,
+      "logits/rejected": -3.3734116554260254,
+      "logps/chosen": -2.124861001968384,
+      "logps/rejected": -2.406578540802002,
+      "loss": 1.0465,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -2.124861001968384,
+      "rewards/margins": 0.2817172408103943,
+      "rewards/rejected": -2.406578540802002,
+      "step": 9200
+    },
+    {
+      "epoch": 1.5851137146795313,
+      "eval_logits/chosen": -3.43153715133667,
+      "eval_logits/rejected": -3.4278717041015625,
+      "eval_logps/chosen": -1.8948811292648315,
+      "eval_logps/rejected": -2.088118076324463,
+      "eval_loss": 1.0881016254425049,
+      "eval_rewards/accuracies": 0.6233736276626587,
+      "eval_rewards/chosen": -1.8948811292648315,
+      "eval_rewards/margins": 0.1932368129491806,
+      "eval_rewards/rejected": -2.088118076324463,
+      "eval_runtime": 155.5172,
+      "eval_samples_per_second": 27.675,
+      "eval_steps_per_second": 3.459,
+      "step": 9200
+    },
+    {
+      "epoch": 1.5868366643694003,
+      "grad_norm": 13.327783702808004,
+      "learning_rate": 6.223329025256896e-09,
+      "logits/chosen": -3.3009021282196045,
+      "logits/rejected": -3.289257049560547,
+      "logps/chosen": -2.149623394012451,
+      "logps/rejected": -2.464599609375,
+      "loss": 1.0588,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -2.149623394012451,
+      "rewards/margins": 0.31497618556022644,
+      "rewards/rejected": -2.464599609375,
+      "step": 9210
+    },
+    {
+      "epoch": 1.5885596140592695,
+      "grad_norm": 14.13291947807714,
+      "learning_rate": 6.173778656729678e-09,
+      "logits/chosen": -3.373137950897217,
+      "logits/rejected": -3.360830783843994,
+      "logps/chosen": -2.0582056045532227,
+      "logps/rejected": -2.3628385066986084,
+      "loss": 1.0411,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -2.0582056045532227,
+      "rewards/margins": 0.30463308095932007,
+      "rewards/rejected": -2.3628385066986084,
+      "step": 9220
+    },
+    {
+      "epoch": 1.5902825637491387,
+      "grad_norm": 13.024866968021042,
+      "learning_rate": 6.124398535171654e-09,
+      "logits/chosen": -3.3157849311828613,
+      "logits/rejected": -3.309462308883667,
+      "logps/chosen": -2.0114243030548096,
+      "logps/rejected": -2.322011947631836,
+      "loss": 1.0248,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -2.0114243030548096,
+      "rewards/margins": 0.3105877935886383,
+      "rewards/rejected": -2.322011947631836,
+      "step": 9230
+    },
+    {
+      "epoch": 1.5920055134390076,
+      "grad_norm": 10.41978077586609,
+      "learning_rate": 6.075189107131059e-09,
+      "logits/chosen": -3.3338351249694824,
+      "logits/rejected": -3.3291172981262207,
+      "logps/chosen": -2.1281228065490723,
+      "logps/rejected": -2.3280417919158936,
+      "loss": 1.1224,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -2.1281228065490723,
+      "rewards/margins": 0.19991934299468994,
+      "rewards/rejected": -2.3280417919158936,
+      "step": 9240
+    },
+    {
+      "epoch": 1.5937284631288766,
+      "grad_norm": 10.32348113445322,
+      "learning_rate": 6.026150817612544e-09,
+      "logits/chosen": -3.3548481464385986,
+      "logits/rejected": -3.3425300121307373,
+      "logps/chosen": -1.974169135093689,
+      "logps/rejected": -2.4033076763153076,
+      "loss": 1.0149,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.974169135093689,
+      "rewards/margins": 0.42913857102394104,
+      "rewards/rejected": -2.4033076763153076,
+      "step": 9250
+    },
+    {
+      "epoch": 1.5954514128187456,
+      "grad_norm": 13.51071201981465,
+      "learning_rate": 5.977284110073136e-09,
+      "logits/chosen": -3.336449146270752,
+      "logits/rejected": -3.3261878490448,
+      "logps/chosen": -2.131319999694824,
+      "logps/rejected": -2.4847559928894043,
+      "loss": 1.0129,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -2.131319999694824,
+      "rewards/margins": 0.3534359037876129,
+      "rewards/rejected": -2.4847559928894043,
+      "step": 9260
+    },
+    {
+      "epoch": 1.5971743625086148,
+      "grad_norm": 12.508499920881167,
+      "learning_rate": 5.928589426418234e-09,
+      "logits/chosen": -3.427445650100708,
+      "logits/rejected": -3.4098167419433594,
+      "logps/chosen": -2.0914363861083984,
+      "logps/rejected": -2.462337017059326,
+      "loss": 1.0081,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -2.0914363861083984,
+      "rewards/margins": 0.37090060114860535,
+      "rewards/rejected": -2.462337017059326,
+      "step": 9270
+    },
+    {
+      "epoch": 1.598897312198484,
+      "grad_norm": 11.40271796583052,
+      "learning_rate": 5.880067206997611e-09,
+      "logits/chosen": -3.3713676929473877,
+      "logits/rejected": -3.362244129180908,
+      "logps/chosen": -2.041980743408203,
+      "logps/rejected": -2.362905263900757,
+      "loss": 1.0256,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -2.041980743408203,
+      "rewards/margins": 0.3209245800971985,
+      "rewards/rejected": -2.362905263900757,
+      "step": 9280
+    },
+    {
+      "epoch": 1.600620261888353,
+      "grad_norm": 11.696210239827854,
+      "learning_rate": 5.831717890601434e-09,
+      "logits/chosen": -3.3021342754364014,
+      "logits/rejected": -3.2909436225891113,
+      "logps/chosen": -2.0872135162353516,
+      "logps/rejected": -2.342451572418213,
+      "loss": 1.0679,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -2.0872135162353516,
+      "rewards/margins": 0.2552381753921509,
+      "rewards/rejected": -2.342451572418213,
+      "step": 9290
+    },
+    {
+      "epoch": 1.602343211578222,
+      "grad_norm": 16.423065971940982,
+      "learning_rate": 5.7835419144563e-09,
+      "logits/chosen": -3.3588509559631348,
+      "logits/rejected": -3.3558120727539062,
+      "logps/chosen": -2.2164688110351562,
+      "logps/rejected": -2.4646754264831543,
+      "loss": 1.0754,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -2.2164688110351562,
+      "rewards/margins": 0.24820654094219208,
+      "rewards/rejected": -2.4646754264831543,
+      "step": 9300
+    },
+    {
+      "epoch": 1.602343211578222,
+      "eval_logits/chosen": -3.429793119430542,
+      "eval_logits/rejected": -3.4261205196380615,
+      "eval_logps/chosen": -1.8954914808273315,
+      "eval_logps/rejected": -2.0892794132232666,
+      "eval_loss": 1.0877987146377563,
+      "eval_rewards/accuracies": 0.6233736276626587,
+      "eval_rewards/chosen": -1.8954914808273315,
+      "eval_rewards/margins": 0.19378821551799774,
+      "eval_rewards/rejected": -2.0892794132232666,
+      "eval_runtime": 155.6265,
+      "eval_samples_per_second": 27.656,
+      "eval_steps_per_second": 3.457,
+      "step": 9300
+    },
+    {
+      "epoch": 1.6040661612680909,
+      "grad_norm": 13.660528446581749,
+      "learning_rate": 5.7355397142212495e-09,
+      "logits/chosen": -3.3764259815216064,
+      "logits/rejected": -3.359527587890625,
+      "logps/chosen": -2.090332269668579,
+      "logps/rejected": -2.4477055072784424,
+      "loss": 1.0103,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -2.090332269668579,
+      "rewards/margins": 0.35737311840057373,
+      "rewards/rejected": -2.4477055072784424,
+      "step": 9310
+    },
+    {
+      "epoch": 1.60578911095796,
+      "grad_norm": 11.503317351717007,
+      "learning_rate": 5.687711723983907e-09,
+      "logits/chosen": -3.410529375076294,
+      "logits/rejected": -3.4004740715026855,
+      "logps/chosen": -2.0949411392211914,
+      "logps/rejected": -2.4656567573547363,
+      "loss": 0.9947,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -2.0949411392211914,
+      "rewards/margins": 0.370715856552124,
+      "rewards/rejected": -2.4656567573547363,
+      "step": 9320
+    },
+    {
+      "epoch": 1.607512060647829,
+      "grad_norm": 15.294611663534052,
+      "learning_rate": 5.640058376256437e-09,
+      "logits/chosen": -3.380633592605591,
+      "logits/rejected": -3.3665294647216797,
+      "logps/chosen": -2.1556270122528076,
+      "logps/rejected": -2.458939552307129,
+      "loss": 1.0513,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -2.1556270122528076,
+      "rewards/margins": 0.30331239104270935,
+      "rewards/rejected": -2.458939552307129,
+      "step": 9330
+    },
+    {
+      "epoch": 1.6092350103376982,
+      "grad_norm": 11.182414232118026,
+      "learning_rate": 5.592580101971764e-09,
+      "logits/chosen": -3.334028720855713,
+      "logits/rejected": -3.3275108337402344,
+      "logps/chosen": -2.1675596237182617,
+      "logps/rejected": -2.415195941925049,
+      "loss": 1.076,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -2.1675596237182617,
+      "rewards/margins": 0.24763670563697815,
+      "rewards/rejected": -2.415195941925049,
+      "step": 9340
+    },
+    {
+      "epoch": 1.6109579600275672,
+      "grad_norm": 12.429913227944946,
+      "learning_rate": 5.545277330479558e-09,
+      "logits/chosen": -3.3760714530944824,
+      "logits/rejected": -3.3607094287872314,
+      "logps/chosen": -2.0925936698913574,
+      "logps/rejected": -2.4348902702331543,
+      "loss": 1.02,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -2.0925936698913574,
+      "rewards/margins": 0.3422970175743103,
+      "rewards/rejected": -2.4348902702331543,
+      "step": 9350
+    },
+    {
+      "epoch": 1.6126809097174362,
+      "grad_norm": 16.769317844174537,
+      "learning_rate": 5.498150489542428e-09,
+      "logits/chosen": -3.4136459827423096,
+      "logits/rejected": -3.3948302268981934,
+      "logps/chosen": -2.020965814590454,
+      "logps/rejected": -2.4000935554504395,
+      "loss": 1.0075,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -2.020965814590454,
+      "rewards/margins": 0.37912774085998535,
+      "rewards/rejected": -2.4000935554504395,
+      "step": 9360
+    },
+    {
+      "epoch": 1.6144038594073054,
+      "grad_norm": 10.292877484429384,
+      "learning_rate": 5.4512000053320264e-09,
+      "logits/chosen": -3.421590805053711,
+      "logits/rejected": -3.405540943145752,
+      "logps/chosen": -2.044325351715088,
+      "logps/rejected": -2.356584310531616,
+      "loss": 1.0372,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -2.044325351715088,
+      "rewards/margins": 0.3122590184211731,
+      "rewards/rejected": -2.356584310531616,
+      "step": 9370
+    },
+    {
+      "epoch": 1.6161268090971743,
+      "grad_norm": 13.334415124971036,
+      "learning_rate": 5.4044263024251994e-09,
+      "logits/chosen": -3.4242262840270996,
+      "logits/rejected": -3.415149688720703,
+      "logps/chosen": -2.0508854389190674,
+      "logps/rejected": -2.2779788970947266,
+      "loss": 1.1034,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -2.0508854389190674,
+      "rewards/margins": 0.22709333896636963,
+      "rewards/rejected": -2.2779788970947266,
+      "step": 9380
+    },
+    {
+      "epoch": 1.6178497587870435,
+      "grad_norm": 13.573178037832594,
+      "learning_rate": 5.3578298038001375e-09,
+      "logits/chosen": -3.286930799484253,
+      "logits/rejected": -3.2799816131591797,
+      "logps/chosen": -2.141655206680298,
+      "logps/rejected": -2.4343278408050537,
+      "loss": 1.0439,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -2.141655206680298,
+      "rewards/margins": 0.2926730513572693,
+      "rewards/rejected": -2.4343278408050537,
+      "step": 9390
+    },
+    {
+      "epoch": 1.6195727084769125,
+      "grad_norm": 11.947665983275742,
+      "learning_rate": 5.311410930832574e-09,
+      "logits/chosen": -3.3373520374298096,
+      "logits/rejected": -3.3287463188171387,
+      "logps/chosen": -2.0495505332946777,
+      "logps/rejected": -2.309657573699951,
+      "loss": 1.0633,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -2.0495505332946777,
+      "rewards/margins": 0.26010701060295105,
+      "rewards/rejected": -2.309657573699951,
+      "step": 9400
+    },
+    {
+      "epoch": 1.6195727084769125,
+      "eval_logits/chosen": -3.431198835372925,
+      "eval_logits/rejected": -3.4275388717651367,
+      "eval_logps/chosen": -1.8962520360946655,
+      "eval_logps/rejected": -2.0902750492095947,
+      "eval_loss": 1.0877717733383179,
+      "eval_rewards/accuracies": 0.6226765513420105,
+      "eval_rewards/chosen": -1.8962520360946655,
+      "eval_rewards/margins": 0.19402296841144562,
+      "eval_rewards/rejected": -2.0902750492095947,
+      "eval_runtime": 155.5766,
+      "eval_samples_per_second": 27.665,
+      "eval_steps_per_second": 3.458,
+      "step": 9400
+    },
+    {
+      "epoch": 1.6212956581667815,
+      "grad_norm": 13.685896051790033,
+      "learning_rate": 5.265170103291952e-09,
+      "logits/chosen": -3.3592631816864014,
+      "logits/rejected": -3.3491835594177246,
+      "logps/chosen": -2.171970844268799,
+      "logps/rejected": -2.4620048999786377,
+      "loss": 1.0872,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -2.171970844268799,
+      "rewards/margins": 0.2900337278842926,
+      "rewards/rejected": -2.4620048999786377,
+      "step": 9410
+    },
+    {
+      "epoch": 1.6230186078566504,
+      "grad_norm": 12.988934452207129,
+      "learning_rate": 5.219107739337616e-09,
+      "logits/chosen": -3.3874995708465576,
+      "logits/rejected": -3.3777148723602295,
+      "logps/chosen": -2.1965394020080566,
+      "logps/rejected": -2.380235195159912,
+      "loss": 1.0973,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -2.1965394020080566,
+      "rewards/margins": 0.18369609117507935,
+      "rewards/rejected": -2.380235195159912,
+      "step": 9420
+    },
+    {
+      "epoch": 1.6247415575465196,
+      "grad_norm": 12.899896866955636,
+      "learning_rate": 5.173224255515099e-09,
+      "logits/chosen": -3.369549512863159,
+      "logits/rejected": -3.36438250541687,
+      "logps/chosen": -2.1105456352233887,
+      "logps/rejected": -2.288483142852783,
+      "loss": 1.1177,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -2.1105456352233887,
+      "rewards/margins": 0.17793770134449005,
+      "rewards/rejected": -2.288483142852783,
+      "step": 9430
+    },
+    {
+      "epoch": 1.6264645072363888,
+      "grad_norm": 14.598589059675074,
+      "learning_rate": 5.127520066752256e-09,
+      "logits/chosen": -3.362807512283325,
+      "logits/rejected": -3.3588976860046387,
+      "logps/chosen": -2.1138367652893066,
+      "logps/rejected": -2.4428141117095947,
+      "loss": 1.0455,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -2.1138367652893066,
+      "rewards/margins": 0.32897716760635376,
+      "rewards/rejected": -2.4428141117095947,
+      "step": 9440
+    },
+    {
+      "epoch": 1.6281874569262578,
+      "grad_norm": 12.866577166658761,
+      "learning_rate": 5.081995586355592e-09,
+      "logits/chosen": -3.427523136138916,
+      "logits/rejected": -3.4240803718566895,
+      "logps/chosen": -2.196753978729248,
+      "logps/rejected": -2.361006259918213,
+      "loss": 1.1374,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -2.196753978729248,
+      "rewards/margins": 0.16425177454948425,
+      "rewards/rejected": -2.361006259918213,
+      "step": 9450
+    },
+    {
+      "epoch": 1.6299104066161267,
+      "grad_norm": 14.310350569177944,
+      "learning_rate": 5.0366512260064885e-09,
+      "logits/chosen": -3.342683792114258,
+      "logits/rejected": -3.3383781909942627,
+      "logps/chosen": -2.084690809249878,
+      "logps/rejected": -2.471053123474121,
+      "loss": 1.0044,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -2.084690809249878,
+      "rewards/margins": 0.3863624930381775,
+      "rewards/rejected": -2.471053123474121,
+      "step": 9460
+    },
+    {
+      "epoch": 1.6316333563059957,
+      "grad_norm": 13.454926758399262,
+      "learning_rate": 4.99148739575749e-09,
+      "logits/chosen": -3.259220600128174,
+      "logits/rejected": -3.241483688354492,
+      "logps/chosen": -2.1098742485046387,
+      "logps/rejected": -2.3872756958007812,
+      "loss": 1.0647,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -2.1098742485046387,
+      "rewards/margins": 0.2774013876914978,
+      "rewards/rejected": -2.3872756958007812,
+      "step": 9470
+    },
+    {
+      "epoch": 1.633356305995865,
+      "grad_norm": 15.014378793142868,
+      "learning_rate": 4.94650450402859e-09,
+      "logits/chosen": -3.343561887741089,
+      "logits/rejected": -3.3264477252960205,
+      "logps/chosen": -2.193578004837036,
+      "logps/rejected": -2.558976650238037,
+      "loss": 1.0007,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -2.193578004837036,
+      "rewards/margins": 0.3653985857963562,
+      "rewards/rejected": -2.558976650238037,
+      "step": 9480
+    },
+    {
+      "epoch": 1.635079255685734,
+      "grad_norm": 12.527760822960422,
+      "learning_rate": 4.90170295760354e-09,
+      "logits/chosen": -3.349276065826416,
+      "logits/rejected": -3.338439464569092,
+      "logps/chosen": -2.0582518577575684,
+      "logps/rejected": -2.3660459518432617,
+      "loss": 1.0411,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -2.0582518577575684,
+      "rewards/margins": 0.3077942728996277,
+      "rewards/rejected": -2.3660459518432617,
+      "step": 9490
+    },
+    {
+      "epoch": 1.636802205375603,
+      "grad_norm": 10.938073378271854,
+      "learning_rate": 4.857083161626174e-09,
+      "logits/chosen": -3.364483594894409,
+      "logits/rejected": -3.355731964111328,
+      "logps/chosen": -2.093919038772583,
+      "logps/rejected": -2.407203435897827,
+      "loss": 1.0392,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -2.093919038772583,
+      "rewards/margins": 0.3132845163345337,
+      "rewards/rejected": -2.407203435897827,
+      "step": 9500
+    },
+    {
+      "epoch": 1.636802205375603,
+      "eval_logits/chosen": -3.4392518997192383,
+      "eval_logits/rejected": -3.435641050338745,
+      "eval_logps/chosen": -1.898242473602295,
+      "eval_logps/rejected": -2.091705560684204,
+      "eval_loss": 1.0881397724151611,
+      "eval_rewards/accuracies": 0.6231412887573242,
+      "eval_rewards/chosen": -1.898242473602295,
+      "eval_rewards/margins": 0.1934633105993271,
+      "eval_rewards/rejected": -2.091705560684204,
+      "eval_runtime": 155.4509,
+      "eval_samples_per_second": 27.687,
+      "eval_steps_per_second": 3.461,
+      "step": 9500
+    },
+    {
+      "epoch": 1.638525155065472,
+      "grad_norm": 14.878827195032263,
+      "learning_rate": 4.812645519596748e-09,
+      "logits/chosen": -3.3158748149871826,
+      "logits/rejected": -3.3086421489715576,
+      "logps/chosen": -2.0506715774536133,
+      "logps/rejected": -2.3861775398254395,
+      "loss": 1.029,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -2.0506715774536133,
+      "rewards/margins": 0.3355058431625366,
+      "rewards/rejected": -2.3861775398254395,
+      "step": 9510
+    },
+    {
+      "epoch": 1.640248104755341,
+      "grad_norm": 12.488782776760193,
+      "learning_rate": 4.768390433368272e-09,
+      "logits/chosen": -3.4360764026641846,
+      "logits/rejected": -3.4298343658447266,
+      "logps/chosen": -2.08872652053833,
+      "logps/rejected": -2.5136094093322754,
+      "loss": 0.9754,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -2.08872652053833,
+      "rewards/margins": 0.4248831868171692,
+      "rewards/rejected": -2.5136094093322754,
+      "step": 9520
+    },
+    {
+      "epoch": 1.6419710544452102,
+      "grad_norm": 13.118481223407132,
+      "learning_rate": 4.72431830314291e-09,
+      "logits/chosen": -3.378871202468872,
+      "logits/rejected": -3.3674113750457764,
+      "logps/chosen": -2.069153308868408,
+      "logps/rejected": -2.4596283435821533,
+      "loss": 1.0267,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -2.069153308868408,
+      "rewards/margins": 0.3904748558998108,
+      "rewards/rejected": -2.4596283435821533,
+      "step": 9530
+    },
+    {
+      "epoch": 1.6436940041350794,
+      "grad_norm": 11.436402854206408,
+      "learning_rate": 4.680429527468311e-09,
+      "logits/chosen": -3.354602098464966,
+      "logits/rejected": -3.342398166656494,
+      "logps/chosen": -2.102980852127075,
+      "logps/rejected": -2.4730350971221924,
+      "loss": 1.0103,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -2.102980852127075,
+      "rewards/margins": 0.3700545132160187,
+      "rewards/rejected": -2.4730350971221924,
+      "step": 9540
+    },
+    {
+      "epoch": 1.6454169538249483,
+      "grad_norm": 12.444870932255025,
+      "learning_rate": 4.636724503234074e-09,
+      "logits/chosen": -3.383526563644409,
+      "logits/rejected": -3.384101390838623,
+      "logps/chosen": -2.116713047027588,
+      "logps/rejected": -2.409874439239502,
+      "loss": 1.0539,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -2.116713047027588,
+      "rewards/margins": 0.2931613326072693,
+      "rewards/rejected": -2.409874439239502,
+      "step": 9550
+    },
+    {
+      "epoch": 1.6471399035148173,
+      "grad_norm": 12.075865848371867,
+      "learning_rate": 4.593203625668077e-09,
+      "logits/chosen": -3.4046032428741455,
+      "logits/rejected": -3.398500442504883,
+      "logps/chosen": -2.055119037628174,
+      "logps/rejected": -2.307112216949463,
+      "loss": 1.0692,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -2.055119037628174,
+      "rewards/margins": 0.251993328332901,
+      "rewards/rejected": -2.307112216949463,
+      "step": 9560
+    },
+    {
+      "epoch": 1.6488628532046863,
+      "grad_norm": 11.580441574462,
+      "learning_rate": 4.549867288332987e-09,
+      "logits/chosen": -3.363790988922119,
+      "logits/rejected": -3.3553245067596436,
+      "logps/chosen": -2.09773588180542,
+      "logps/rejected": -2.3956751823425293,
+      "loss": 1.0418,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -2.09773588180542,
+      "rewards/margins": 0.2979391813278198,
+      "rewards/rejected": -2.3956751823425293,
+      "step": 9570
+    },
+    {
+      "epoch": 1.6505858028945555,
+      "grad_norm": 11.726683498342142,
+      "learning_rate": 4.506715883122628e-09,
+      "logits/chosen": -3.4020702838897705,
+      "logits/rejected": -3.3931572437286377,
+      "logps/chosen": -2.1186861991882324,
+      "logps/rejected": -2.4140563011169434,
+      "loss": 1.052,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -2.1186861991882324,
+      "rewards/margins": 0.29536956548690796,
+      "rewards/rejected": -2.4140563011169434,
+      "step": 9580
+    },
+    {
+      "epoch": 1.6523087525844247,
+      "grad_norm": 15.112718054121524,
+      "learning_rate": 4.463749800258479e-09,
+      "logits/chosen": -3.4473323822021484,
+      "logits/rejected": -3.4371731281280518,
+      "logps/chosen": -2.1276679039001465,
+      "logps/rejected": -2.4121670722961426,
+      "loss": 1.0706,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -2.1276679039001465,
+      "rewards/margins": 0.2844991981983185,
+      "rewards/rejected": -2.4121670722961426,
+      "step": 9590
+    },
+    {
+      "epoch": 1.6540317022742936,
+      "grad_norm": 11.753107324374444,
+      "learning_rate": 4.420969428286139e-09,
+      "logits/chosen": -3.3470969200134277,
+      "logits/rejected": -3.3290939331054688,
+      "logps/chosen": -2.0506644248962402,
+      "logps/rejected": -2.3609237670898438,
+      "loss": 1.0565,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -2.0506644248962402,
+      "rewards/margins": 0.310259073972702,
+      "rewards/rejected": -2.3609237670898438,
+      "step": 9600
+    },
+    {
+      "epoch": 1.6540317022742936,
+      "eval_logits/chosen": -3.4421937465667725,
+      "eval_logits/rejected": -3.438598155975342,
+      "eval_logps/chosen": -1.8976588249206543,
+      "eval_logps/rejected": -2.0916953086853027,
+      "eval_loss": 1.0877597332000732,
+      "eval_rewards/accuracies": 0.6231412887573242,
+      "eval_rewards/chosen": -1.8976588249206543,
+      "eval_rewards/margins": 0.19403612613677979,
+      "eval_rewards/rejected": -2.0916953086853027,
+      "eval_runtime": 155.741,
+      "eval_samples_per_second": 27.636,
+      "eval_steps_per_second": 3.454,
+      "step": 9600
+    },
+    {
+      "epoch": 1.6557546519641626,
+      "grad_norm": 12.832484035343724,
+      "learning_rate": 4.3783751540718065e-09,
+      "logits/chosen": -3.3633384704589844,
+      "logits/rejected": -3.351158857345581,
+      "logps/chosen": -2.054800510406494,
+      "logps/rejected": -2.38541841506958,
+      "loss": 1.0457,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -2.054800510406494,
+      "rewards/margins": 0.33061763644218445,
+      "rewards/rejected": -2.38541841506958,
+      "step": 9610
+    },
+    {
+      "epoch": 1.6574776016540316,
+      "grad_norm": 11.684785643775855,
+      "learning_rate": 4.335967362798787e-09,
+      "logits/chosen": -3.43591046333313,
+      "logits/rejected": -3.4332385063171387,
+      "logps/chosen": -2.103566884994507,
+      "logps/rejected": -2.3165924549102783,
+      "loss": 1.1119,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -2.103566884994507,
+      "rewards/margins": 0.2130255401134491,
+      "rewards/rejected": -2.3165924549102783,
+      "step": 9620
+    },
+    {
+      "epoch": 1.6592005513439008,
+      "grad_norm": 11.807459182135727,
+      "learning_rate": 4.2937464379639824e-09,
+      "logits/chosen": -3.377290725708008,
+      "logits/rejected": -3.3612732887268066,
+      "logps/chosen": -2.1389691829681396,
+      "logps/rejected": -2.397247314453125,
+      "loss": 1.0637,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -2.1389691829681396,
+      "rewards/margins": 0.2582780420780182,
+      "rewards/rejected": -2.397247314453125,
+      "step": 9630
+    },
+    {
+      "epoch": 1.66092350103377,
+      "grad_norm": 12.632273516924831,
+      "learning_rate": 4.251712761374499e-09,
+      "logits/chosen": -3.422273635864258,
+      "logits/rejected": -3.405437469482422,
+      "logps/chosen": -2.1169934272766113,
+      "logps/rejected": -2.413539409637451,
+      "loss": 1.0568,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -2.1169934272766113,
+      "rewards/margins": 0.29654619097709656,
+      "rewards/rejected": -2.413539409637451,
+      "step": 9640
+    },
+    {
+      "epoch": 1.662646450723639,
+      "grad_norm": 13.680542084417683,
+      "learning_rate": 4.209866713144078e-09,
+      "logits/chosen": -3.3381519317626953,
+      "logits/rejected": -3.3278777599334717,
+      "logps/chosen": -2.2129263877868652,
+      "logps/rejected": -2.4320414066314697,
+      "loss": 1.0943,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -2.2129263877868652,
+      "rewards/margins": 0.21911516785621643,
+      "rewards/rejected": -2.4320414066314697,
+      "step": 9650
+    },
+    {
+      "epoch": 1.664369400413508,
+      "grad_norm": 14.602885169595801,
+      "learning_rate": 4.1682086716897824e-09,
+      "logits/chosen": -3.36253023147583,
+      "logits/rejected": -3.358379364013672,
+      "logps/chosen": -2.0204660892486572,
+      "logps/rejected": -2.280316114425659,
+      "loss": 1.0717,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -2.0204660892486572,
+      "rewards/margins": 0.25985026359558105,
+      "rewards/rejected": -2.280316114425659,
+      "step": 9660
+    },
+    {
+      "epoch": 1.6660923501033769,
+      "grad_norm": 11.944866072294065,
+      "learning_rate": 4.1267390137284725e-09,
+      "logits/chosen": -3.403502941131592,
+      "logits/rejected": -3.3873705863952637,
+      "logps/chosen": -2.04728364944458,
+      "logps/rejected": -2.3696484565734863,
+      "loss": 1.0383,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -2.04728364944458,
+      "rewards/margins": 0.3223644196987152,
+      "rewards/rejected": -2.3696484565734863,
+      "step": 9670
+    },
+    {
+      "epoch": 1.667815299793246,
+      "grad_norm": 12.520724965798093,
+      "learning_rate": 4.0854581142734625e-09,
+      "logits/chosen": -3.38374662399292,
+      "logits/rejected": -3.3745150566101074,
+      "logps/chosen": -2.157973051071167,
+      "logps/rejected": -2.298199415206909,
+      "loss": 1.1405,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -2.157973051071167,
+      "rewards/margins": 0.1402260959148407,
+      "rewards/rejected": -2.298199415206909,
+      "step": 9680
+    },
+    {
+      "epoch": 1.6695382494831152,
+      "grad_norm": 13.899817105961,
+      "learning_rate": 4.044366346631107e-09,
+      "logits/chosen": -3.3172478675842285,
+      "logits/rejected": -3.3064167499542236,
+      "logps/chosen": -2.137002468109131,
+      "logps/rejected": -2.3931641578674316,
+      "loss": 1.0782,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -2.137002468109131,
+      "rewards/margins": 0.25616174936294556,
+      "rewards/rejected": -2.3931641578674316,
+      "step": 9690
+    },
+    {
+      "epoch": 1.6712611991729842,
+      "grad_norm": 11.92456713984713,
+      "learning_rate": 4.003464082397421e-09,
+      "logits/chosen": -3.3557827472686768,
+      "logits/rejected": -3.3385818004608154,
+      "logps/chosen": -2.051290512084961,
+      "logps/rejected": -2.4180870056152344,
+      "loss": 1.0101,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -2.051290512084961,
+      "rewards/margins": 0.3667964041233063,
+      "rewards/rejected": -2.4180870056152344,
+      "step": 9700
+    },
+    {
+      "epoch": 1.6712611991729842,
+      "eval_logits/chosen": -3.4392876625061035,
+      "eval_logits/rejected": -3.435683012008667,
+      "eval_logps/chosen": -1.8987349271774292,
+      "eval_logps/rejected": -2.0924413204193115,
+      "eval_loss": 1.0880305767059326,
+      "eval_rewards/accuracies": 0.6222118735313416,
+      "eval_rewards/chosen": -1.8987349271774292,
+      "eval_rewards/margins": 0.1937064677476883,
+      "eval_rewards/rejected": -2.0924413204193115,
+      "eval_runtime": 155.7844,
+      "eval_samples_per_second": 27.628,
+      "eval_steps_per_second": 3.453,
+      "step": 9700
+    },
+    {
+      "epoch": 1.6729841488628532,
+      "grad_norm": 14.035387649526937,
+      "learning_rate": 3.9627516914547295e-09,
+      "logits/chosen": -3.2790961265563965,
+      "logits/rejected": -3.268620729446411,
+      "logps/chosen": -2.129859209060669,
+      "logps/rejected": -2.3758325576782227,
+      "loss": 1.1017,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -2.129859209060669,
+      "rewards/margins": 0.24597349762916565,
+      "rewards/rejected": -2.3758325576782227,
+      "step": 9710
+    },
+    {
+      "epoch": 1.6747070985527222,
+      "grad_norm": 13.690128976415274,
+      "learning_rate": 3.922229541968322e-09,
+      "logits/chosen": -3.380765914916992,
+      "logits/rejected": -3.373138904571533,
+      "logps/chosen": -2.08109974861145,
+      "logps/rejected": -2.3047163486480713,
+      "loss": 1.0832,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -2.08109974861145,
+      "rewards/margins": 0.22361686825752258,
+      "rewards/rejected": -2.3047163486480713,
+      "step": 9720
+    },
+    {
+      "epoch": 1.6764300482425913,
+      "grad_norm": 12.739181266603872,
+      "learning_rate": 3.8818980003831155e-09,
+      "logits/chosen": -3.3571105003356934,
+      "logits/rejected": -3.3476860523223877,
+      "logps/chosen": -2.0872559547424316,
+      "logps/rejected": -2.404416561126709,
+      "loss": 1.0351,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -2.0872559547424316,
+      "rewards/margins": 0.3171602189540863,
+      "rewards/rejected": -2.404416561126709,
+      "step": 9730
+    },
+    {
+      "epoch": 1.6781529979324605,
+      "grad_norm": 11.65489517572302,
+      "learning_rate": 3.841757431420351e-09,
+      "logits/chosen": -3.383352756500244,
+      "logits/rejected": -3.3744492530822754,
+      "logps/chosen": -2.1142263412475586,
+      "logps/rejected": -2.3153090476989746,
+      "loss": 1.1021,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -2.1142263412475586,
+      "rewards/margins": 0.20108266174793243,
+      "rewards/rejected": -2.3153090476989746,
+      "step": 9740
+    },
+    {
+      "epoch": 1.6798759476223295,
+      "grad_norm": 12.965020104001871,
+      "learning_rate": 3.8018081980742664e-09,
+      "logits/chosen": -3.377241611480713,
+      "logits/rejected": -3.357161045074463,
+      "logps/chosen": -2.0872387886047363,
+      "logps/rejected": -2.4027187824249268,
+      "loss": 1.0305,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -2.0872387886047363,
+      "rewards/margins": 0.31548011302948,
+      "rewards/rejected": -2.4027187824249268,
+      "step": 9750
+    },
+    {
+      "epoch": 1.6815988973121985,
+      "grad_norm": 12.757590315756632,
+      "learning_rate": 3.7620506616088815e-09,
+      "logits/chosen": -3.3867461681365967,
+      "logits/rejected": -3.375096082687378,
+      "logps/chosen": -2.1326630115509033,
+      "logps/rejected": -2.4355766773223877,
+      "loss": 1.0551,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -2.1326630115509033,
+      "rewards/margins": 0.30291393399238586,
+      "rewards/rejected": -2.4355766773223877,
+      "step": 9760
+    },
+    {
+      "epoch": 1.6833218470020674,
+      "grad_norm": 11.80456612082025,
+      "learning_rate": 3.7224851815546298e-09,
+      "logits/chosen": -3.3413166999816895,
+      "logits/rejected": -3.330176591873169,
+      "logps/chosen": -2.02840518951416,
+      "logps/rejected": -2.3230557441711426,
+      "loss": 1.0513,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -2.02840518951416,
+      "rewards/margins": 0.2946508228778839,
+      "rewards/rejected": -2.3230557441711426,
+      "step": 9770
+    },
+    {
+      "epoch": 1.6850447966919366,
+      "grad_norm": 11.759845391951071,
+      "learning_rate": 3.6831121157052254e-09,
+      "logits/chosen": -3.389039993286133,
+      "logits/rejected": -3.365095853805542,
+      "logps/chosen": -2.071049213409424,
+      "logps/rejected": -2.4179999828338623,
+      "loss": 1.0216,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -2.071049213409424,
+      "rewards/margins": 0.3469507694244385,
+      "rewards/rejected": -2.4179999828338623,
+      "step": 9780
+    },
+    {
+      "epoch": 1.6867677463818056,
+      "grad_norm": 12.225327726905787,
+      "learning_rate": 3.64393182011431e-09,
+      "logits/chosen": -3.363889694213867,
+      "logits/rejected": -3.3623149394989014,
+      "logps/chosen": -2.025153398513794,
+      "logps/rejected": -2.428903102874756,
+      "loss": 1.0024,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -2.025153398513794,
+      "rewards/margins": 0.4037497937679291,
+      "rewards/rejected": -2.428903102874756,
+      "step": 9790
+    },
+    {
+      "epoch": 1.6884906960716748,
+      "grad_norm": 12.474400657696151,
+      "learning_rate": 3.604944649092323e-09,
+      "logits/chosen": -3.3829147815704346,
+      "logits/rejected": -3.3645756244659424,
+      "logps/chosen": -2.0064713954925537,
+      "logps/rejected": -2.4253318309783936,
+      "loss": 0.9686,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -2.0064713954925537,
+      "rewards/margins": 0.41886061429977417,
+      "rewards/rejected": -2.4253318309783936,
+      "step": 9800
+    },
+    {
+      "epoch": 1.6884906960716748,
+      "eval_logits/chosen": -3.431636095046997,
+      "eval_logits/rejected": -3.4279942512512207,
+      "eval_logps/chosen": -1.8991549015045166,
+      "eval_logps/rejected": -2.093291997909546,
+      "eval_loss": 1.0878663063049316,
+      "eval_rewards/accuracies": 0.6231412887573242,
+      "eval_rewards/chosen": -1.8991549015045166,
+      "eval_rewards/margins": 0.19413697719573975,
+      "eval_rewards/rejected": -2.093291997909546,
+      "eval_runtime": 155.3588,
+      "eval_samples_per_second": 27.704,
+      "eval_steps_per_second": 3.463,
+      "step": 9800
+    },
+    {
+      "epoch": 1.6902136457615438,
+      "grad_norm": 11.213153817452016,
+      "learning_rate": 3.566150955203251e-09,
+      "logits/chosen": -3.337503433227539,
+      "logits/rejected": -3.319955348968506,
+      "logps/chosen": -2.088658571243286,
+      "logps/rejected": -2.5349936485290527,
+      "loss": 0.9756,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -2.088658571243286,
+      "rewards/margins": 0.44633474946022034,
+      "rewards/rejected": -2.5349936485290527,
+      "step": 9810
+    },
+    {
+      "epoch": 1.6919365954514127,
+      "grad_norm": 15.144188960338285,
+      "learning_rate": 3.52755108926146e-09,
+      "logits/chosen": -3.3765571117401123,
+      "logits/rejected": -3.3640060424804688,
+      "logps/chosen": -2.0144588947296143,
+      "logps/rejected": -2.36830735206604,
+      "loss": 1.0223,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -2.0144588947296143,
+      "rewards/margins": 0.3538484275341034,
+      "rewards/rejected": -2.36830735206604,
+      "step": 9820
+    },
+    {
+      "epoch": 1.693659545141282,
+      "grad_norm": 10.505250399397598,
+      "learning_rate": 3.489145400328511e-09,
+      "logits/chosen": -3.3944358825683594,
+      "logits/rejected": -3.390552520751953,
+      "logps/chosen": -2.1232962608337402,
+      "logps/rejected": -2.327637195587158,
+      "loss": 1.1209,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -2.1232962608337402,
+      "rewards/margins": 0.20434105396270752,
+      "rewards/rejected": -2.327637195587158,
+      "step": 9830
+    },
+    {
+      "epoch": 1.6953824948311509,
+      "grad_norm": 10.95661477260746,
+      "learning_rate": 3.4509342357099904e-09,
+      "logits/chosen": -3.3548266887664795,
+      "logits/rejected": -3.3380870819091797,
+      "logps/chosen": -2.0794105529785156,
+      "logps/rejected": -2.4375908374786377,
+      "loss": 1.0117,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -2.0794105529785156,
+      "rewards/margins": 0.3581800162792206,
+      "rewards/rejected": -2.4375908374786377,
+      "step": 9840
+    },
+    {
+      "epoch": 1.69710544452102,
+      "grad_norm": 11.480677604371474,
+      "learning_rate": 3.412917940952423e-09,
+      "logits/chosen": -3.3643875122070312,
+      "logits/rejected": -3.3582496643066406,
+      "logps/chosen": -2.037616729736328,
+      "logps/rejected": -2.256088972091675,
+      "loss": 1.0927,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -2.037616729736328,
+      "rewards/margins": 0.2184722125530243,
+      "rewards/rejected": -2.256088972091675,
+      "step": 9850
+    },
+    {
+      "epoch": 1.698828394210889,
+      "grad_norm": 15.458928772681844,
+      "learning_rate": 3.375096859840071e-09,
+      "logits/chosen": -3.4079995155334473,
+      "logits/rejected": -3.402322292327881,
+      "logps/chosen": -2.2665247917175293,
+      "logps/rejected": -2.3680005073547363,
+      "loss": 1.1636,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -2.2665247917175293,
+      "rewards/margins": 0.10147533565759659,
+      "rewards/rejected": -2.3680005073547363,
+      "step": 9860
+    },
+    {
+      "epoch": 1.700551343900758,
+      "grad_norm": 13.148596374660032,
+      "learning_rate": 3.337471334391903e-09,
+      "logits/chosen": -3.395045757293701,
+      "logits/rejected": -3.379577159881592,
+      "logps/chosen": -2.061051368713379,
+      "logps/rejected": -2.4077844619750977,
+      "loss": 1.0225,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -2.061051368713379,
+      "rewards/margins": 0.34673285484313965,
+      "rewards/rejected": -2.4077844619750977,
+      "step": 9870
+    },
+    {
+      "epoch": 1.7022742935906272,
+      "grad_norm": 11.884415433790643,
+      "learning_rate": 3.300041704858425e-09,
+      "logits/chosen": -3.330756425857544,
+      "logits/rejected": -3.327300548553467,
+      "logps/chosen": -2.068145513534546,
+      "logps/rejected": -2.3367857933044434,
+      "loss": 1.0702,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -2.068145513534546,
+      "rewards/margins": 0.2686399817466736,
+      "rewards/rejected": -2.3367857933044434,
+      "step": 9880
+    },
+    {
+      "epoch": 1.7039972432804962,
+      "grad_norm": 12.737468487841062,
+      "learning_rate": 3.2628083097186675e-09,
+      "logits/chosen": -3.302340269088745,
+      "logits/rejected": -3.297541379928589,
+      "logps/chosen": -2.1667723655700684,
+      "logps/rejected": -2.395362138748169,
+      "loss": 1.1001,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -2.1667723655700684,
+      "rewards/margins": 0.22858986258506775,
+      "rewards/rejected": -2.395362138748169,
+      "step": 9890
+    },
+    {
+      "epoch": 1.7057201929703654,
+      "grad_norm": 13.250093844750376,
+      "learning_rate": 3.2257714856770866e-09,
+      "logits/chosen": -3.3646183013916016,
+      "logits/rejected": -3.3481998443603516,
+      "logps/chosen": -1.9987783432006836,
+      "logps/rejected": -2.4336705207824707,
+      "loss": 0.9781,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.9987783432006836,
+      "rewards/margins": 0.43489211797714233,
+      "rewards/rejected": -2.4336705207824707,
+      "step": 9900
+    },
+    {
+      "epoch": 1.7057201929703654,
+      "eval_logits/chosen": -3.4352614879608154,
+      "eval_logits/rejected": -3.4316303730010986,
+      "eval_logps/chosen": -1.8996483087539673,
+      "eval_logps/rejected": -2.0942230224609375,
+      "eval_loss": 1.0875462293624878,
+      "eval_rewards/accuracies": 0.6229089498519897,
+      "eval_rewards/chosen": -1.8996483087539673,
+      "eval_rewards/margins": 0.1945747435092926,
+      "eval_rewards/rejected": -2.0942230224609375,
+      "eval_runtime": 155.4284,
+      "eval_samples_per_second": 27.691,
+      "eval_steps_per_second": 3.461,
+      "step": 9900
+    },
+    {
+      "epoch": 1.7074431426602343,
+      "grad_norm": 12.624704970144144,
+      "learning_rate": 3.188931567660533e-09,
+      "logits/chosen": -3.4036242961883545,
+      "logits/rejected": -3.3812363147735596,
+      "logps/chosen": -2.131389617919922,
+      "logps/rejected": -2.429779052734375,
+      "loss": 1.0552,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -2.131389617919922,
+      "rewards/margins": 0.2983894646167755,
+      "rewards/rejected": -2.429779052734375,
+      "step": 9910
+    },
+    {
+      "epoch": 1.7091660923501033,
+      "grad_norm": 12.456998026818756,
+      "learning_rate": 3.152288888815227e-09,
+      "logits/chosen": -3.406803846359253,
+      "logits/rejected": -3.391112804412842,
+      "logps/chosen": -2.0501558780670166,
+      "logps/rejected": -2.360121488571167,
+      "loss": 1.0362,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -2.0501558780670166,
+      "rewards/margins": 0.30996543169021606,
+      "rewards/rejected": -2.360121488571167,
+      "step": 9920
+    },
+    {
+      "epoch": 1.7108890420399723,
+      "grad_norm": 11.149199782118654,
+      "learning_rate": 3.1158437805037296e-09,
+      "logits/chosen": -3.371018648147583,
+      "logits/rejected": -3.3672492504119873,
+      "logps/chosen": -2.0402259826660156,
+      "logps/rejected": -2.3050878047943115,
+      "loss": 1.0522,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -2.0402259826660156,
+      "rewards/margins": 0.2648618221282959,
+      "rewards/rejected": -2.3050878047943115,
+      "step": 9930
+    },
+    {
+      "epoch": 1.7126119917298415,
+      "grad_norm": 10.810103838253388,
+      "learning_rate": 3.0795965723019653e-09,
+      "logits/chosen": -3.4189116954803467,
+      "logits/rejected": -3.4162826538085938,
+      "logps/chosen": -2.0555267333984375,
+      "logps/rejected": -2.291738271713257,
+      "loss": 1.0758,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -2.0555267333984375,
+      "rewards/margins": 0.23621153831481934,
+      "rewards/rejected": -2.291738271713257,
+      "step": 9940
+    },
+    {
+      "epoch": 1.7143349414197107,
+      "grad_norm": 15.696853405454267,
+      "learning_rate": 3.043547591996226e-09,
+      "logits/chosen": -3.3843836784362793,
+      "logits/rejected": -3.3662116527557373,
+      "logps/chosen": -2.025460720062256,
+      "logps/rejected": -2.394860029220581,
+      "loss": 1.0047,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -2.025460720062256,
+      "rewards/margins": 0.36939913034439087,
+      "rewards/rejected": -2.394860029220581,
+      "step": 9950
+    },
+    {
+      "epoch": 1.7160578911095796,
+      "grad_norm": 13.269835492466566,
+      "learning_rate": 3.0076971655802196e-09,
+      "logits/chosen": -3.4025161266326904,
+      "logits/rejected": -3.392662525177002,
+      "logps/chosen": -2.1722710132598877,
+      "logps/rejected": -2.390134811401367,
+      "loss": 1.11,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -2.1722710132598877,
+      "rewards/margins": 0.21786388754844666,
+      "rewards/rejected": -2.390134811401367,
+      "step": 9960
+    },
+    {
+      "epoch": 1.7177808407994486,
+      "grad_norm": 11.813054436371418,
+      "learning_rate": 2.972045617252114e-09,
+      "logits/chosen": -3.3628287315368652,
+      "logits/rejected": -3.351527690887451,
+      "logps/chosen": -2.1331210136413574,
+      "logps/rejected": -2.4996793270111084,
+      "loss": 1.0263,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -2.1331210136413574,
+      "rewards/margins": 0.3665587306022644,
+      "rewards/rejected": -2.4996793270111084,
+      "step": 9970
+    },
+    {
+      "epoch": 1.7195037904893176,
+      "grad_norm": 12.489039600275445,
+      "learning_rate": 2.9365932694115913e-09,
+      "logits/chosen": -3.3461010456085205,
+      "logits/rejected": -3.3394932746887207,
+      "logps/chosen": -2.115257740020752,
+      "logps/rejected": -2.3778798580169678,
+      "loss": 1.0683,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -2.115257740020752,
+      "rewards/margins": 0.26262184977531433,
+      "rewards/rejected": -2.3778798580169678,
+      "step": 9980
+    },
+    {
+      "epoch": 1.7212267401791868,
+      "grad_norm": 13.215791108137498,
+      "learning_rate": 2.9013404426569853e-09,
+      "logits/chosen": -3.39528226852417,
+      "logits/rejected": -3.3690237998962402,
+      "logps/chosen": -2.101757049560547,
+      "logps/rejected": -2.5460236072540283,
+      "loss": 0.9912,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -2.101757049560547,
+      "rewards/margins": 0.4442666172981262,
+      "rewards/rejected": -2.5460236072540283,
+      "step": 9990
+    },
+    {
+      "epoch": 1.722949689869056,
+      "grad_norm": 12.408648592291556,
+      "learning_rate": 2.8662874557823015e-09,
+      "logits/chosen": -3.398831605911255,
+      "logits/rejected": -3.3864665031433105,
+      "logps/chosen": -2.0159056186676025,
+      "logps/rejected": -2.3872179985046387,
+      "loss": 0.9985,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -2.0159056186676025,
+      "rewards/margins": 0.3713122308254242,
+      "rewards/rejected": -2.3872179985046387,
+      "step": 10000
+    },
+    {
+      "epoch": 1.722949689869056,
+      "eval_logits/chosen": -3.437005043029785,
+      "eval_logits/rejected": -3.433393716812134,
+      "eval_logps/chosen": -1.900423526763916,
+      "eval_logps/rejected": -2.094653844833374,
+      "eval_loss": 1.0878093242645264,
+      "eval_rewards/accuracies": 0.622444212436676,
+      "eval_rewards/chosen": -1.900423526763916,
+      "eval_rewards/margins": 0.1942305564880371,
+      "eval_rewards/rejected": -2.094653844833374,
+      "eval_runtime": 155.4349,
+      "eval_samples_per_second": 27.69,
+      "eval_steps_per_second": 3.461,
+      "step": 10000
+    },
+    {
+      "epoch": 1.724672639558925,
+      "grad_norm": 13.027340598248555,
+      "learning_rate": 2.8314346257744175e-09,
+      "logits/chosen": -3.3841655254364014,
+      "logits/rejected": -3.3730568885803223,
+      "logps/chosen": -2.092644214630127,
+      "logps/rejected": -2.453434467315674,
+      "loss": 1.0121,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -2.092644214630127,
+      "rewards/margins": 0.3607902228832245,
+      "rewards/rejected": -2.453434467315674,
+      "step": 10010
+    },
+    {
+      "epoch": 1.7263955892487939,
+      "grad_norm": 14.037769829244121,
+      "learning_rate": 2.7967822678101468e-09,
+      "logits/chosen": -3.3598244190216064,
+      "logits/rejected": -3.343372344970703,
+      "logps/chosen": -2.1410627365112305,
+      "logps/rejected": -2.449596643447876,
+      "loss": 1.04,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -2.1410627365112305,
+      "rewards/margins": 0.3085339665412903,
+      "rewards/rejected": -2.449596643447876,
+      "step": 10020
+    },
+    {
+      "epoch": 1.7281185389386629,
+      "grad_norm": 15.327257331737053,
+      "learning_rate": 2.7623306952534314e-09,
+      "logits/chosen": -3.3849072456359863,
+      "logits/rejected": -3.367274522781372,
+      "logps/chosen": -2.103039026260376,
+      "logps/rejected": -2.3627519607543945,
+      "loss": 1.065,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -2.103039026260376,
+      "rewards/margins": 0.2597127854824066,
+      "rewards/rejected": -2.3627519607543945,
+      "step": 10030
+    },
+    {
+      "epoch": 1.729841488628532,
+      "grad_norm": 12.091789600380661,
+      "learning_rate": 2.7280802196525036e-09,
+      "logits/chosen": -3.456063747406006,
+      "logits/rejected": -3.448279619216919,
+      "logps/chosen": -2.0574851036071777,
+      "logps/rejected": -2.377424955368042,
+      "loss": 1.0432,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -2.0574851036071777,
+      "rewards/margins": 0.3199399709701538,
+      "rewards/rejected": -2.377424955368042,
+      "step": 10040
+    },
+    {
+      "epoch": 1.7315644383184012,
+      "grad_norm": 14.926661661141676,
+      "learning_rate": 2.694031150737036e-09,
+      "logits/chosen": -3.3561477661132812,
+      "logits/rejected": -3.3528614044189453,
+      "logps/chosen": -2.190943956375122,
+      "logps/rejected": -2.292475700378418,
+      "loss": 1.159,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -2.190943956375122,
+      "rewards/margins": 0.10153186321258545,
+      "rewards/rejected": -2.292475700378418,
+      "step": 10050
+    },
+    {
+      "epoch": 1.7332873880082702,
+      "grad_norm": 13.032442699639736,
+      "learning_rate": 2.6601837964153994e-09,
+      "logits/chosen": -3.331810712814331,
+      "logits/rejected": -3.328859806060791,
+      "logps/chosen": -2.1173417568206787,
+      "logps/rejected": -2.408304452896118,
+      "loss": 1.0712,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -2.1173417568206787,
+      "rewards/margins": 0.29096275568008423,
+      "rewards/rejected": -2.408304452896118,
+      "step": 10060
+    },
+    {
+      "epoch": 1.7350103376981392,
+      "grad_norm": 13.255790986669478,
+      "learning_rate": 2.6265384627718046e-09,
+      "logits/chosen": -3.3181490898132324,
+      "logits/rejected": -3.310483455657959,
+      "logps/chosen": -2.03804612159729,
+      "logps/rejected": -2.324540138244629,
+      "loss": 1.0438,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -2.03804612159729,
+      "rewards/margins": 0.28649407625198364,
+      "rewards/rejected": -2.324540138244629,
+      "step": 10070
+    },
+    {
+      "epoch": 1.7367332873880081,
+      "grad_norm": 16.238681044307526,
+      "learning_rate": 2.593095454063615e-09,
+      "logits/chosen": -3.407952070236206,
+      "logits/rejected": -3.4002318382263184,
+      "logps/chosen": -2.1662182807922363,
+      "logps/rejected": -2.316488265991211,
+      "loss": 1.1303,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -2.1662182807922363,
+      "rewards/margins": 0.15027007460594177,
+      "rewards/rejected": -2.316488265991211,
+      "step": 10080
+    },
+    {
+      "epoch": 1.7384562370778773,
+      "grad_norm": 12.700524092400045,
+      "learning_rate": 2.5598550727185142e-09,
+      "logits/chosen": -3.4056758880615234,
+      "logits/rejected": -3.3952739238739014,
+      "logps/chosen": -1.9950271844863892,
+      "logps/rejected": -2.2925987243652344,
+      "loss": 1.0262,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.9950271844863892,
+      "rewards/margins": 0.297571063041687,
+      "rewards/rejected": -2.2925987243652344,
+      "step": 10090
+    },
+    {
+      "epoch": 1.7401791867677465,
+      "grad_norm": 14.244562710196927,
+      "learning_rate": 2.5268176193318473e-09,
+      "logits/chosen": -3.384169101715088,
+      "logits/rejected": -3.378526210784912,
+      "logps/chosen": -2.114086151123047,
+      "logps/rejected": -2.386474609375,
+      "loss": 1.0605,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -2.114086151123047,
+      "rewards/margins": 0.2723882794380188,
+      "rewards/rejected": -2.386474609375,
+      "step": 10100
+    },
+    {
+      "epoch": 1.7401791867677465,
+      "eval_logits/chosen": -3.424624443054199,
+      "eval_logits/rejected": -3.420955181121826,
+      "eval_logps/chosen": -1.900676965713501,
+      "eval_logps/rejected": -2.094648838043213,
+      "eval_loss": 1.08788001537323,
+      "eval_rewards/accuracies": 0.6226765513420105,
+      "eval_rewards/chosen": -1.900676965713501,
+      "eval_rewards/margins": 0.19397184252738953,
+      "eval_rewards/rejected": -2.094648838043213,
+      "eval_runtime": 155.5942,
+      "eval_samples_per_second": 27.662,
+      "eval_steps_per_second": 3.458,
+      "step": 10100
+    },
+    {
+      "epoch": 1.7419021364576155,
+      "grad_norm": 13.424228560991413,
+      "learning_rate": 2.4939833926638397e-09,
+      "logits/chosen": -3.4055869579315186,
+      "logits/rejected": -3.408522844314575,
+      "logps/chosen": -2.242760181427002,
+      "logps/rejected": -2.487175941467285,
+      "loss": 1.127,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -2.242760181427002,
+      "rewards/margins": 0.24441590905189514,
+      "rewards/rejected": -2.487175941467285,
+      "step": 10110
+    },
+    {
+      "epoch": 1.7436250861474845,
+      "grad_norm": 14.1205467201749,
+      "learning_rate": 2.4613526896369308e-09,
+      "logits/chosen": -3.379404067993164,
+      "logits/rejected": -3.3688244819641113,
+      "logps/chosen": -2.148313045501709,
+      "logps/rejected": -2.4625396728515625,
+      "loss": 1.0481,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -2.148313045501709,
+      "rewards/margins": 0.314226895570755,
+      "rewards/rejected": -2.4625396728515625,
+      "step": 10120
+    },
+    {
+      "epoch": 1.7453480358373534,
+      "grad_norm": 13.270786223999773,
+      "learning_rate": 2.428925805333082e-09,
+      "logits/chosen": -3.4070522785186768,
+      "logits/rejected": -3.394615650177002,
+      "logps/chosen": -1.9925920963287354,
+      "logps/rejected": -2.343778371810913,
+      "loss": 1.0125,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.9925920963287354,
+      "rewards/margins": 0.35118645429611206,
+      "rewards/rejected": -2.343778371810913,
+      "step": 10130
+    },
+    {
+      "epoch": 1.7470709855272226,
+      "grad_norm": 16.078741150778022,
+      "learning_rate": 2.396703032991107e-09,
+      "logits/chosen": -3.3780009746551514,
+      "logits/rejected": -3.3559749126434326,
+      "logps/chosen": -2.0255932807922363,
+      "logps/rejected": -2.324061155319214,
+      "loss": 1.0415,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -2.0255932807922363,
+      "rewards/margins": 0.2984677255153656,
+      "rewards/rejected": -2.324061155319214,
+      "step": 10140
+    },
+    {
+      "epoch": 1.7487939352170918,
+      "grad_norm": 11.456423480296932,
+      "learning_rate": 2.364684664004016e-09,
+      "logits/chosen": -3.3548197746276855,
+      "logits/rejected": -3.3446431159973145,
+      "logps/chosen": -2.132145404815674,
+      "logps/rejected": -2.4262607097625732,
+      "loss": 1.0583,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -2.132145404815674,
+      "rewards/margins": 0.29411548376083374,
+      "rewards/rejected": -2.4262607097625732,
+      "step": 10150
+    },
+    {
+      "epoch": 1.7505168849069608,
+      "grad_norm": 14.524460707007709,
+      "learning_rate": 2.3328709879163826e-09,
+      "logits/chosen": -3.3664422035217285,
+      "logits/rejected": -3.3584625720977783,
+      "logps/chosen": -2.1687536239624023,
+      "logps/rejected": -2.443345069885254,
+      "loss": 1.0793,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -2.1687536239624023,
+      "rewards/margins": 0.2745913863182068,
+      "rewards/rejected": -2.443345069885254,
+      "step": 10160
+    },
+    {
+      "epoch": 1.7522398345968297,
+      "grad_norm": 10.768714029026182,
+      "learning_rate": 2.301262292421732e-09,
+      "logits/chosen": -3.3718020915985107,
+      "logits/rejected": -3.3654046058654785,
+      "logps/chosen": -2.0777223110198975,
+      "logps/rejected": -2.3268139362335205,
+      "loss": 1.0933,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -2.0777223110198975,
+      "rewards/margins": 0.24909155070781708,
+      "rewards/rejected": -2.3268139362335205,
+      "step": 10170
+    },
+    {
+      "epoch": 1.7539627842866987,
+      "grad_norm": 11.457162273004704,
+      "learning_rate": 2.269858863359936e-09,
+      "logits/chosen": -3.3331665992736816,
+      "logits/rejected": -3.3179256916046143,
+      "logps/chosen": -1.9872100353240967,
+      "logps/rejected": -2.369452476501465,
+      "loss": 0.996,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.9872100353240967,
+      "rewards/margins": 0.3822426497936249,
+      "rewards/rejected": -2.369452476501465,
+      "step": 10180
+    },
+    {
+      "epoch": 1.755685733976568,
+      "grad_norm": 13.678199123356226,
+      "learning_rate": 2.2386609847146077e-09,
+      "logits/chosen": -3.3618178367614746,
+      "logits/rejected": -3.347688674926758,
+      "logps/chosen": -2.019031047821045,
+      "logps/rejected": -2.3042445182800293,
+      "loss": 1.0542,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -2.019031047821045,
+      "rewards/margins": 0.2852138578891754,
+      "rewards/rejected": -2.3042445182800293,
+      "step": 10190
+    },
+    {
+      "epoch": 1.757408683666437,
+      "grad_norm": 12.72142805423001,
+      "learning_rate": 2.207668938610582e-09,
+      "logits/chosen": -3.389802932739258,
+      "logits/rejected": -3.379854679107666,
+      "logps/chosen": -2.1316587924957275,
+      "logps/rejected": -2.4421894550323486,
+      "loss": 1.0453,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -2.1316587924957275,
+      "rewards/margins": 0.3105306923389435,
+      "rewards/rejected": -2.4421894550323486,
+      "step": 10200
+    },
+    {
+      "epoch": 1.757408683666437,
+      "eval_logits/chosen": -3.4221835136413574,
+      "eval_logits/rejected": -3.4185032844543457,
+      "eval_logps/chosen": -1.902440071105957,
+      "eval_logps/rejected": -2.096791982650757,
+      "eval_loss": 1.0878218412399292,
+      "eval_rewards/accuracies": 0.622444212436676,
+      "eval_rewards/chosen": -1.902440071105957,
+      "eval_rewards/margins": 0.19435177743434906,
+      "eval_rewards/rejected": -2.096791982650757,
+      "eval_runtime": 155.1861,
+      "eval_samples_per_second": 27.734,
+      "eval_steps_per_second": 3.467,
+      "step": 10200
+    },
+    {
+      "epoch": 1.759131633356306,
+      "grad_norm": 14.748559746647677,
+      "learning_rate": 2.176883005311303e-09,
+      "logits/chosen": -3.393372058868408,
+      "logits/rejected": -3.39158296585083,
+      "logps/chosen": -2.13710880279541,
+      "logps/rejected": -2.523911237716675,
+      "loss": 1.008,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -2.13710880279541,
+      "rewards/margins": 0.386802613735199,
+      "rewards/rejected": -2.523911237716675,
+      "step": 10210
+    },
+    {
+      "epoch": 1.760854583046175,
+      "grad_norm": 12.255114528961405,
+      "learning_rate": 2.1463034632163533e-09,
+      "logits/chosen": -3.3667187690734863,
+      "logits/rejected": -3.3653061389923096,
+      "logps/chosen": -2.0325591564178467,
+      "logps/rejected": -2.381286144256592,
+      "loss": 1.0295,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -2.0325591564178467,
+      "rewards/margins": 0.34872668981552124,
+      "rewards/rejected": -2.381286144256592,
+      "step": 10220
+    },
+    {
+      "epoch": 1.762577532736044,
+      "grad_norm": 13.153328365423903,
+      "learning_rate": 2.1159305888588664e-09,
+      "logits/chosen": -3.3162925243377686,
+      "logits/rejected": -3.300133228302002,
+      "logps/chosen": -2.0939431190490723,
+      "logps/rejected": -2.4561893939971924,
+      "loss": 1.0109,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -2.0939431190490723,
+      "rewards/margins": 0.3622463643550873,
+      "rewards/rejected": -2.4561893939971924,
+      "step": 10230
+    },
+    {
+      "epoch": 1.7643004824259132,
+      "grad_norm": 13.0006916169091,
+      "learning_rate": 2.085764656903105e-09,
+      "logits/chosen": -3.3498923778533936,
+      "logits/rejected": -3.3381507396698,
+      "logps/chosen": -2.0724053382873535,
+      "logps/rejected": -2.380021333694458,
+      "loss": 1.0407,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -2.0724053382873535,
+      "rewards/margins": 0.30761581659317017,
+      "rewards/rejected": -2.380021333694458,
+      "step": 10240
+    },
+    {
+      "epoch": 1.7660234321157822,
+      "grad_norm": 13.190297717886967,
+      "learning_rate": 2.055805940141897e-09,
+      "logits/chosen": -3.3954715728759766,
+      "logits/rejected": -3.37440824508667,
+      "logps/chosen": -2.0411157608032227,
+      "logps/rejected": -2.428973913192749,
+      "loss": 0.9957,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -2.0411157608032227,
+      "rewards/margins": 0.3878583014011383,
+      "rewards/rejected": -2.428973913192749,
+      "step": 10250
+    },
+    {
+      "epoch": 1.7677463818056514,
+      "grad_norm": 11.953201477762688,
+      "learning_rate": 2.026054709494235e-09,
+      "logits/chosen": -3.3544421195983887,
+      "logits/rejected": -3.3544986248016357,
+      "logps/chosen": -2.1528735160827637,
+      "logps/rejected": -2.415017604827881,
+      "loss": 1.0659,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -2.1528735160827637,
+      "rewards/margins": 0.2621440291404724,
+      "rewards/rejected": -2.415017604827881,
+      "step": 10260
+    },
+    {
+      "epoch": 1.7694693314955203,
+      "grad_norm": 12.659104634995758,
+      "learning_rate": 1.9965112340027874e-09,
+      "logits/chosen": -3.362326145172119,
+      "logits/rejected": -3.356783628463745,
+      "logps/chosen": -2.1153759956359863,
+      "logps/rejected": -2.3739404678344727,
+      "loss": 1.0689,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -2.1153759956359863,
+      "rewards/margins": 0.2585643231868744,
+      "rewards/rejected": -2.3739404678344727,
+      "step": 10270
+    },
+    {
+      "epoch": 1.7711922811853893,
+      "grad_norm": 13.238915250592138,
+      "learning_rate": 1.9671757808314675e-09,
+      "logits/chosen": -3.34789776802063,
+      "logits/rejected": -3.3375649452209473,
+      "logps/chosen": -2.2042412757873535,
+      "logps/rejected": -2.4372763633728027,
+      "loss": 1.0823,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -2.2042412757873535,
+      "rewards/margins": 0.23303496837615967,
+      "rewards/rejected": -2.4372763633728027,
+      "step": 10280
+    },
+    {
+      "epoch": 1.7729152308752585,
+      "grad_norm": 12.131176774446358,
+      "learning_rate": 1.9380486152630547e-09,
+      "logits/chosen": -3.322694778442383,
+      "logits/rejected": -3.3231632709503174,
+      "logps/chosen": -2.0964009761810303,
+      "logps/rejected": -2.3539462089538574,
+      "loss": 1.07,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -2.0964009761810303,
+      "rewards/margins": 0.2575451135635376,
+      "rewards/rejected": -2.3539462089538574,
+      "step": 10290
+    },
+    {
+      "epoch": 1.7746381805651275,
+      "grad_norm": 12.983976805229169,
+      "learning_rate": 1.909130000696732e-09,
+      "logits/chosen": -3.3594024181365967,
+      "logits/rejected": -3.3507938385009766,
+      "logps/chosen": -2.1097540855407715,
+      "logps/rejected": -2.349905490875244,
+      "loss": 1.0919,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -2.1097540855407715,
+      "rewards/margins": 0.2401510775089264,
+      "rewards/rejected": -2.349905490875244,
+      "step": 10300
+    },
+    {
+      "epoch": 1.7746381805651275,
+      "eval_logits/chosen": -3.4382922649383545,
+      "eval_logits/rejected": -3.4346916675567627,
+      "eval_logps/chosen": -1.9026802778244019,
+      "eval_logps/rejected": -2.0973362922668457,
+      "eval_loss": 1.087674856185913,
+      "eval_rewards/accuracies": 0.6219795346260071,
+      "eval_rewards/chosen": -1.9026802778244019,
+      "eval_rewards/margins": 0.1946561485528946,
+      "eval_rewards/rejected": -2.0973362922668457,
+      "eval_runtime": 155.2202,
+      "eval_samples_per_second": 27.728,
+      "eval_steps_per_second": 3.466,
+      "step": 10300
+    },
+    {
+      "epoch": 1.7763611302549966,
+      "grad_norm": 11.439755705201888,
+      "learning_rate": 1.880420198645774e-09,
+      "logits/chosen": -3.3207366466522217,
+      "logits/rejected": -3.3069560527801514,
+      "logps/chosen": -2.087655544281006,
+      "logps/rejected": -2.4661524295806885,
+      "loss": 0.993,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -2.087655544281006,
+      "rewards/margins": 0.37849703431129456,
+      "rewards/rejected": -2.4661524295806885,
+      "step": 10310
+    },
+    {
+      "epoch": 1.7780840799448656,
+      "grad_norm": 11.526175541198672,
+      "learning_rate": 1.8519194687351191e-09,
+      "logits/chosen": -3.3676364421844482,
+      "logits/rejected": -3.352550506591797,
+      "logps/chosen": -2.144334316253662,
+      "logps/rejected": -2.4295384883880615,
+      "loss": 1.0638,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -2.144334316253662,
+      "rewards/margins": 0.28520435094833374,
+      "rewards/rejected": -2.4295384883880615,
+      "step": 10320
+    },
+    {
+      "epoch": 1.7798070296347346,
+      "grad_norm": 12.014695114286555,
+      "learning_rate": 1.8236280686990653e-09,
+      "logits/chosen": -3.3817317485809326,
+      "logits/rejected": -3.3741111755371094,
+      "logps/chosen": -2.067434072494507,
+      "logps/rejected": -2.4220669269561768,
+      "loss": 1.0077,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -2.067434072494507,
+      "rewards/margins": 0.3546326160430908,
+      "rewards/rejected": -2.4220669269561768,
+      "step": 10330
+    },
+    {
+      "epoch": 1.7815299793246038,
+      "grad_norm": 14.348054793474132,
+      "learning_rate": 1.7955462543789268e-09,
+      "logits/chosen": -3.379500150680542,
+      "logits/rejected": -3.369227647781372,
+      "logps/chosen": -2.1099209785461426,
+      "logps/rejected": -2.39658784866333,
+      "loss": 1.0444,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -2.1099209785461426,
+      "rewards/margins": 0.2866671085357666,
+      "rewards/rejected": -2.39658784866333,
+      "step": 10340
+    },
+    {
+      "epoch": 1.7832529290144727,
+      "grad_norm": 12.505486079029396,
+      "learning_rate": 1.7676742797207045e-09,
+      "logits/chosen": -3.4374077320098877,
+      "logits/rejected": -3.4231765270233154,
+      "logps/chosen": -2.0471019744873047,
+      "logps/rejected": -2.332987070083618,
+      "loss": 1.0508,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -2.0471019744873047,
+      "rewards/margins": 0.2858850359916687,
+      "rewards/rejected": -2.332987070083618,
+      "step": 10350
+    },
+    {
+      "epoch": 1.784975878704342,
+      "grad_norm": 13.824593521072613,
+      "learning_rate": 1.7400123967728192e-09,
+      "logits/chosen": -3.338933229446411,
+      "logits/rejected": -3.3212294578552246,
+      "logps/chosen": -2.0418267250061035,
+      "logps/rejected": -2.346783399581909,
+      "loss": 1.043,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -2.0418267250061035,
+      "rewards/margins": 0.30495673418045044,
+      "rewards/rejected": -2.346783399581909,
+      "step": 10360
+    },
+    {
+      "epoch": 1.786698828394211,
+      "grad_norm": 12.196133657942976,
+      "learning_rate": 1.7125608556838034e-09,
+      "logits/chosen": -3.275792360305786,
+      "logits/rejected": -3.259190320968628,
+      "logps/chosen": -2.0164945125579834,
+      "logps/rejected": -2.481198787689209,
+      "loss": 0.9462,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -2.0164945125579834,
+      "rewards/margins": 0.46470412611961365,
+      "rewards/rejected": -2.481198787689209,
+      "step": 10370
+    },
+    {
+      "epoch": 1.7884217780840799,
+      "grad_norm": 13.214258266999465,
+      "learning_rate": 1.6853199047000583e-09,
+      "logits/chosen": -3.3676280975341797,
+      "logits/rejected": -3.3611721992492676,
+      "logps/chosen": -2.235682725906372,
+      "logps/rejected": -2.395051956176758,
+      "loss": 1.1539,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -2.235682725906372,
+      "rewards/margins": 0.15936926007270813,
+      "rewards/rejected": -2.395051956176758,
+      "step": 10380
+    },
+    {
+      "epoch": 1.7901447277739488,
+      "grad_norm": 11.848357768248995,
+      "learning_rate": 1.6582897901636028e-09,
+      "logits/chosen": -3.3751397132873535,
+      "logits/rejected": -3.3618693351745605,
+      "logps/chosen": -2.108635425567627,
+      "logps/rejected": -2.474367618560791,
+      "loss": 1.0067,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -2.108635425567627,
+      "rewards/margins": 0.36573225259780884,
+      "rewards/rejected": -2.474367618560791,
+      "step": 10390
+    },
+    {
+      "epoch": 1.791867677463818,
+      "grad_norm": 12.279327295406548,
+      "learning_rate": 1.6314707565098396e-09,
+      "logits/chosen": -3.3423194885253906,
+      "logits/rejected": -3.330577850341797,
+      "logps/chosen": -2.0364503860473633,
+      "logps/rejected": -2.4654791355133057,
+      "loss": 0.9683,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -2.0364503860473633,
+      "rewards/margins": 0.4290287494659424,
+      "rewards/rejected": -2.4654791355133057,
+      "step": 10400
+    },
+    {
+      "epoch": 1.791867677463818,
+      "eval_logits/chosen": -3.430449962615967,
+      "eval_logits/rejected": -3.4268064498901367,
+      "eval_logps/chosen": -1.9023096561431885,
+      "eval_logps/rejected": -2.096813678741455,
+      "eval_loss": 1.08774995803833,
+      "eval_rewards/accuracies": 0.6231412887573242,
+      "eval_rewards/chosen": -1.9023096561431885,
+      "eval_rewards/margins": 0.19450390338897705,
+      "eval_rewards/rejected": -2.096813678741455,
+      "eval_runtime": 155.2955,
+      "eval_samples_per_second": 27.715,
+      "eval_steps_per_second": 3.464,
+      "step": 10400
+    },
+    {
+      "epoch": 1.7935906271536872,
+      "grad_norm": 11.925629687977967,
+      "learning_rate": 1.6048630462653618e-09,
+      "logits/chosen": -3.359221935272217,
+      "logits/rejected": -3.338473081588745,
+      "logps/chosen": -2.1006007194519043,
+      "logps/rejected": -2.469526767730713,
+      "loss": 1.0079,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -2.1006007194519043,
+      "rewards/margins": 0.36892610788345337,
+      "rewards/rejected": -2.469526767730713,
+      "step": 10410
+    },
+    {
+      "epoch": 1.7953135768435562,
+      "grad_norm": 12.608338583150015,
+      "learning_rate": 1.5784669000457328e-09,
+      "logits/chosen": -3.371291399002075,
+      "logits/rejected": -3.35632586479187,
+      "logps/chosen": -2.099696159362793,
+      "logps/rejected": -2.3917317390441895,
+      "loss": 1.0509,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -2.099696159362793,
+      "rewards/margins": 0.29203587770462036,
+      "rewards/rejected": -2.3917317390441895,
+      "step": 10420
+    },
+    {
+      "epoch": 1.7970365265334252,
+      "grad_norm": 14.168705069966549,
+      "learning_rate": 1.5522825565533443e-09,
+      "logits/chosen": -3.404088258743286,
+      "logits/rejected": -3.3958823680877686,
+      "logps/chosen": -2.1119003295898438,
+      "logps/rejected": -2.3166277408599854,
+      "loss": 1.1063,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -2.1119003295898438,
+      "rewards/margins": 0.20472726225852966,
+      "rewards/rejected": -2.3166277408599854,
+      "step": 10430
+    },
+    {
+      "epoch": 1.7987594762232941,
+      "grad_norm": 11.595487481918715,
+      "learning_rate": 1.5263102525752219e-09,
+      "logits/chosen": -3.4109675884246826,
+      "logits/rejected": -3.404555082321167,
+      "logps/chosen": -2.1607463359832764,
+      "logps/rejected": -2.3388116359710693,
+      "loss": 1.1101,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -2.1607463359832764,
+      "rewards/margins": 0.1780654340982437,
+      "rewards/rejected": -2.3388116359710693,
+      "step": 10440
+    },
+    {
+      "epoch": 1.8004824259131633,
+      "grad_norm": 12.275002852048914,
+      "learning_rate": 1.500550222980923e-09,
+      "logits/chosen": -3.3777499198913574,
+      "logits/rejected": -3.3743388652801514,
+      "logps/chosen": -2.161961555480957,
+      "logps/rejected": -2.4404006004333496,
+      "loss": 1.0611,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -2.161961555480957,
+      "rewards/margins": 0.27843889594078064,
+      "rewards/rejected": -2.4404006004333496,
+      "step": 10450
+    },
+    {
+      "epoch": 1.8022053756030325,
+      "grad_norm": 15.607663987792392,
+      "learning_rate": 1.4750027007203653e-09,
+      "logits/chosen": -3.379298448562622,
+      "logits/rejected": -3.3669419288635254,
+      "logps/chosen": -2.023942708969116,
+      "logps/rejected": -2.2930197715759277,
+      "loss": 1.0586,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -2.023942708969116,
+      "rewards/margins": 0.26907679438591003,
+      "rewards/rejected": -2.2930197715759277,
+      "step": 10460
+    },
+    {
+      "epoch": 1.8039283252929015,
+      "grad_norm": 12.730122824905648,
+      "learning_rate": 1.4496679168217645e-09,
+      "logits/chosen": -3.2812869548797607,
+      "logits/rejected": -3.269512891769409,
+      "logps/chosen": -2.054513931274414,
+      "logps/rejected": -2.348254442214966,
+      "loss": 1.0569,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -2.054513931274414,
+      "rewards/margins": 0.29374057054519653,
+      "rewards/rejected": -2.348254442214966,
+      "step": 10470
+    },
+    {
+      "epoch": 1.8056512749827704,
+      "grad_norm": 13.648301954640491,
+      "learning_rate": 1.424546100389523e-09,
+      "logits/chosen": -3.3760788440704346,
+      "logits/rejected": -3.357637882232666,
+      "logps/chosen": -2.051119565963745,
+      "logps/rejected": -2.407987356185913,
+      "loss": 1.0355,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -2.051119565963745,
+      "rewards/margins": 0.3568675220012665,
+      "rewards/rejected": -2.407987356185913,
+      "step": 10480
+    },
+    {
+      "epoch": 1.8073742246726394,
+      "grad_norm": 12.795104541497471,
+      "learning_rate": 1.3996374786021641e-09,
+      "logits/chosen": -3.3745617866516113,
+      "logits/rejected": -3.358513355255127,
+      "logps/chosen": -2.0083248615264893,
+      "logps/rejected": -2.4178032875061035,
+      "loss": 0.9906,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -2.0083248615264893,
+      "rewards/margins": 0.4094786047935486,
+      "rewards/rejected": -2.4178032875061035,
+      "step": 10490
+    },
+    {
+      "epoch": 1.8090971743625086,
+      "grad_norm": 13.12206258727388,
+      "learning_rate": 1.3749422767102697e-09,
+      "logits/chosen": -3.368983745574951,
+      "logits/rejected": -3.363375186920166,
+      "logps/chosen": -2.0600662231445312,
+      "logps/rejected": -2.3615713119506836,
+      "loss": 1.0501,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -2.0600662231445312,
+      "rewards/margins": 0.30150535702705383,
+      "rewards/rejected": -2.3615713119506836,
+      "step": 10500
+    },
+    {
+      "epoch": 1.8090971743625086,
+      "eval_logits/chosen": -3.430471897125244,
+      "eval_logits/rejected": -3.426827907562256,
+      "eval_logps/chosen": -1.9027376174926758,
+      "eval_logps/rejected": -2.0970709323883057,
+      "eval_loss": 1.087856411933899,
+      "eval_rewards/accuracies": 0.6226765513420105,
+      "eval_rewards/chosen": -1.9027376174926758,
+      "eval_rewards/margins": 0.1943332999944687,
+      "eval_rewards/rejected": -2.0970709323883057,
+      "eval_runtime": 155.3328,
+      "eval_samples_per_second": 27.708,
+      "eval_steps_per_second": 3.464,
+      "step": 10500
+    },
+    {
+      "epoch": 1.8108201240523778,
+      "grad_norm": 13.552757015037983,
+      "learning_rate": 1.3504607180344462e-09,
+      "logits/chosen": -3.3568336963653564,
+      "logits/rejected": -3.3446903228759766,
+      "logps/chosen": -2.168165683746338,
+      "logps/rejected": -2.4172043800354004,
+      "loss": 1.0739,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -2.168165683746338,
+      "rewards/margins": 0.24903841316699982,
+      "rewards/rejected": -2.4172043800354004,
+      "step": 10510
+    },
+    {
+      "epoch": 1.8125430737422468,
+      "grad_norm": 11.838538342157852,
+      "learning_rate": 1.3261930239633263e-09,
+      "logits/chosen": -3.4039294719696045,
+      "logits/rejected": -3.4056854248046875,
+      "logps/chosen": -2.1421000957489014,
+      "logps/rejected": -2.4267849922180176,
+      "loss": 1.0601,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -2.1421000957489014,
+      "rewards/margins": 0.2846847176551819,
+      "rewards/rejected": -2.4267849922180176,
+      "step": 10520
+    },
+    {
+      "epoch": 1.8142660234321157,
+      "grad_norm": 17.04880417926912,
+      "learning_rate": 1.3021394139515196e-09,
+      "logits/chosen": -3.348702907562256,
+      "logits/rejected": -3.3384876251220703,
+      "logps/chosen": -2.127981662750244,
+      "logps/rejected": -2.324708938598633,
+      "loss": 1.1152,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -2.127981662750244,
+      "rewards/margins": 0.19672763347625732,
+      "rewards/rejected": -2.324708938598633,
+      "step": 10530
+    },
+    {
+      "epoch": 1.8159889731219847,
+      "grad_norm": 13.457179740457885,
+      "learning_rate": 1.2783001055176905e-09,
+      "logits/chosen": -3.3494484424591064,
+      "logits/rejected": -3.335627317428589,
+      "logps/chosen": -2.0796799659729004,
+      "logps/rejected": -2.4020657539367676,
+      "loss": 1.0398,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -2.0796799659729004,
+      "rewards/margins": 0.3223859667778015,
+      "rewards/rejected": -2.4020657539367676,
+      "step": 10540
+    },
+    {
+      "epoch": 1.817711922811854,
+      "grad_norm": 14.584077544052192,
+      "learning_rate": 1.2546753142425314e-09,
+      "logits/chosen": -3.4281165599823,
+      "logits/rejected": -3.4279541969299316,
+      "logps/chosen": -2.176887273788452,
+      "logps/rejected": -2.4040017127990723,
+      "loss": 1.1065,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -2.176887273788452,
+      "rewards/margins": 0.22711482644081116,
+      "rewards/rejected": -2.4040017127990723,
+      "step": 10550
+    },
+    {
+      "epoch": 1.819434872501723,
+      "grad_norm": 13.05909974406496,
+      "learning_rate": 1.23126525376685e-09,
+      "logits/chosen": -3.3435750007629395,
+      "logits/rejected": -3.334912061691284,
+      "logps/chosen": -2.102079391479492,
+      "logps/rejected": -2.4306812286376953,
+      "loss": 1.0318,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -2.102079391479492,
+      "rewards/margins": 0.3286019265651703,
+      "rewards/rejected": -2.4306812286376953,
+      "step": 10560
+    },
+    {
+      "epoch": 1.821157822191592,
+      "grad_norm": 10.831956464247373,
+      "learning_rate": 1.2080701357896266e-09,
+      "logits/chosen": -3.390256404876709,
+      "logits/rejected": -3.3824265003204346,
+      "logps/chosen": -2.0863471031188965,
+      "logps/rejected": -2.4140074253082275,
+      "loss": 1.0324,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -2.0863471031188965,
+      "rewards/margins": 0.32766005396842957,
+      "rewards/rejected": -2.4140074253082275,
+      "step": 10570
+    },
+    {
+      "epoch": 1.822880771881461,
+      "grad_norm": 11.935122300062522,
+      "learning_rate": 1.185090170066097e-09,
+      "logits/chosen": -3.39304780960083,
+      "logits/rejected": -3.384188175201416,
+      "logps/chosen": -2.1546950340270996,
+      "logps/rejected": -2.4045581817626953,
+      "loss": 1.084,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -2.1546950340270996,
+      "rewards/margins": 0.24986326694488525,
+      "rewards/rejected": -2.4045581817626953,
+      "step": 10580
+    },
+    {
+      "epoch": 1.82460372157133,
+      "grad_norm": 11.294031785336323,
+      "learning_rate": 1.1623255644058637e-09,
+      "logits/chosen": -3.357901096343994,
+      "logits/rejected": -3.3374786376953125,
+      "logps/chosen": -2.1280291080474854,
+      "logps/rejected": -2.4897360801696777,
+      "loss": 0.999,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -2.1280291080474854,
+      "rewards/margins": 0.36170709133148193,
+      "rewards/rejected": -2.4897360801696777,
+      "step": 10590
+    },
+    {
+      "epoch": 1.8263266712611992,
+      "grad_norm": 11.275161976574736,
+      "learning_rate": 1.1397765246710072e-09,
+      "logits/chosen": -3.4174561500549316,
+      "logits/rejected": -3.4114556312561035,
+      "logps/chosen": -2.0700762271881104,
+      "logps/rejected": -2.3095810413360596,
+      "loss": 1.0827,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -2.0700762271881104,
+      "rewards/margins": 0.23950473964214325,
+      "rewards/rejected": -2.3095810413360596,
+      "step": 10600
+    },
+    {
+      "epoch": 1.8263266712611992,
+      "eval_logits/chosen": -3.429658889770508,
+      "eval_logits/rejected": -3.426017999649048,
+      "eval_logps/chosen": -1.9026654958724976,
+      "eval_logps/rejected": -2.0970540046691895,
+      "eval_loss": 1.0877981185913086,
+      "eval_rewards/accuracies": 0.6222118735313416,
+      "eval_rewards/chosen": -1.9026654958724976,
+      "eval_rewards/margins": 0.19438877701759338,
+      "eval_rewards/rejected": -2.0970540046691895,
+      "eval_runtime": 155.4771,
+      "eval_samples_per_second": 27.683,
+      "eval_steps_per_second": 3.46,
+      "step": 10600
+    },
+    {
+      "epoch": 1.8280496209510684,
+      "grad_norm": 12.026715042502424,
+      "learning_rate": 1.1174432547742307e-09,
+      "logits/chosen": -3.3564600944519043,
+      "logits/rejected": -3.3497726917266846,
+      "logps/chosen": -2.0688459873199463,
+      "logps/rejected": -2.362697124481201,
+      "loss": 1.0507,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -2.0688459873199463,
+      "rewards/margins": 0.29385146498680115,
+      "rewards/rejected": -2.362697124481201,
+      "step": 10610
+    },
+    {
+      "epoch": 1.8297725706409373,
+      "grad_norm": 13.853635172507026,
+      "learning_rate": 1.095325956677015e-09,
+      "logits/chosen": -3.3415732383728027,
+      "logits/rejected": -3.3285019397735596,
+      "logps/chosen": -2.074765920639038,
+      "logps/rejected": -2.3213586807250977,
+      "loss": 1.0915,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -2.074765920639038,
+      "rewards/margins": 0.24659287929534912,
+      "rewards/rejected": -2.3213586807250977,
+      "step": 10620
+    },
+    {
+      "epoch": 1.8314955203308063,
+      "grad_norm": 14.301473140301628,
+      "learning_rate": 1.0734248303877812e-09,
+      "logits/chosen": -3.3819801807403564,
+      "logits/rejected": -3.3735568523406982,
+      "logps/chosen": -2.1064040660858154,
+      "logps/rejected": -2.365077495574951,
+      "loss": 1.0666,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -2.1064040660858154,
+      "rewards/margins": 0.25867316126823425,
+      "rewards/rejected": -2.365077495574951,
+      "step": 10630
+    },
+    {
+      "epoch": 1.8332184700206753,
+      "grad_norm": 17.854416084623217,
+      "learning_rate": 1.051740073960114e-09,
+      "logits/chosen": -3.3433985710144043,
+      "logits/rejected": -3.330411434173584,
+      "logps/chosen": -2.1446850299835205,
+      "logps/rejected": -2.4913668632507324,
+      "loss": 1.0486,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -2.1446850299835205,
+      "rewards/margins": 0.3466815948486328,
+      "rewards/rejected": -2.4913668632507324,
+      "step": 10640
+    },
+    {
+      "epoch": 1.8349414197105445,
+      "grad_norm": 12.052877780205732,
+      "learning_rate": 1.0302718834909213e-09,
+      "logits/chosen": -3.3956596851348877,
+      "logits/rejected": -3.384077787399292,
+      "logps/chosen": -2.0636467933654785,
+      "logps/rejected": -2.4027602672576904,
+      "loss": 1.0083,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -2.0636467933654785,
+      "rewards/margins": 0.33911341428756714,
+      "rewards/rejected": -2.4027602672576904,
+      "step": 10650
+    },
+    {
+      "epoch": 1.8366643694004137,
+      "grad_norm": 12.739380255035373,
+      "learning_rate": 1.0090204531187168e-09,
+      "logits/chosen": -3.3551197052001953,
+      "logits/rejected": -3.3493106365203857,
+      "logps/chosen": -2.171748399734497,
+      "logps/rejected": -2.4226536750793457,
+      "loss": 1.0741,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -2.171748399734497,
+      "rewards/margins": 0.25090548396110535,
+      "rewards/rejected": -2.4226536750793457,
+      "step": 10660
+    },
+    {
+      "epoch": 1.8383873190902826,
+      "grad_norm": 14.324246257764774,
+      "learning_rate": 9.8798597502181e-10,
+      "logits/chosen": -3.3596701622009277,
+      "logits/rejected": -3.3516311645507812,
+      "logps/chosen": -2.157257556915283,
+      "logps/rejected": -2.4818389415740967,
+      "loss": 1.0572,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -2.157257556915283,
+      "rewards/margins": 0.3245818316936493,
+      "rewards/rejected": -2.4818389415740967,
+      "step": 10670
+    },
+    {
+      "epoch": 1.8401102687801516,
+      "grad_norm": 12.548697617476604,
+      "learning_rate": 9.671686394166156e-10,
+      "logits/chosen": -3.3716468811035156,
+      "logits/rejected": -3.3475215435028076,
+      "logps/chosen": -2.094630241394043,
+      "logps/rejected": -2.4450325965881348,
+      "loss": 1.0306,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -2.094630241394043,
+      "rewards/margins": 0.35040196776390076,
+      "rewards/rejected": -2.4450325965881348,
+      "step": 10680
+    },
+    {
+      "epoch": 1.8418332184700206,
+      "grad_norm": 13.443076685874841,
+      "learning_rate": 9.465686345558944e-10,
+      "logits/chosen": -3.369330883026123,
+      "logits/rejected": -3.3655285835266113,
+      "logps/chosen": -2.074748992919922,
+      "logps/rejected": -2.3235344886779785,
+      "loss": 1.0788,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -2.074748992919922,
+      "rewards/margins": 0.24878545105457306,
+      "rewards/rejected": -2.3235344886779785,
+      "step": 10690
+    },
+    {
+      "epoch": 1.8435561681598898,
+      "grad_norm": 11.658427755866073,
+      "learning_rate": 9.261861467270788e-10,
+      "logits/chosen": -3.4064345359802246,
+      "logits/rejected": -3.3883044719696045,
+      "logps/chosen": -2.0625128746032715,
+      "logps/rejected": -2.395512104034424,
+      "loss": 1.0259,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -2.0625128746032715,
+      "rewards/margins": 0.3329991102218628,
+      "rewards/rejected": -2.395512104034424,
+      "step": 10700
+    },
+    {
+      "epoch": 1.8435561681598898,
+      "eval_logits/chosen": -3.4368579387664795,
+      "eval_logits/rejected": -3.433255672454834,
+      "eval_logps/chosen": -1.9030276536941528,
+      "eval_logps/rejected": -2.097609281539917,
+      "eval_loss": 1.087764024734497,
+      "eval_rewards/accuracies": 0.6219795346260071,
+      "eval_rewards/chosen": -1.9030276536941528,
+      "eval_rewards/margins": 0.19458140432834625,
+      "eval_rewards/rejected": -2.097609281539917,
+      "eval_runtime": 155.2444,
+      "eval_samples_per_second": 27.724,
+      "eval_steps_per_second": 3.466,
+      "step": 10700
+    },
+    {
+      "epoch": 1.8452791178497587,
+      "grad_norm": 11.712105891650838,
+      "learning_rate": 9.060213602505778e-10,
+      "logits/chosen": -3.3562419414520264,
+      "logits/rejected": -3.341096878051758,
+      "logps/chosen": -2.0797438621520996,
+      "logps/rejected": -2.3550209999084473,
+      "loss": 1.0651,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -2.0797438621520996,
+      "rewards/margins": 0.2752768099308014,
+      "rewards/rejected": -2.3550209999084473,
+      "step": 10710
+    },
+    {
+      "epoch": 1.847002067539628,
+      "grad_norm": 13.001792388243752,
+      "learning_rate": 8.860744574781032e-10,
+      "logits/chosen": -3.3692259788513184,
+      "logits/rejected": -3.353421449661255,
+      "logps/chosen": -2.1292169094085693,
+      "logps/rejected": -2.528550624847412,
+      "loss": 1.0364,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -2.1292169094085693,
+      "rewards/margins": 0.39933347702026367,
+      "rewards/rejected": -2.528550624847412,
+      "step": 10720
+    },
+    {
+      "epoch": 1.848725017229497,
+      "grad_norm": 12.828649129415584,
+      "learning_rate": 8.663456187910423e-10,
+      "logits/chosen": -3.4019572734832764,
+      "logits/rejected": -3.3832669258117676,
+      "logps/chosen": -2.099780797958374,
+      "logps/rejected": -2.375732660293579,
+      "loss": 1.064,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -2.099780797958374,
+      "rewards/margins": 0.27595168352127075,
+      "rewards/rejected": -2.375732660293579,
+      "step": 10730
+    },
+    {
+      "epoch": 1.8504479669193659,
+      "grad_norm": 14.341807339896812,
+      "learning_rate": 8.468350225987909e-10,
+      "logits/chosen": -3.339916944503784,
+      "logits/rejected": -3.3359158039093018,
+      "logps/chosen": -2.1776702404022217,
+      "logps/rejected": -2.292058229446411,
+      "loss": 1.1516,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -2.1776702404022217,
+      "rewards/margins": 0.1143876165151596,
+      "rewards/rejected": -2.292058229446411,
+      "step": 10740
+    },
+    {
+      "epoch": 1.852170916609235,
+      "grad_norm": 11.028510510773259,
+      "learning_rate": 8.275428453371813e-10,
+      "logits/chosen": -3.3471245765686035,
+      "logits/rejected": -3.331299304962158,
+      "logps/chosen": -2.029507637023926,
+      "logps/rejected": -2.444199800491333,
+      "loss": 0.977,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -2.029507637023926,
+      "rewards/margins": 0.41469234228134155,
+      "rewards/rejected": -2.444199800491333,
+      "step": 10750
+    },
+    {
+      "epoch": 1.853893866299104,
+      "grad_norm": 13.92690698586694,
+      "learning_rate": 8.084692614668543e-10,
+      "logits/chosen": -3.352430820465088,
+      "logits/rejected": -3.3421454429626465,
+      "logps/chosen": -2.054394245147705,
+      "logps/rejected": -2.320951461791992,
+      "loss": 1.0622,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -2.054394245147705,
+      "rewards/margins": 0.2665572762489319,
+      "rewards/rejected": -2.320951461791992,
+      "step": 10760
+    },
+    {
+      "epoch": 1.8556168159889732,
+      "grad_norm": 11.138978534450118,
+      "learning_rate": 7.896144434716951e-10,
+      "logits/chosen": -3.3435721397399902,
+      "logits/rejected": -3.335465669631958,
+      "logps/chosen": -2.071582317352295,
+      "logps/rejected": -2.3845934867858887,
+      "loss": 1.0603,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -2.071582317352295,
+      "rewards/margins": 0.3130114674568176,
+      "rewards/rejected": -2.3845934867858887,
+      "step": 10770
+    },
+    {
+      "epoch": 1.8573397656788422,
+      "grad_norm": 12.47103698230256,
+      "learning_rate": 7.709785618572801e-10,
+      "logits/chosen": -3.4187636375427246,
+      "logits/rejected": -3.40879487991333,
+      "logps/chosen": -2.0638985633850098,
+      "logps/rejected": -2.364750862121582,
+      "loss": 1.0406,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -2.0638985633850098,
+      "rewards/margins": 0.30085259675979614,
+      "rewards/rejected": -2.364750862121582,
+      "step": 10780
+    },
+    {
+      "epoch": 1.8590627153687111,
+      "grad_norm": 13.19571125899579,
+      "learning_rate": 7.525617851493166e-10,
+      "logits/chosen": -3.4133524894714355,
+      "logits/rejected": -3.3992912769317627,
+      "logps/chosen": -2.010606288909912,
+      "logps/rejected": -2.454699754714966,
+      "loss": 0.9923,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -2.010606288909912,
+      "rewards/margins": 0.4440933167934418,
+      "rewards/rejected": -2.454699754714966,
+      "step": 10790
+    },
+    {
+      "epoch": 1.8607856650585803,
+      "grad_norm": 13.68348055266554,
+      "learning_rate": 7.343642798921384e-10,
+      "logits/chosen": -3.421435832977295,
+      "logits/rejected": -3.411466121673584,
+      "logps/chosen": -2.0860986709594727,
+      "logps/rejected": -2.5100345611572266,
+      "loss": 0.9896,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -2.0860986709594727,
+      "rewards/margins": 0.42393580079078674,
+      "rewards/rejected": -2.5100345611572266,
+      "step": 10800
+    },
+    {
+      "epoch": 1.8607856650585803,
+      "eval_logits/chosen": -3.434206247329712,
+      "eval_logits/rejected": -3.430586576461792,
+      "eval_logps/chosen": -1.9030910730361938,
+      "eval_logps/rejected": -2.097501039505005,
+      "eval_loss": 1.0877608060836792,
+      "eval_rewards/accuracies": 0.6229089498519897,
+      "eval_rewards/chosen": -1.9030910730361938,
+      "eval_rewards/margins": 0.19441010057926178,
+      "eval_rewards/rejected": -2.097501039505005,
+      "eval_runtime": 155.2607,
+      "eval_samples_per_second": 27.721,
+      "eval_steps_per_second": 3.465,
+      "step": 10800
+    },
+    {
+      "epoch": 1.8625086147484493,
+      "grad_norm": 11.189036245892867,
+      "learning_rate": 7.163862106471852e-10,
+      "logits/chosen": -3.372267484664917,
+      "logits/rejected": -3.3510661125183105,
+      "logps/chosen": -2.096553325653076,
+      "logps/rejected": -2.4203038215637207,
+      "loss": 1.0424,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -2.096553325653076,
+      "rewards/margins": 0.3237501084804535,
+      "rewards/rejected": -2.4203038215637207,
+      "step": 10810
+    },
+    {
+      "epoch": 1.8642315644383185,
+      "grad_norm": 13.737382705498405,
+      "learning_rate": 6.986277399915197e-10,
+      "logits/chosen": -3.3400790691375732,
+      "logits/rejected": -3.32795786857605,
+      "logps/chosen": -2.0366899967193604,
+      "logps/rejected": -2.3653857707977295,
+      "loss": 1.0524,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -2.0366899967193604,
+      "rewards/margins": 0.32869598269462585,
+      "rewards/rejected": -2.3653857707977295,
+      "step": 10820
+    },
+    {
+      "epoch": 1.8659545141281875,
+      "grad_norm": 15.019474669192983,
+      "learning_rate": 6.810890285163628e-10,
+      "logits/chosen": -3.3526294231414795,
+      "logits/rejected": -3.337596893310547,
+      "logps/chosen": -2.0863125324249268,
+      "logps/rejected": -2.4578702449798584,
+      "loss": 0.9998,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -2.0863125324249268,
+      "rewards/margins": 0.3715580403804779,
+      "rewards/rejected": -2.4578702449798584,
+      "step": 10830
+    },
+    {
+      "epoch": 1.8676774638180564,
+      "grad_norm": 12.491930475012577,
+      "learning_rate": 6.637702348256307e-10,
+      "logits/chosen": -3.3891868591308594,
+      "logits/rejected": -3.372649669647217,
+      "logps/chosen": -2.0170319080352783,
+      "logps/rejected": -2.403036594390869,
+      "loss": 1.0428,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -2.0170319080352783,
+      "rewards/margins": 0.38600438833236694,
+      "rewards/rejected": -2.403036594390869,
+      "step": 10840
+    },
+    {
+      "epoch": 1.8694004135079254,
+      "grad_norm": 12.348477226386583,
+      "learning_rate": 6.466715155345109e-10,
+      "logits/chosen": -3.3253891468048096,
+      "logits/rejected": -3.3204426765441895,
+      "logps/chosen": -2.018181562423706,
+      "logps/rejected": -2.334841012954712,
+      "loss": 1.0372,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -2.018181562423706,
+      "rewards/margins": 0.31665918231010437,
+      "rewards/rejected": -2.334841012954712,
+      "step": 10850
+    },
+    {
+      "epoch": 1.8711233631977946,
+      "grad_norm": 13.02265499331817,
+      "learning_rate": 6.2979302526803e-10,
+      "logits/chosen": -3.4152603149414062,
+      "logits/rejected": -3.398376941680908,
+      "logps/chosen": -2.0805695056915283,
+      "logps/rejected": -2.337893486022949,
+      "loss": 1.0622,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -2.0805695056915283,
+      "rewards/margins": 0.2573241591453552,
+      "rewards/rejected": -2.337893486022949,
+      "step": 10860
+    },
+    {
+      "epoch": 1.8728463128876638,
+      "grad_norm": 12.285916203888767,
+      "learning_rate": 6.131349166596883e-10,
+      "logits/chosen": -3.3103365898132324,
+      "logits/rejected": -3.3081138134002686,
+      "logps/chosen": -2.072908639907837,
+      "logps/rejected": -2.42985200881958,
+      "loss": 1.0057,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -2.072908639907837,
+      "rewards/margins": 0.35694319009780884,
+      "rewards/rejected": -2.42985200881958,
+      "step": 10870
+    },
+    {
+      "epoch": 1.8745692625775328,
+      "grad_norm": 13.367433560629292,
+      "learning_rate": 5.966973403500303e-10,
+      "logits/chosen": -3.3695406913757324,
+      "logits/rejected": -3.3593716621398926,
+      "logps/chosen": -2.1072018146514893,
+      "logps/rejected": -2.3483669757843018,
+      "loss": 1.0773,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -2.1072018146514893,
+      "rewards/margins": 0.2411651611328125,
+      "rewards/rejected": -2.3483669757843018,
+      "step": 10880
+    },
+    {
+      "epoch": 1.8762922122674017,
+      "grad_norm": 11.791152424004396,
+      "learning_rate": 5.804804449853401e-10,
+      "logits/chosen": -3.394758939743042,
+      "logits/rejected": -3.3861842155456543,
+      "logps/chosen": -2.0737011432647705,
+      "logps/rejected": -2.425340414047241,
+      "loss": 1.0114,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -2.0737011432647705,
+      "rewards/margins": 0.35163959860801697,
+      "rewards/rejected": -2.425340414047241,
+      "step": 10890
+    },
+    {
+      "epoch": 1.8780151619572707,
+      "grad_norm": 11.173073296395863,
+      "learning_rate": 5.644843772162373e-10,
+      "logits/chosen": -3.4298863410949707,
+      "logits/rejected": -3.4135594367980957,
+      "logps/chosen": -2.002847909927368,
+      "logps/rejected": -2.2676548957824707,
+      "loss": 1.0559,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -2.002847909927368,
+      "rewards/margins": 0.26480674743652344,
+      "rewards/rejected": -2.2676548957824707,
+      "step": 10900
+    },
+    {
+      "epoch": 1.8780151619572707,
+      "eval_logits/chosen": -3.428328275680542,
+      "eval_logits/rejected": -3.424682378768921,
+      "eval_logps/chosen": -1.9023834466934204,
+      "eval_logps/rejected": -2.097043514251709,
+      "eval_loss": 1.0876299142837524,
+      "eval_rewards/accuracies": 0.6233736276626587,
+      "eval_rewards/chosen": -1.9023834466934204,
+      "eval_rewards/margins": 0.19466012716293335,
+      "eval_rewards/rejected": -2.097043514251709,
+      "eval_runtime": 155.2634,
+      "eval_samples_per_second": 27.721,
+      "eval_steps_per_second": 3.465,
+      "step": 10900
+    },
+    {
+      "epoch": 1.8797381116471399,
+      "grad_norm": 11.547971180161776,
+      "learning_rate": 5.487092816963995e-10,
+      "logits/chosen": -3.366682767868042,
+      "logits/rejected": -3.3501670360565186,
+      "logps/chosen": -2.049309730529785,
+      "logps/rejected": -2.4726383686065674,
+      "loss": 0.9857,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -2.049309730529785,
+      "rewards/margins": 0.42332854866981506,
+      "rewards/rejected": -2.4726383686065674,
+      "step": 10910
+    },
+    {
+      "epoch": 1.881461061337009,
+      "grad_norm": 11.715032497793327,
+      "learning_rate": 5.331553010812311e-10,
+      "logits/chosen": -3.3498311042785645,
+      "logits/rejected": -3.3381659984588623,
+      "logps/chosen": -2.1523618698120117,
+      "logps/rejected": -2.4105031490325928,
+      "loss": 1.0611,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -2.1523618698120117,
+      "rewards/margins": 0.2581414580345154,
+      "rewards/rejected": -2.4105031490325928,
+      "step": 10920
+    },
+    {
+      "epoch": 1.883184011026878,
+      "grad_norm": 14.096328187442833,
+      "learning_rate": 5.178225760265775e-10,
+      "logits/chosen": -3.345090389251709,
+      "logits/rejected": -3.3289718627929688,
+      "logps/chosen": -2.0852391719818115,
+      "logps/rejected": -2.4490764141082764,
+      "loss": 1.0155,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -2.0852391719818115,
+      "rewards/margins": 0.36383721232414246,
+      "rewards/rejected": -2.4490764141082764,
+      "step": 10930
+    },
+    {
+      "epoch": 1.884906960716747,
+      "grad_norm": 14.505781617628099,
+      "learning_rate": 5.027112451874482e-10,
+      "logits/chosen": -3.305384397506714,
+      "logits/rejected": -3.2950267791748047,
+      "logps/chosen": -2.1521658897399902,
+      "logps/rejected": -2.4319567680358887,
+      "loss": 1.0613,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -2.1521658897399902,
+      "rewards/margins": 0.27979087829589844,
+      "rewards/rejected": -2.4319567680358887,
+      "step": 10940
+    },
+    {
+      "epoch": 1.886629910406616,
+      "grad_norm": 12.133928218458395,
+      "learning_rate": 4.87821445216774e-10,
+      "logits/chosen": -3.334432601928711,
+      "logits/rejected": -3.3244426250457764,
+      "logps/chosen": -2.1142685413360596,
+      "logps/rejected": -2.4086549282073975,
+      "loss": 1.0488,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -2.1142685413360596,
+      "rewards/margins": 0.29438644647598267,
+      "rewards/rejected": -2.4086549282073975,
+      "step": 10950
+    },
+    {
+      "epoch": 1.8883528600964852,
+      "grad_norm": 13.590226053086653,
+      "learning_rate": 4.731533107641627e-10,
+      "logits/chosen": -3.3870062828063965,
+      "logits/rejected": -3.37575101852417,
+      "logps/chosen": -2.070929527282715,
+      "logps/rejected": -2.389671802520752,
+      "loss": 1.0326,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -2.070929527282715,
+      "rewards/margins": 0.3187422752380371,
+      "rewards/rejected": -2.389671802520752,
+      "step": 10960
+    },
+    {
+      "epoch": 1.8900758097863544,
+      "grad_norm": 12.327714012349615,
+      "learning_rate": 4.587069744746791e-10,
+      "logits/chosen": -3.392911911010742,
+      "logits/rejected": -3.3770089149475098,
+      "logps/chosen": -2.004321575164795,
+      "logps/rejected": -2.3203532695770264,
+      "loss": 1.0254,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -2.004321575164795,
+      "rewards/margins": 0.3160319924354553,
+      "rewards/rejected": -2.3203532695770264,
+      "step": 10970
+    },
+    {
+      "epoch": 1.8917987594762233,
+      "grad_norm": 11.663159407249708,
+      "learning_rate": 4.4448256698766393e-10,
+      "logits/chosen": -3.3813071250915527,
+      "logits/rejected": -3.375123977661133,
+      "logps/chosen": -2.102527141571045,
+      "logps/rejected": -2.434429168701172,
+      "loss": 1.0311,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -2.102527141571045,
+      "rewards/margins": 0.33190178871154785,
+      "rewards/rejected": -2.434429168701172,
+      "step": 10980
+    },
+    {
+      "epoch": 1.8935217091660923,
+      "grad_norm": 10.549243043780942,
+      "learning_rate": 4.3048021693552206e-10,
+      "logits/chosen": -3.3315024375915527,
+      "logits/rejected": -3.32507061958313,
+      "logps/chosen": -2.128762722015381,
+      "logps/rejected": -2.4148340225219727,
+      "loss": 1.0572,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -2.128762722015381,
+      "rewards/margins": 0.2860715389251709,
+      "rewards/rejected": -2.4148340225219727,
+      "step": 10990
+    },
+    {
+      "epoch": 1.8952446588559613,
+      "grad_norm": 14.19444931938008,
+      "learning_rate": 4.167000509425811e-10,
+      "logits/chosen": -3.4404704570770264,
+      "logits/rejected": -3.4350407123565674,
+      "logps/chosen": -2.1696078777313232,
+      "logps/rejected": -2.4388937950134277,
+      "loss": 1.0904,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -2.1696078777313232,
+      "rewards/margins": 0.26928597688674927,
+      "rewards/rejected": -2.4388937950134277,
+      "step": 11000
+    },
+    {
+      "epoch": 1.8952446588559613,
+      "eval_logits/chosen": -3.436129570007324,
+      "eval_logits/rejected": -3.4325172901153564,
+      "eval_logps/chosen": -1.902899146080017,
+      "eval_logps/rejected": -2.09749436378479,
+      "eval_loss": 1.0877759456634521,
+      "eval_rewards/accuracies": 0.6236059665679932,
+      "eval_rewards/chosen": -1.902899146080017,
+      "eval_rewards/margins": 0.19459529221057892,
+      "eval_rewards/rejected": -2.09749436378479,
+      "eval_runtime": 155.2335,
+      "eval_samples_per_second": 27.726,
+      "eval_steps_per_second": 3.466,
+      "step": 11000
+    },
+    {
+      "epoch": 1.8969676085458305,
+      "grad_norm": 12.778534419316642,
+      "learning_rate": 4.0314219362395095e-10,
+      "logits/chosen": -3.408313274383545,
+      "logits/rejected": -3.391356945037842,
+      "logps/chosen": -2.007122755050659,
+      "logps/rejected": -2.4340591430664062,
+      "loss": 0.973,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -2.007122755050659,
+      "rewards/margins": 0.4269363284111023,
+      "rewards/rejected": -2.4340591430664062,
+      "step": 11010
+    },
+    {
+      "epoch": 1.8986905582356997,
+      "grad_norm": 14.663347667183158,
+      "learning_rate": 3.898067675843747e-10,
+      "logits/chosen": -3.407651424407959,
+      "logits/rejected": -3.398714780807495,
+      "logps/chosen": -2.035029888153076,
+      "logps/rejected": -2.312566041946411,
+      "loss": 1.0645,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -2.035029888153076,
+      "rewards/margins": 0.27753573656082153,
+      "rewards/rejected": -2.312566041946411,
+      "step": 11020
+    },
+    {
+      "epoch": 1.9004135079255686,
+      "grad_norm": 13.966954504436922,
+      "learning_rate": 3.766938934171349e-10,
+      "logits/chosen": -3.384514570236206,
+      "logits/rejected": -3.380664348602295,
+      "logps/chosen": -2.202213764190674,
+      "logps/rejected": -2.5059237480163574,
+      "loss": 1.0576,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -2.202213764190674,
+      "rewards/margins": 0.30371007323265076,
+      "rewards/rejected": -2.5059237480163574,
+      "step": 11030
+    },
+    {
+      "epoch": 1.9021364576154376,
+      "grad_norm": 15.52673700408528,
+      "learning_rate": 3.6380368970296836e-10,
+      "logits/chosen": -3.399095058441162,
+      "logits/rejected": -3.3930277824401855,
+      "logps/chosen": -2.2382025718688965,
+      "logps/rejected": -2.4566731452941895,
+      "loss": 1.1013,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -2.2382025718688965,
+      "rewards/margins": 0.21847081184387207,
+      "rewards/rejected": -2.4566731452941895,
+      "step": 11040
+    },
+    {
+      "epoch": 1.9038594073053066,
+      "grad_norm": 12.644978664461455,
+      "learning_rate": 3.5113627300897284e-10,
+      "logits/chosen": -3.3537259101867676,
+      "logits/rejected": -3.34211802482605,
+      "logps/chosen": -2.012291669845581,
+      "logps/rejected": -2.38850998878479,
+      "loss": 1.0042,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -2.012291669845581,
+      "rewards/margins": 0.3762182295322418,
+      "rewards/rejected": -2.38850998878479,
+      "step": 11050
+    },
+    {
+      "epoch": 1.9055823569951758,
+      "grad_norm": 13.044225654987311,
+      "learning_rate": 3.38691757887577e-10,
+      "logits/chosen": -3.387716293334961,
+      "logits/rejected": -3.3666560649871826,
+      "logps/chosen": -2.051743507385254,
+      "logps/rejected": -2.4177958965301514,
+      "loss": 1.0053,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -2.051743507385254,
+      "rewards/margins": 0.3660527765750885,
+      "rewards/rejected": -2.4177958965301514,
+      "step": 11060
+    },
+    {
+      "epoch": 1.907305306685045,
+      "grad_norm": 11.376074129403003,
+      "learning_rate": 3.264702568754912e-10,
+      "logits/chosen": -3.38842511177063,
+      "logits/rejected": -3.363692045211792,
+      "logps/chosen": -1.9596443176269531,
+      "logps/rejected": -2.4868717193603516,
+      "loss": 0.9384,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.9596443176269531,
+      "rewards/margins": 0.5272272229194641,
+      "rewards/rejected": -2.4868717193603516,
+      "step": 11070
+    },
+    {
+      "epoch": 1.909028256374914,
+      "grad_norm": 11.727642258376596,
+      "learning_rate": 3.1447188049268656e-10,
+      "logits/chosen": -3.3900771141052246,
+      "logits/rejected": -3.376296281814575,
+      "logps/chosen": -2.103996753692627,
+      "logps/rejected": -2.425795555114746,
+      "loss": 1.0182,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -2.103996753692627,
+      "rewards/margins": 0.32179901003837585,
+      "rewards/rejected": -2.425795555114746,
+      "step": 11080
+    },
+    {
+      "epoch": 1.9107512060647829,
+      "grad_norm": 14.76658166292719,
+      "learning_rate": 3.0269673724140353e-10,
+      "logits/chosen": -3.3710951805114746,
+      "logits/rejected": -3.361649990081787,
+      "logps/chosen": -2.156318426132202,
+      "logps/rejected": -2.305778741836548,
+      "loss": 1.1534,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -2.156318426132202,
+      "rewards/margins": 0.1494603157043457,
+      "rewards/rejected": -2.305778741836548,
+      "step": 11090
+    },
+    {
+      "epoch": 1.9124741557546519,
+      "grad_norm": 11.207632605415386,
+      "learning_rate": 2.9114493360517243e-10,
+      "logits/chosen": -3.3156726360321045,
+      "logits/rejected": -3.3075993061065674,
+      "logps/chosen": -2.0650382041931152,
+      "logps/rejected": -2.3682491779327393,
+      "loss": 1.0518,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -2.0650382041931152,
+      "rewards/margins": 0.30321088433265686,
+      "rewards/rejected": -2.3682491779327393,
+      "step": 11100
+    },
+    {
+      "epoch": 1.9124741557546519,
+      "eval_logits/chosen": -3.4272027015686035,
+      "eval_logits/rejected": -3.4235482215881348,
+      "eval_logps/chosen": -1.9026610851287842,
+      "eval_logps/rejected": -2.0973074436187744,
+      "eval_loss": 1.087652325630188,
+      "eval_rewards/accuracies": 0.6233736276626587,
+      "eval_rewards/chosen": -1.9026610851287842,
+      "eval_rewards/margins": 0.19464656710624695,
+      "eval_rewards/rejected": -2.0973074436187744,
+      "eval_runtime": 155.1838,
+      "eval_samples_per_second": 27.735,
+      "eval_steps_per_second": 3.467,
+      "step": 11100
+    },
+    {
+      "epoch": 1.914197105444521,
+      "grad_norm": 11.843811806581604,
+      "learning_rate": 2.79816574047842e-10,
+      "logits/chosen": -3.3977198600769043,
+      "logits/rejected": -3.3867850303649902,
+      "logps/chosen": -2.1600794792175293,
+      "logps/rejected": -2.5312583446502686,
+      "loss": 1.0214,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -2.1600794792175293,
+      "rewards/margins": 0.37117916345596313,
+      "rewards/rejected": -2.5312583446502686,
+      "step": 11110
+    },
+    {
+      "epoch": 1.9159200551343902,
+      "grad_norm": 11.508825776190255,
+      "learning_rate": 2.6871176101263826e-10,
+      "logits/chosen": -3.44368314743042,
+      "logits/rejected": -3.427811861038208,
+      "logps/chosen": -2.1585259437561035,
+      "logps/rejected": -2.452444076538086,
+      "loss": 1.0511,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -2.1585259437561035,
+      "rewards/margins": 0.2939181327819824,
+      "rewards/rejected": -2.452444076538086,
+      "step": 11120
+    },
+    {
+      "epoch": 1.9176430048242592,
+      "grad_norm": 13.994713761810102,
+      "learning_rate": 2.5783059492124335e-10,
+      "logits/chosen": -3.338284969329834,
+      "logits/rejected": -3.3270678520202637,
+      "logps/chosen": -2.1920764446258545,
+      "logps/rejected": -2.4402761459350586,
+      "loss": 1.0787,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -2.1920764446258545,
+      "rewards/margins": 0.24819938838481903,
+      "rewards/rejected": -2.4402761459350586,
+      "step": 11130
+    },
+    {
+      "epoch": 1.9193659545141282,
+      "grad_norm": 11.891007268508062,
+      "learning_rate": 2.471731741728794e-10,
+      "logits/chosen": -3.324566602706909,
+      "logits/rejected": -3.313788890838623,
+      "logps/chosen": -2.032819986343384,
+      "logps/rejected": -2.3811733722686768,
+      "loss": 1.0301,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -2.032819986343384,
+      "rewards/margins": 0.3483535051345825,
+      "rewards/rejected": -2.3811733722686768,
+      "step": 11140
+    },
+    {
+      "epoch": 1.9210889042039971,
+      "grad_norm": 12.604402683505741,
+      "learning_rate": 2.367395951434231e-10,
+      "logits/chosen": -3.3702826499938965,
+      "logits/rejected": -3.3622519969940186,
+      "logps/chosen": -2.1398637294769287,
+      "logps/rejected": -2.4212422370910645,
+      "loss": 1.0621,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -2.1398637294769287,
+      "rewards/margins": 0.28137844800949097,
+      "rewards/rejected": -2.4212422370910645,
+      "step": 11150
+    },
+    {
+      "epoch": 1.9228118538938663,
+      "grad_norm": 11.962416859186373,
+      "learning_rate": 2.2652995218452876e-10,
+      "logits/chosen": -3.429924726486206,
+      "logits/rejected": -3.4192497730255127,
+      "logps/chosen": -1.9931097030639648,
+      "logps/rejected": -2.229854106903076,
+      "loss": 1.0905,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.9931097030639648,
+      "rewards/margins": 0.23674440383911133,
+      "rewards/rejected": -2.229854106903076,
+      "step": 11160
+    },
+    {
+      "epoch": 1.9245348035837355,
+      "grad_norm": 12.14694789899942,
+      "learning_rate": 2.1654433762278713e-10,
+      "logits/chosen": -3.3314127922058105,
+      "logits/rejected": -3.316654682159424,
+      "logps/chosen": -2.120065927505493,
+      "logps/rejected": -2.278688430786133,
+      "loss": 1.133,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -2.120065927505493,
+      "rewards/margins": 0.1586225926876068,
+      "rewards/rejected": -2.278688430786133,
+      "step": 11170
+    },
+    {
+      "epoch": 1.9262577532736045,
+      "grad_norm": 12.87715248759552,
+      "learning_rate": 2.0678284175887906e-10,
+      "logits/chosen": -3.4023120403289795,
+      "logits/rejected": -3.389636993408203,
+      "logps/chosen": -2.0793826580047607,
+      "logps/rejected": -2.4095470905303955,
+      "loss": 1.0442,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -2.0793826580047607,
+      "rewards/margins": 0.33016437292099,
+      "rewards/rejected": -2.4095470905303955,
+      "step": 11180
+    },
+    {
+      "epoch": 1.9279807029634735,
+      "grad_norm": 16.56148596443503,
+      "learning_rate": 1.972455528667677e-10,
+      "logits/chosen": -3.4009487628936768,
+      "logits/rejected": -3.3848464488983154,
+      "logps/chosen": -2.0897891521453857,
+      "logps/rejected": -2.4557230472564697,
+      "loss": 1.0207,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -2.0897891521453857,
+      "rewards/margins": 0.3659338355064392,
+      "rewards/rejected": -2.4557230472564697,
+      "step": 11190
+    },
+    {
+      "epoch": 1.9297036526533424,
+      "grad_norm": 13.521642000363293,
+      "learning_rate": 1.8793255719288248e-10,
+      "logits/chosen": -3.3953139781951904,
+      "logits/rejected": -3.3836302757263184,
+      "logps/chosen": -2.0078935623168945,
+      "logps/rejected": -2.334378242492676,
+      "loss": 1.0111,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -2.0078935623168945,
+      "rewards/margins": 0.326484739780426,
+      "rewards/rejected": -2.334378242492676,
+      "step": 11200
+    },
+    {
+      "epoch": 1.9297036526533424,
+      "eval_logits/chosen": -3.4233367443084717,
+      "eval_logits/rejected": -3.419663190841675,
+      "eval_logps/chosen": -1.9032413959503174,
+      "eval_logps/rejected": -2.0975701808929443,
+      "eval_loss": 1.0878137350082397,
+      "eval_rewards/accuracies": 0.6231412887573242,
+      "eval_rewards/chosen": -1.9032413959503174,
+      "eval_rewards/margins": 0.19432875514030457,
+      "eval_rewards/rejected": -2.0975701808929443,
+      "eval_runtime": 155.3182,
+      "eval_samples_per_second": 27.711,
+      "eval_steps_per_second": 3.464,
+      "step": 11200
+    },
+    {
+      "epoch": 1.9314266023432116,
+      "grad_norm": 12.13145523282108,
+      "learning_rate": 1.7884393895536697e-10,
+      "logits/chosen": -3.3061413764953613,
+      "logits/rejected": -3.2975776195526123,
+      "logps/chosen": -2.1401844024658203,
+      "logps/rejected": -2.4794278144836426,
+      "loss": 1.0289,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -2.1401844024658203,
+      "rewards/margins": 0.33924344182014465,
+      "rewards/rejected": -2.4794278144836426,
+      "step": 11210
+    },
+    {
+      "epoch": 1.9331495520330806,
+      "grad_norm": 11.430268319012553,
+      "learning_rate": 1.6997978034329342e-10,
+      "logits/chosen": -3.3808796405792236,
+      "logits/rejected": -3.3743369579315186,
+      "logps/chosen": -2.007667064666748,
+      "logps/rejected": -2.350980281829834,
+      "loss": 1.0316,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -2.007667064666748,
+      "rewards/margins": 0.3433131277561188,
+      "rewards/rejected": -2.350980281829834,
+      "step": 11220
+    },
+    {
+      "epoch": 1.9348725017229498,
+      "grad_norm": 12.23928191397691,
+      "learning_rate": 1.6134016151592988e-10,
+      "logits/chosen": -3.3675129413604736,
+      "logits/rejected": -3.3612351417541504,
+      "logps/chosen": -2.2054741382598877,
+      "logps/rejected": -2.436591625213623,
+      "loss": 1.1432,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -2.2054741382598877,
+      "rewards/margins": 0.23111757636070251,
+      "rewards/rejected": -2.436591625213623,
+      "step": 11230
+    },
+    {
+      "epoch": 1.9365954514128187,
+      "grad_norm": 15.243589186623765,
+      "learning_rate": 1.5292516060201598e-10,
+      "logits/chosen": -3.3532981872558594,
+      "logits/rejected": -3.345937728881836,
+      "logps/chosen": -2.1670279502868652,
+      "logps/rejected": -2.4089243412017822,
+      "loss": 1.1361,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -2.1670279502868652,
+      "rewards/margins": 0.24189671874046326,
+      "rewards/rejected": -2.4089243412017822,
+      "step": 11240
+    },
+    {
+      "epoch": 1.9383184011026877,
+      "grad_norm": 15.25888243596088,
+      "learning_rate": 1.4473485369905225e-10,
+      "logits/chosen": -3.3725922107696533,
+      "logits/rejected": -3.3594326972961426,
+      "logps/chosen": -2.1099636554718018,
+      "logps/rejected": -2.4158244132995605,
+      "loss": 1.0381,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -2.1099636554718018,
+      "rewards/margins": 0.3058607876300812,
+      "rewards/rejected": -2.4158244132995605,
+      "step": 11250
+    },
+    {
+      "epoch": 1.940041350792557,
+      "grad_norm": 13.994328648894102,
+      "learning_rate": 1.3676931487261456e-10,
+      "logits/chosen": -3.3330256938934326,
+      "logits/rejected": -3.3156158924102783,
+      "logps/chosen": -2.122960090637207,
+      "logps/rejected": -2.4536781311035156,
+      "loss": 1.0593,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -2.122960090637207,
+      "rewards/margins": 0.3307178020477295,
+      "rewards/rejected": -2.4536781311035156,
+      "step": 11260
+    },
+    {
+      "epoch": 1.9417643004824259,
+      "grad_norm": 12.930594514808007,
+      "learning_rate": 1.2902861615568529e-10,
+      "logits/chosen": -3.3748440742492676,
+      "logits/rejected": -3.358053207397461,
+      "logps/chosen": -2.1334071159362793,
+      "logps/rejected": -2.384780168533325,
+      "loss": 1.0713,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -2.1334071159362793,
+      "rewards/margins": 0.25137320160865784,
+      "rewards/rejected": -2.384780168533325,
+      "step": 11270
+    },
+    {
+      "epoch": 1.943487250172295,
+      "grad_norm": 11.964413234714861,
+      "learning_rate": 1.215128275479954e-10,
+      "logits/chosen": -3.4035720825195312,
+      "logits/rejected": -3.389472246170044,
+      "logps/chosen": -2.105055093765259,
+      "logps/rejected": -2.294032573699951,
+      "loss": 1.0976,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -2.105055093765259,
+      "rewards/margins": 0.1889771968126297,
+      "rewards/rejected": -2.294032573699951,
+      "step": 11280
+    },
+    {
+      "epoch": 1.945210199862164,
+      "grad_norm": 13.244452370457006,
+      "learning_rate": 1.1422201701540569e-10,
+      "logits/chosen": -3.378789186477661,
+      "logits/rejected": -3.3690407276153564,
+      "logps/chosen": -2.125610113143921,
+      "logps/rejected": -2.420356512069702,
+      "loss": 1.0663,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -2.125610113143921,
+      "rewards/margins": 0.2947464883327484,
+      "rewards/rejected": -2.420356512069702,
+      "step": 11290
+    },
+    {
+      "epoch": 1.946933149552033,
+      "grad_norm": 12.404221764432213,
+      "learning_rate": 1.0715625048927091e-10,
+      "logits/chosen": -3.3621768951416016,
+      "logits/rejected": -3.3486781120300293,
+      "logps/chosen": -2.1365556716918945,
+      "logps/rejected": -2.3125698566436768,
+      "loss": 1.1208,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -2.1365556716918945,
+      "rewards/margins": 0.17601452767848969,
+      "rewards/rejected": -2.3125698566436768,
+      "step": 11300
+    },
+    {
+      "epoch": 1.946933149552033,
+      "eval_logits/chosen": -3.4309909343719482,
+      "eval_logits/rejected": -3.4273648262023926,
+      "eval_logps/chosen": -1.903232216835022,
+      "eval_logps/rejected": -2.0979044437408447,
+      "eval_loss": 1.0876781940460205,
+      "eval_rewards/accuracies": 0.6236059665679932,
+      "eval_rewards/chosen": -1.903232216835022,
+      "eval_rewards/margins": 0.1946723908185959,
+      "eval_rewards/rejected": -2.0979044437408447,
+      "eval_runtime": 155.1737,
+      "eval_samples_per_second": 27.737,
+      "eval_steps_per_second": 3.467,
+      "step": 11300
+    },
+    {
+      "epoch": 1.948656099241902,
+      "grad_norm": 11.25123915728846,
+      "learning_rate": 1.0031559186586824e-10,
+      "logits/chosen": -3.42964243888855,
+      "logits/rejected": -3.4259274005889893,
+      "logps/chosen": -2.0626184940338135,
+      "logps/rejected": -2.3826873302459717,
+      "loss": 1.0346,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -2.0626184940338135,
+      "rewards/margins": 0.3200687766075134,
+      "rewards/rejected": -2.3826873302459717,
+      "step": 11310
+    },
+    {
+      "epoch": 1.9503790489317712,
+      "grad_norm": 12.032817071418277,
+      "learning_rate": 9.370010300579212e-11,
+      "logits/chosen": -3.37153959274292,
+      "logits/rejected": -3.3554604053497314,
+      "logps/chosen": -2.0281190872192383,
+      "logps/rejected": -2.4068078994750977,
+      "loss": 1.002,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -2.0281190872192383,
+      "rewards/margins": 0.37868863344192505,
+      "rewards/rejected": -2.4068078994750977,
+      "step": 11320
+    },
+    {
+      "epoch": 1.9521019986216404,
+      "grad_norm": 14.111002718512383,
+      "learning_rate": 8.73098437334241e-11,
+      "logits/chosen": -3.3801636695861816,
+      "logits/rejected": -3.3593897819519043,
+      "logps/chosen": -2.0447661876678467,
+      "logps/rejected": -2.4529759883880615,
+      "loss": 0.976,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -2.0447661876678467,
+      "rewards/margins": 0.40820980072021484,
+      "rewards/rejected": -2.4529759883880615,
+      "step": 11330
+    },
+    {
+      "epoch": 1.9538249483115093,
+      "grad_norm": 13.20467746002715,
+      "learning_rate": 8.114487183636942e-11,
+      "logits/chosen": -3.2990574836730957,
+      "logits/rejected": -3.2889652252197266,
+      "logps/chosen": -2.1248221397399902,
+      "logps/rejected": -2.373281478881836,
+      "loss": 1.0817,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -2.1248221397399902,
+      "rewards/margins": 0.2484598606824875,
+      "rewards/rejected": -2.373281478881836,
+      "step": 11340
+    },
+    {
+      "epoch": 1.9555478980013783,
+      "grad_norm": 13.134104964317764,
+      "learning_rate": 7.520524306494358e-11,
+      "logits/chosen": -3.402188539505005,
+      "logits/rejected": -3.396623134613037,
+      "logps/chosen": -2.177111864089966,
+      "logps/rejected": -2.3756070137023926,
+      "loss": 1.0963,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -2.177111864089966,
+      "rewards/margins": 0.19849511981010437,
+      "rewards/rejected": -2.3756070137023926,
+      "step": 11350
+    },
+    {
+      "epoch": 1.9572708476912473,
+      "grad_norm": 14.007821998464907,
+      "learning_rate": 6.949101113166711e-11,
+      "logits/chosen": -3.3811874389648438,
+      "logits/rejected": -3.3660151958465576,
+      "logps/chosen": -2.110015392303467,
+      "logps/rejected": -2.5120224952697754,
+      "loss": 1.0291,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -2.110015392303467,
+      "rewards/margins": 0.4020071029663086,
+      "rewards/rejected": -2.5120224952697754,
+      "step": 11360
+    },
+    {
+      "epoch": 1.9589937973811165,
+      "grad_norm": 14.238507912727151,
+      "learning_rate": 6.40022277107799e-11,
+      "logits/chosen": -3.366037368774414,
+      "logits/rejected": -3.3551642894744873,
+      "logps/chosen": -2.1116557121276855,
+      "logps/rejected": -2.4214229583740234,
+      "loss": 1.0653,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -2.1116557121276855,
+      "rewards/margins": 0.30976754426956177,
+      "rewards/rejected": -2.4214229583740234,
+      "step": 11370
+    },
+    {
+      "epoch": 1.9607167470709856,
+      "grad_norm": 14.129708693356516,
+      "learning_rate": 5.873894243776933e-11,
+      "logits/chosen": -3.3444607257843018,
+      "logits/rejected": -3.3351454734802246,
+      "logps/chosen": -2.170552968978882,
+      "logps/rejected": -2.429842710494995,
+      "loss": 1.0908,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -2.170552968978882,
+      "rewards/margins": 0.25928962230682373,
+      "rewards/rejected": -2.429842710494995,
+      "step": 11380
+    },
+    {
+      "epoch": 1.9624396967608546,
+      "grad_norm": 12.591736534983873,
+      "learning_rate": 5.3701202908931766e-11,
+      "logits/chosen": -3.4077141284942627,
+      "logits/rejected": -3.3969788551330566,
+      "logps/chosen": -2.0658583641052246,
+      "logps/rejected": -2.421964406967163,
+      "loss": 1.0058,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -2.0658583641052246,
+      "rewards/margins": 0.3561056852340698,
+      "rewards/rejected": -2.421964406967163,
+      "step": 11390
+    },
+    {
+      "epoch": 1.9641626464507236,
+      "grad_norm": 11.705293630674344,
+      "learning_rate": 4.8889054680936736e-11,
+      "logits/chosen": -3.377544403076172,
+      "logits/rejected": -3.3593029975891113,
+      "logps/chosen": -2.0342864990234375,
+      "logps/rejected": -2.3666818141937256,
+      "loss": 1.0322,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -2.0342864990234375,
+      "rewards/margins": 0.33239540457725525,
+      "rewards/rejected": -2.3666818141937256,
+      "step": 11400
+    },
+    {
+      "epoch": 1.9641626464507236,
+      "eval_logits/chosen": -3.4293296337127686,
+      "eval_logits/rejected": -3.425692081451416,
+      "eval_logps/chosen": -1.9032641649246216,
+      "eval_logps/rejected": -2.0977141857147217,
+      "eval_loss": 1.0877939462661743,
+      "eval_rewards/accuracies": 0.6231412887573242,
+      "eval_rewards/chosen": -1.9032641649246216,
+      "eval_rewards/margins": 0.19444987177848816,
+      "eval_rewards/rejected": -2.0977141857147217,
+      "eval_runtime": 155.0554,
+      "eval_samples_per_second": 27.758,
+      "eval_steps_per_second": 3.47,
+      "step": 11400
+    },
+    {
+      "epoch": 1.9658855961405926,
+      "grad_norm": 11.382510767471317,
+      "learning_rate": 4.4302541270407887e-11,
+      "logits/chosen": -3.3813977241516113,
+      "logits/rejected": -3.367804765701294,
+      "logps/chosen": -2.0391297340393066,
+      "logps/rejected": -2.381086826324463,
+      "loss": 1.0183,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -2.0391297340393066,
+      "rewards/margins": 0.3419574499130249,
+      "rewards/rejected": -2.381086826324463,
+      "step": 11410
+    },
+    {
+      "epoch": 1.9676085458304617,
+      "grad_norm": 12.700029207961334,
+      "learning_rate": 3.994170415353715e-11,
+      "logits/chosen": -3.3806509971618652,
+      "logits/rejected": -3.3639254570007324,
+      "logps/chosen": -2.058842182159424,
+      "logps/rejected": -2.3905513286590576,
+      "loss": 1.0521,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -2.058842182159424,
+      "rewards/margins": 0.3317093551158905,
+      "rewards/rejected": -2.3905513286590576,
+      "step": 11420
+    },
+    {
+      "epoch": 1.969331495520331,
+      "grad_norm": 12.74523714095472,
+      "learning_rate": 3.5806582765715576e-11,
+      "logits/chosen": -3.3293819427490234,
+      "logits/rejected": -3.315412998199463,
+      "logps/chosen": -2.1127078533172607,
+      "logps/rejected": -2.408663511276245,
+      "loss": 1.035,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -2.1127078533172607,
+      "rewards/margins": 0.2959558367729187,
+      "rewards/rejected": -2.408663511276245,
+      "step": 11430
+    },
+    {
+      "epoch": 1.9710544452102,
+      "grad_norm": 13.247332600865136,
+      "learning_rate": 3.189721450116145e-11,
+      "logits/chosen": -3.375026226043701,
+      "logits/rejected": -3.3688488006591797,
+      "logps/chosen": -2.121727228164673,
+      "logps/rejected": -2.326486110687256,
+      "loss": 1.1076,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -2.121727228164673,
+      "rewards/margins": 0.20475880801677704,
+      "rewards/rejected": -2.326486110687256,
+      "step": 11440
+    },
+    {
+      "epoch": 1.9727773949000689,
+      "grad_norm": 14.041547499303341,
+      "learning_rate": 2.821363471259275e-11,
+      "logits/chosen": -3.3193302154541016,
+      "logits/rejected": -3.306530714035034,
+      "logps/chosen": -2.0359814167022705,
+      "logps/rejected": -2.3488707542419434,
+      "loss": 1.0315,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -2.0359814167022705,
+      "rewards/margins": 0.31288936734199524,
+      "rewards/rejected": -2.3488707542419434,
+      "step": 11450
+    },
+    {
+      "epoch": 1.9745003445899378,
+      "grad_norm": 13.861457134523123,
+      "learning_rate": 2.4755876710905176e-11,
+      "logits/chosen": -3.354097843170166,
+      "logits/rejected": -3.3450183868408203,
+      "logps/chosen": -2.144233226776123,
+      "logps/rejected": -2.455078125,
+      "loss": 1.0525,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -2.144233226776123,
+      "rewards/margins": 0.3108449876308441,
+      "rewards/rejected": -2.455078125,
+      "step": 11460
+    },
+    {
+      "epoch": 1.976223294279807,
+      "grad_norm": 12.680223306497183,
+      "learning_rate": 2.1523971764869642e-11,
+      "logits/chosen": -3.3820927143096924,
+      "logits/rejected": -3.3675663471221924,
+      "logps/chosen": -2.2054085731506348,
+      "logps/rejected": -2.446868896484375,
+      "loss": 1.0947,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -2.2054085731506348,
+      "rewards/margins": 0.24146027863025665,
+      "rewards/rejected": -2.446868896484375,
+      "step": 11470
+    },
+    {
+      "epoch": 1.9779462439696762,
+      "grad_norm": 10.376167663404901,
+      "learning_rate": 1.851794910085469e-11,
+      "logits/chosen": -3.4022879600524902,
+      "logits/rejected": -3.3857600688934326,
+      "logps/chosen": -2.002114772796631,
+      "logps/rejected": -2.3253207206726074,
+      "loss": 1.0313,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -2.002114772796631,
+      "rewards/margins": 0.3232058882713318,
+      "rewards/rejected": -2.3253207206726074,
+      "step": 11480
+    },
+    {
+      "epoch": 1.9796691936595452,
+      "grad_norm": 10.07635637871587,
+      "learning_rate": 1.5737835902551733e-11,
+      "logits/chosen": -3.350571393966675,
+      "logits/rejected": -3.3361728191375732,
+      "logps/chosen": -2.0728001594543457,
+      "logps/rejected": -2.3029935359954834,
+      "loss": 1.077,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -2.0728001594543457,
+      "rewards/margins": 0.23019346594810486,
+      "rewards/rejected": -2.3029935359954834,
+      "step": 11490
+    },
+    {
+      "epoch": 1.9813921433494142,
+      "grad_norm": 11.600735311094244,
+      "learning_rate": 1.3183657310741891e-11,
+      "logits/chosen": -3.382777452468872,
+      "logits/rejected": -3.371913433074951,
+      "logps/chosen": -2.1116020679473877,
+      "logps/rejected": -2.3688037395477295,
+      "loss": 1.0917,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -2.1116020679473877,
+      "rewards/margins": 0.2572017312049866,
+      "rewards/rejected": -2.3688037395477295,
+      "step": 11500
+    },
+    {
+      "epoch": 1.9813921433494142,
+      "eval_logits/chosen": -3.428719997406006,
+      "eval_logits/rejected": -3.4250733852386475,
+      "eval_logps/chosen": -1.9032909870147705,
+      "eval_logps/rejected": -2.097705125808716,
+      "eval_loss": 1.0878164768218994,
+      "eval_rewards/accuracies": 0.6233736276626587,
+      "eval_rewards/chosen": -1.9032909870147705,
+      "eval_rewards/margins": 0.19441412389278412,
+      "eval_rewards/rejected": -2.097705125808716,
+      "eval_runtime": 155.1588,
+      "eval_samples_per_second": 27.739,
+      "eval_steps_per_second": 3.467,
+      "step": 11500
+    },
+    {
+      "epoch": 1.9831150930392831,
+      "grad_norm": 14.325238786098456,
+      "learning_rate": 1.0855436423054531e-11,
+      "logits/chosen": -3.32232666015625,
+      "logits/rejected": -3.316922426223755,
+      "logps/chosen": -2.1368823051452637,
+      "logps/rejected": -2.35145902633667,
+      "loss": 1.1082,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -2.1368823051452637,
+      "rewards/margins": 0.21457639336585999,
+      "rewards/rejected": -2.35145902633667,
+      "step": 11510
+    },
+    {
+      "epoch": 1.9848380427291523,
+      "grad_norm": 11.867680139193366,
+      "learning_rate": 8.753194293770194e-12,
+      "logits/chosen": -3.3948593139648438,
+      "logits/rejected": -3.3673081398010254,
+      "logps/chosen": -2.0416219234466553,
+      "logps/rejected": -2.3833141326904297,
+      "loss": 1.0096,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -2.0416219234466553,
+      "rewards/margins": 0.34169191122055054,
+      "rewards/rejected": -2.3833141326904297,
+      "step": 11520
+    },
+    {
+      "epoch": 1.9865609924190215,
+      "grad_norm": 11.071234954789558,
+      "learning_rate": 6.876949933631859e-12,
+      "logits/chosen": -3.377716064453125,
+      "logits/rejected": -3.3592116832733154,
+      "logps/chosen": -2.138582706451416,
+      "logps/rejected": -2.4559216499328613,
+      "loss": 1.0553,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -2.138582706451416,
+      "rewards/margins": 0.31733882427215576,
+      "rewards/rejected": -2.4559216499328613,
+      "step": 11530
+    },
+    {
+      "epoch": 1.9882839421088905,
+      "grad_norm": 12.573286915937377,
+      "learning_rate": 5.226720309656207e-12,
+      "logits/chosen": -3.3969674110412598,
+      "logits/rejected": -3.3900516033172607,
+      "logps/chosen": -1.9932247400283813,
+      "logps/rejected": -2.4063262939453125,
+      "loss": 0.981,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -1.9932247400283813,
+      "rewards/margins": 0.4131016135215759,
+      "rewards/rejected": -2.4063262939453125,
+      "step": 11540
+    },
+    {
+      "epoch": 1.9900068917987594,
+      "grad_norm": 10.704384546791854,
+      "learning_rate": 3.802520345000393e-12,
+      "logits/chosen": -3.3569183349609375,
+      "logits/rejected": -3.3454978466033936,
+      "logps/chosen": -2.002999782562256,
+      "logps/rejected": -2.4116389751434326,
+      "loss": 0.9913,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -2.002999782562256,
+      "rewards/margins": 0.40863895416259766,
+      "rewards/rejected": -2.4116389751434326,
+      "step": 11550
+    },
+    {
+      "epoch": 1.9917298414886284,
+      "grad_norm": 12.120377850597936,
+      "learning_rate": 2.604362918812164e-12,
+      "logits/chosen": -3.373047351837158,
+      "logits/rejected": -3.3589210510253906,
+      "logps/chosen": -2.1687874794006348,
+      "logps/rejected": -2.432572364807129,
+      "loss": 1.0581,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -2.1687874794006348,
+      "rewards/margins": 0.26378458738327026,
+      "rewards/rejected": -2.432572364807129,
+      "step": 11560
+    },
+    {
+      "epoch": 1.9934527911784976,
+      "grad_norm": 12.762929766181275,
+      "learning_rate": 1.6322588661216163e-12,
+      "logits/chosen": -3.3721160888671875,
+      "logits/rejected": -3.3621368408203125,
+      "logps/chosen": -2.1780457496643066,
+      "logps/rejected": -2.4682888984680176,
+      "loss": 1.06,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -2.1780457496643066,
+      "rewards/margins": 0.2902430593967438,
+      "rewards/rejected": -2.4682888984680176,
+      "step": 11570
+    },
+    {
+      "epoch": 1.9951757408683668,
+      "grad_norm": 13.443047490887379,
+      "learning_rate": 8.862169777440476e-13,
+      "logits/chosen": -3.391150951385498,
+      "logits/rejected": -3.3800835609436035,
+      "logps/chosen": -2.1820244789123535,
+      "logps/rejected": -2.459082841873169,
+      "loss": 1.0599,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -2.1820244789123535,
+      "rewards/margins": 0.2770578861236572,
+      "rewards/rejected": -2.459082841873169,
+      "step": 11580
+    },
+    {
+      "epoch": 1.9968986905582358,
+      "grad_norm": 13.50869656690931,
+      "learning_rate": 3.662440001883649e-13,
+      "logits/chosen": -3.328286647796631,
+      "logits/rejected": -3.309921979904175,
+      "logps/chosen": -2.078847646713257,
+      "logps/rejected": -2.386496067047119,
+      "loss": 1.068,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -2.078847646713257,
+      "rewards/margins": 0.30764856934547424,
+      "rewards/rejected": -2.386496067047119,
+      "step": 11590
+    },
+    {
+      "epoch": 1.9986216402481047,
+      "grad_norm": 13.81974690504573,
+      "learning_rate": 7.234463561267556e-14,
+      "logits/chosen": -3.347663402557373,
+      "logits/rejected": -3.341860294342041,
+      "logps/chosen": -2.08423113822937,
+      "logps/rejected": -2.4332072734832764,
+      "loss": 1.0116,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -2.08423113822937,
+      "rewards/margins": 0.3489760756492615,
+      "rewards/rejected": -2.4332072734832764,
+      "step": 11600
+    },
+    {
+      "epoch": 1.9986216402481047,
+      "eval_logits/chosen": -3.428753614425659,
+      "eval_logits/rejected": -3.4251012802124023,
+      "eval_logps/chosen": -1.9033112525939941,
+      "eval_logps/rejected": -2.097701072692871,
+      "eval_loss": 1.0879029035568237,
+      "eval_rewards/accuracies": 0.6229089498519897,
+      "eval_rewards/chosen": -1.9033112525939941,
+      "eval_rewards/margins": 0.19438959658145905,
+      "eval_rewards/rejected": -2.097701072692871,
+      "eval_runtime": 155.2653,
+      "eval_samples_per_second": 27.72,
+      "eval_steps_per_second": 3.465,
+      "step": 11600
+    },
+    {
+      "epoch": 2.0,
+      "step": 11608,
+      "total_flos": 0.0,
+      "train_loss": 1.0820189664726994,
+      "train_runtime": 53357.1715,
+      "train_samples_per_second": 3.481,
+      "train_steps_per_second": 0.218
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 11608,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 2,
+  "save_steps": 100,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 8,
+  "trial_name": null,
+  "trial_params": null
+}