{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.0,
  "eval_steps": 500,
  "global_step": 637,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0,
      "learning_rate": 7.8125e-08,
      "logits/chosen": 0.14330744743347168,
      "logits/rejected": 0.18303190171718597,
      "logps/chosen": -714.194580078125,
      "logps/rejected": -783.393798828125,
      "loss": 250000.0,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.02,
      "learning_rate": 7.8125e-07,
      "logits/chosen": 0.1265290379524231,
      "logits/rejected": 0.17839111387729645,
      "logps/chosen": -714.8074340820312,
      "logps/rejected": -733.1469116210938,
      "loss": 250078.1667,
      "rewards/accuracies": 0.3611111044883728,
      "rewards/chosen": -0.00013909974950365722,
      "rewards/margins": -0.00010831771942321211,
      "rewards/rejected": -3.078207009821199e-05,
      "step": 10
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.5625e-06,
      "logits/chosen": 0.11676273494958878,
      "logits/rejected": 0.15343983471393585,
      "logps/chosen": -693.2666015625,
      "logps/rejected": -712.1083374023438,
      "loss": 249833.05,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0006469396757893264,
      "rewards/margins": 0.00024659099290147424,
      "rewards/rejected": -0.0008935307268984616,
      "step": 20
    },
    {
      "epoch": 0.05,
      "learning_rate": 2.3437500000000002e-06,
      "logits/chosen": 0.09228599816560745,
      "logits/rejected": 0.14996477961540222,
      "logps/chosen": -694.0119018554688,
      "logps/rejected": -726.91796875,
      "loss": 249133.3,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.005162687040865421,
      "rewards/margins": 0.0009325396385975182,
      "rewards/rejected": -0.006095226388424635,
      "step": 30
    },
    {
      "epoch": 0.06,
      "learning_rate": 3.125e-06,
      "logits/chosen": 0.05852314084768295,
      "logits/rejected": 0.1720169633626938,
      "logps/chosen": -751.1815795898438,
      "logps/rejected": -776.6752319335938,
      "loss": 247264.85,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.01876734383404255,
      "rewards/margins": 0.0032565854489803314,
      "rewards/rejected": -0.02202392928302288,
      "step": 40
    },
    {
      "epoch": 0.08,
      "learning_rate": 3.90625e-06,
      "logits/chosen": 0.10499389469623566,
      "logits/rejected": 0.16752108931541443,
      "logps/chosen": -779.3970947265625,
      "logps/rejected": -821.4215698242188,
      "loss": 243623.9,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.04817109555006027,
      "rewards/margins": 0.007549160160124302,
      "rewards/rejected": -0.0557202585041523,
      "step": 50
    },
    {
      "epoch": 0.09,
      "learning_rate": 4.6875000000000004e-06,
      "logits/chosen": 0.10056255757808685,
      "logits/rejected": 0.1355510652065277,
      "logps/chosen": -761.1482543945312,
      "logps/rejected": -791.7299194335938,
      "loss": 241589.05,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.06844565272331238,
      "rewards/margins": 0.00780609343200922,
      "rewards/rejected": -0.07625173777341843,
      "step": 60
    },
    {
      "epoch": 0.11,
      "learning_rate": 4.998647417232375e-06,
      "logits/chosen": 0.08108599483966827,
      "logits/rejected": 0.12395985424518585,
      "logps/chosen": -819.1280517578125,
      "logps/rejected": -899.9339599609375,
      "loss": 236403.25,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -0.0921686664223671,
      "rewards/margins": 0.011640750803053379,
      "rewards/rejected": -0.1038094162940979,
      "step": 70
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.990386933279973e-06,
      "logits/chosen": 0.07269687950611115,
      "logits/rejected": 0.12636110186576843,
      "logps/chosen": -799.1640625,
      "logps/rejected": -871.1455078125,
      "loss": 228858.95,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.09310872852802277,
      "rewards/margins": 0.03275894373655319,
      "rewards/rejected": -0.12586766481399536,
      "step": 80
    },
    {
      "epoch": 0.14,
      "learning_rate": 4.974642195009681e-06,
      "logits/chosen": 0.059646714478731155,
      "logits/rejected": 0.11064349114894867,
      "logps/chosen": -906.8914184570312,
      "logps/rejected": -967.6486206054688,
      "loss": 231742.65,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.17958079278469086,
      "rewards/margins": 0.026337388902902603,
      "rewards/rejected": -0.20591816306114197,
      "step": 90
    },
    {
      "epoch": 0.16,
      "learning_rate": 4.951460519416228e-06,
      "logits/chosen": 0.09507628530263901,
      "logits/rejected": 0.06315924227237701,
      "logps/chosen": -831.5269775390625,
      "logps/rejected": -934.2014770507812,
      "loss": 228067.35,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.16508157551288605,
      "rewards/margins": 0.02746037021279335,
      "rewards/rejected": -0.1925419270992279,
      "step": 100
    },
    {
      "epoch": 0.17,
      "learning_rate": 4.920911573406925e-06,
      "logits/chosen": 0.08507482707500458,
      "logits/rejected": 0.06426723301410675,
      "logps/chosen": -886.0023193359375,
      "logps/rejected": -968.9869384765625,
      "loss": 231192.4,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.19480349123477936,
      "rewards/margins": 0.028372664004564285,
      "rewards/rejected": -0.22317615151405334,
      "step": 110
    },
    {
      "epoch": 0.19,
      "learning_rate": 4.883087164434672e-06,
      "logits/chosen": 0.12870684266090393,
      "logits/rejected": 0.19525791704654694,
      "logps/chosen": -918.3961181640625,
      "logps/rejected": -981.2435302734375,
      "loss": 228031.675,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.16016361117362976,
      "rewards/margins": 0.033955834805965424,
      "rewards/rejected": -0.1941194236278534,
      "step": 120
    },
    {
      "epoch": 0.2,
      "learning_rate": 4.8381009645929044e-06,
      "logits/chosen": 0.19024774432182312,
      "logits/rejected": 0.1789695769548416,
      "logps/chosen": -888.7951049804688,
      "logps/rejected": -1009.6251220703125,
      "loss": 225728.075,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.218052938580513,
      "rewards/margins": 0.05132851004600525,
      "rewards/rejected": -0.26938146352767944,
      "step": 130
    },
    {
      "epoch": 0.22,
      "learning_rate": 4.786088169001671e-06,
      "logits/chosen": 0.17952367663383484,
      "logits/rejected": 0.15534032881259918,
      "logps/chosen": -932.0501098632812,
      "logps/rejected": -1063.305419921875,
      "loss": 228587.2,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.2622358500957489,
      "rewards/margins": 0.05344052240252495,
      "rewards/rejected": -0.31567639112472534,
      "step": 140
    },
    {
      "epoch": 0.24,
      "learning_rate": 4.727205089511466e-06,
      "logits/chosen": 0.1564352661371231,
      "logits/rejected": 0.1726730763912201,
      "logps/chosen": -859.6404418945312,
      "logps/rejected": -931.6002807617188,
      "loss": 237987.3,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.1820591390132904,
      "rewards/margins": 0.03248381242156029,
      "rewards/rejected": -0.2145429402589798,
      "step": 150
    },
    {
      "epoch": 0.25,
      "learning_rate": 4.661628684945851e-06,
      "logits/chosen": 0.1257745772600174,
      "logits/rejected": 0.18414214253425598,
      "logps/chosen": -935.7888793945312,
      "logps/rejected": -1013.6844482421875,
      "loss": 227312.15,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.18025702238082886,
      "rewards/margins": 0.035888925194740295,
      "rewards/rejected": -0.21614596247673035,
      "step": 160
    },
    {
      "epoch": 0.27,
      "learning_rate": 4.5895560292946e-06,
      "logits/chosen": 0.2171991616487503,
      "logits/rejected": 0.2624339461326599,
      "logps/chosen": -1001.1527099609375,
      "logps/rejected": -1123.304443359375,
      "loss": 217504.55,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.2904219925403595,
      "rewards/margins": 0.05342133715748787,
      "rewards/rejected": -0.34384334087371826,
      "step": 170
    },
    {
      "epoch": 0.28,
      "learning_rate": 4.511203719455588e-06,
      "logits/chosen": 0.17157816886901855,
      "logits/rejected": 0.3099578320980072,
      "logps/chosen": -1003.0543212890625,
      "logps/rejected": -1025.053955078125,
      "loss": 231876.85,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.28897398710250854,
      "rewards/margins": 0.03390089422464371,
      "rewards/rejected": -0.32287487387657166,
      "step": 180
    },
    {
      "epoch": 0.3,
      "learning_rate": 4.426807224305315e-06,
      "logits/chosen": 0.17717203497886658,
      "logits/rejected": 0.23975355923175812,
      "logps/chosen": -887.27294921875,
      "logps/rejected": -988.5928955078125,
      "loss": 219205.3,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.2065242975950241,
      "rewards/margins": 0.056366823613643646,
      "rewards/rejected": -0.26289111375808716,
      "step": 190
    },
    {
      "epoch": 0.31,
      "learning_rate": 4.336620177054269e-06,
      "logits/chosen": 0.19569167494773865,
      "logits/rejected": 0.23822470009326935,
      "logps/chosen": -890.8048706054688,
      "logps/rejected": -993.693359375,
      "loss": 222280.1,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.23872694373130798,
      "rewards/margins": 0.05283508822321892,
      "rewards/rejected": -0.291562020778656,
      "step": 200
    },
    {
      "epoch": 0.33,
      "learning_rate": 4.240913613013785e-06,
      "logits/chosen": 0.2506358325481415,
      "logits/rejected": 0.184006929397583,
      "logps/chosen": -1007.4181518554688,
      "logps/rejected": -1067.2701416015625,
      "loss": 221823.5,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.2965286672115326,
      "rewards/margins": 0.035524263978004456,
      "rewards/rejected": -0.33205297589302063,
      "step": 210
    },
    {
      "epoch": 0.35,
      "learning_rate": 4.139975155065109e-06,
      "logits/chosen": 0.2400396764278412,
      "logits/rejected": 0.29167982935905457,
      "logps/chosen": -1041.81103515625,
      "logps/rejected": -1105.313720703125,
      "loss": 219784.95,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.2964867353439331,
      "rewards/margins": 0.05159877613186836,
      "rewards/rejected": -0.34808549284935,
      "step": 220
    },
    {
      "epoch": 0.36,
      "learning_rate": 4.034108149278544e-06,
      "logits/chosen": 0.24634487926959991,
      "logits/rejected": 0.2607974112033844,
      "logps/chosen": -1003.81884765625,
      "logps/rejected": -1118.8782958984375,
      "loss": 206378.925,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.3140796422958374,
      "rewards/margins": 0.06415858119726181,
      "rewards/rejected": -0.3782382011413574,
      "step": 230
    },
    {
      "epoch": 0.38,
      "learning_rate": 3.923630753280358e-06,
      "logits/chosen": 0.20628812909126282,
      "logits/rejected": 0.37754902243614197,
      "logps/chosen": -1053.105712890625,
      "logps/rejected": -1159.8330078125,
      "loss": 200137.8125,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.3624956011772156,
      "rewards/margins": 0.06471217423677444,
      "rewards/rejected": -0.4272077977657318,
      "step": 240
    },
    {
      "epoch": 0.39,
      "learning_rate": 3.80887498010715e-06,
      "logits/chosen": 0.29324811697006226,
      "logits/rejected": 0.2819308936595917,
      "logps/chosen": -1104.125244140625,
      "logps/rejected": -1199.90185546875,
      "loss": 227303.975,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.37163108587265015,
      "rewards/margins": 0.04434230178594589,
      "rewards/rejected": -0.41597336530685425,
      "step": 250
    },
    {
      "epoch": 0.41,
      "learning_rate": 3.690185700421145e-06,
      "logits/chosen": 0.2746821343898773,
      "logits/rejected": 0.33129218220710754,
      "logps/chosen": -1043.5469970703125,
      "logps/rejected": -1157.398193359375,
      "loss": 211889.525,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.3348512649536133,
      "rewards/margins": 0.06582818925380707,
      "rewards/rejected": -0.40067943930625916,
      "step": 260
    },
    {
      "epoch": 0.42,
      "learning_rate": 3.567919606085004e-06,
      "logits/chosen": 0.20646102726459503,
      "logits/rejected": 0.3259996473789215,
      "logps/chosen": -1120.896240234375,
      "logps/rejected": -1177.1099853515625,
      "loss": 217746.775,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.37923678755760193,
      "rewards/margins": 0.05121999233961105,
      "rewards/rejected": -0.4304567873477936,
      "step": 270
    },
    {
      "epoch": 0.44,
      "learning_rate": 3.442444138210883e-06,
      "logits/chosen": 0.23033392429351807,
      "logits/rejected": 0.3274117708206177,
      "logps/chosen": -1110.678466796875,
      "logps/rejected": -1200.6253662109375,
      "loss": 223675.5,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.3978205919265747,
      "rewards/margins": 0.051620304584503174,
      "rewards/rejected": -0.4494408667087555,
      "step": 280
    },
    {
      "epoch": 0.46,
      "learning_rate": 3.314136382905234e-06,
      "logits/chosen": 0.16156849265098572,
      "logits/rejected": 0.30493414402008057,
      "logps/chosen": -1073.877685546875,
      "logps/rejected": -1188.1846923828125,
      "loss": 216146.5,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.3979111611843109,
      "rewards/margins": 0.06408871710300446,
      "rewards/rejected": -0.46199989318847656,
      "step": 290
    },
    {
      "epoch": 0.47,
      "learning_rate": 3.1833819380279028e-06,
      "logits/chosen": 0.25436246395111084,
      "logits/rejected": 0.35649871826171875,
      "logps/chosen": -970.6337890625,
      "logps/rejected": -1103.1796875,
      "loss": 211613.25,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.3316889703273773,
      "rewards/margins": 0.06474041938781738,
      "rewards/rejected": -0.3964293897151947,
      "step": 300
    },
    {
      "epoch": 0.49,
      "learning_rate": 3.050573754371228e-06,
      "logits/chosen": 0.19806575775146484,
      "logits/rejected": 0.30608507990837097,
      "logps/chosen": -1002.451171875,
      "logps/rejected": -1153.102783203125,
      "loss": 199471.9375,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3274732828140259,
      "rewards/margins": 0.08164636790752411,
      "rewards/rejected": -0.40911969542503357,
      "step": 310
    },
    {
      "epoch": 0.5,
      "learning_rate": 2.916110954741667e-06,
      "logits/chosen": 0.23385831713676453,
      "logits/rejected": 0.3278101682662964,
      "logps/chosen": -1135.4813232421875,
      "logps/rejected": -1250.9149169921875,
      "loss": 214479.075,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.3785611689090729,
      "rewards/margins": 0.06773276627063751,
      "rewards/rejected": -0.44629397988319397,
      "step": 320
    },
    {
      "epoch": 0.52,
      "learning_rate": 2.7803976344929497e-06,
      "logits/chosen": 0.2279421091079712,
      "logits/rejected": 0.2835574150085449,
      "logps/chosen": -1054.18359375,
      "logps/rejected": -1125.3388671875,
      "loss": 222046.5,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3506453037261963,
      "rewards/margins": 0.05092238262295723,
      "rewards/rejected": -0.4015676975250244,
      "step": 330
    },
    {
      "epoch": 0.53,
      "learning_rate": 2.6438416471154277e-06,
      "logits/chosen": 0.2912670373916626,
      "logits/rejected": 0.41436976194381714,
      "logps/chosen": -993.8232421875,
      "logps/rejected": -1134.54296875,
      "loss": 197803.8,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.34450316429138184,
      "rewards/margins": 0.09640122950077057,
      "rewards/rejected": -0.4409043788909912,
      "step": 340
    },
    {
      "epoch": 0.55,
      "learning_rate": 2.5068533785312673e-06,
      "logits/chosen": 0.259615033864975,
      "logits/rejected": 0.3351518213748932,
      "logps/chosen": -1080.523681640625,
      "logps/rejected": -1217.09521484375,
      "loss": 224075.1,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.3753746747970581,
      "rewards/margins": 0.07089592516422272,
      "rewards/rejected": -0.446270614862442,
      "step": 350
    },
    {
      "epoch": 0.57,
      "learning_rate": 2.369844513779026e-06,
      "logits/chosen": 0.16386622190475464,
      "logits/rejected": 0.20941393077373505,
      "logps/chosen": -1049.8115234375,
      "logps/rejected": -1173.7218017578125,
      "loss": 214063.75,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.30884891748428345,
      "rewards/margins": 0.07134245336055756,
      "rewards/rejected": -0.3801913857460022,
      "step": 360
    },
    {
      "epoch": 0.58,
      "learning_rate": 2.2332267997940514e-06,
      "logits/chosen": 0.1576313078403473,
      "logits/rejected": 0.2314819097518921,
      "logps/chosen": -989.9000244140625,
      "logps/rejected": -1112.245849609375,
      "loss": 212630.15,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.28885313868522644,
      "rewards/margins": 0.05920270085334778,
      "rewards/rejected": -0.3480558395385742,
      "step": 370
    },
    {
      "epoch": 0.6,
      "learning_rate": 2.097410808002869e-06,
      "logits/chosen": 0.2326604425907135,
      "logits/rejected": 0.3002316653728485,
      "logps/chosen": -1086.225341796875,
      "logps/rejected": -1175.258056640625,
      "loss": 216490.55,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.3227333724498749,
      "rewards/margins": 0.06897234916687012,
      "rewards/rejected": -0.3917057514190674,
      "step": 380
    },
    {
      "epoch": 0.61,
      "learning_rate": 1.962804700450265e-06,
      "logits/chosen": 0.2371029108762741,
      "logits/rejected": 0.3002297580242157,
      "logps/chosen": -1030.2000732421875,
      "logps/rejected": -1175.873046875,
      "loss": 214079.6,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.3257220983505249,
      "rewards/margins": 0.07455430179834366,
      "rewards/rejected": -0.40027642250061035,
      "step": 390
    },
    {
      "epoch": 0.63,
      "learning_rate": 1.8298130031671974e-06,
      "logits/chosen": 0.25191420316696167,
      "logits/rejected": 0.29652512073516846,
      "logps/chosen": -1057.87646484375,
      "logps/rejected": -1160.5743408203125,
      "loss": 227584.0,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.3368803858757019,
      "rewards/margins": 0.04710642248392105,
      "rewards/rejected": -0.38398683071136475,
      "step": 400
    },
    {
      "epoch": 0.64,
      "learning_rate": 1.6988353904658495e-06,
      "logits/chosen": 0.25754934549331665,
      "logits/rejected": 0.1926787793636322,
      "logps/chosen": -1006.3465576171875,
      "logps/rejected": -1107.968994140625,
      "loss": 213971.8,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.2910354435443878,
      "rewards/margins": 0.06115920469164848,
      "rewards/rejected": -0.3521946668624878,
      "step": 410
    },
    {
      "epoch": 0.66,
      "learning_rate": 1.5702654838153641e-06,
      "logits/chosen": 0.32284024357795715,
      "logits/rejected": 0.3785749673843384,
      "logps/chosen": -1025.57373046875,
      "logps/rejected": -1150.352783203125,
      "loss": 213707.075,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.32002902030944824,
      "rewards/margins": 0.0713513195514679,
      "rewards/rejected": -0.3913803696632385,
      "step": 420
    },
    {
      "epoch": 0.68,
      "learning_rate": 1.4444896689079142e-06,
      "logits/chosen": 0.24807333946228027,
      "logits/rejected": 0.26119035482406616,
      "logps/chosen": -981.4894409179688,
      "logps/rejected": -1027.531494140625,
      "loss": 228694.3,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.31893014907836914,
      "rewards/margins": 0.03217142075300217,
      "rewards/rejected": -0.3511015772819519,
      "step": 430
    },
    {
      "epoch": 0.69,
      "learning_rate": 1.3218859344701634e-06,
      "logits/chosen": 0.24505586922168732,
      "logits/rejected": 0.23741266131401062,
      "logps/chosen": -976.6732177734375,
      "logps/rejected": -1118.38134765625,
      "loss": 197599.0,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.2975761890411377,
      "rewards/margins": 0.08645204454660416,
      "rewards/rejected": -0.38402819633483887,
      "step": 440
    },
    {
      "epoch": 0.71,
      "learning_rate": 1.2028227363097583e-06,
      "logits/chosen": 0.2540598213672638,
      "logits/rejected": 0.33230239152908325,
      "logps/chosen": -1014.8942260742188,
      "logps/rejected": -1128.714599609375,
      "loss": 204484.4,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.31904861330986023,
      "rewards/margins": 0.0724792331457138,
      "rewards/rejected": -0.39152783155441284,
      "step": 450
    },
    {
      "epoch": 0.72,
      "learning_rate": 1.0876578900107053e-06,
      "logits/chosen": 0.2582911550998688,
      "logits/rejected": 0.27663809061050415,
      "logps/chosen": -1002.5812377929688,
      "logps/rejected": -1135.8900146484375,
      "loss": 212971.9,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.33705443143844604,
      "rewards/margins": 0.0685950368642807,
      "rewards/rejected": -0.40564948320388794,
      "step": 460
    },
    {
      "epoch": 0.74,
      "learning_rate": 9.767374956053584e-07,
      "logits/chosen": 0.21633680164813995,
      "logits/rejected": 0.2309296429157257,
      "logps/chosen": -1026.5072021484375,
      "logps/rejected": -1110.014892578125,
      "loss": 204311.2875,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.33438438177108765,
      "rewards/margins": 0.06735749542713165,
      "rewards/rejected": -0.4017418920993805,
      "step": 470
    },
    {
      "epoch": 0.75,
      "learning_rate": 8.703948974546592e-07,
      "logits/chosen": 0.17137208580970764,
      "logits/rejected": 0.29713553190231323,
      "logps/chosen": -1010.2471923828125,
      "logps/rejected": -1127.9761962890625,
      "loss": 210587.325,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.3343172073364258,
      "rewards/margins": 0.048689451068639755,
      "rewards/rejected": -0.3830066919326782,
      "step": 480
    },
    {
      "epoch": 0.77,
      "learning_rate": 7.689496824624526e-07,
      "logits/chosen": 0.24797169864177704,
      "logits/rejected": 0.3339524269104004,
      "logps/chosen": -1013.1183471679688,
      "logps/rejected": -1139.7716064453125,
      "loss": 205908.4125,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.31524255871772766,
      "rewards/margins": 0.06370928138494492,
      "rewards/rejected": -0.3789518177509308,
      "step": 490
    },
    {
      "epoch": 0.78,
      "learning_rate": 6.7270671963451e-07,
      "logits/chosen": 0.2785577178001404,
      "logits/rejected": 0.34557706117630005,
      "logps/chosen": -1004.1671752929688,
      "logps/rejected": -1095.4044189453125,
      "loss": 222017.075,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.324979692697525,
      "rewards/margins": 0.05109068751335144,
      "rewards/rejected": -0.3760703504085541,
      "step": 500
    },
    {
      "epoch": 0.8,
      "learning_rate": 5.819552438686238e-07,
      "logits/chosen": 0.22823992371559143,
      "logits/rejected": 0.3369165360927582,
      "logps/chosen": -1076.311767578125,
      "logps/rejected": -1174.0753173828125,
      "loss": 216482.85,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.3366367220878601,
      "rewards/margins": 0.06757824122905731,
      "rewards/rejected": -0.404215008020401,
      "step": 510
    },
    {
      "epoch": 0.82,
      "learning_rate": 4.969679867292276e-07,
      "logits/chosen": 0.2003372609615326,
      "logits/rejected": 0.33764129877090454,
      "logps/chosen": -1053.8939208984375,
      "logps/rejected": -1123.6129150390625,
      "loss": 221411.85,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.3437155783176422,
      "rewards/margins": 0.03899475932121277,
      "rewards/rejected": -0.3827103078365326,
      "step": 520
    },
    {
      "epoch": 0.83,
      "learning_rate": 4.1800035681877765e-07,
      "logits/chosen": 0.3177185654640198,
      "logits/rejected": 0.20578141510486603,
      "logps/chosen": -1085.5306396484375,
      "logps/rejected": -1226.7509765625,
      "loss": 199054.875,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.34231036901474,
      "rewards/margins": 0.07949638366699219,
      "rewards/rejected": -0.4218067526817322,
      "step": 530
    },
    {
      "epoch": 0.85,
      "learning_rate": 3.4528967220911287e-07,
      "logits/chosen": 0.26802507042884827,
      "logits/rejected": 0.2959344983100891,
      "logps/chosen": -1103.7427978515625,
      "logps/rejected": -1205.734619140625,
      "loss": 206713.1125,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.34727898240089417,
      "rewards/margins": 0.0809955969452858,
      "rewards/rejected": -0.42827457189559937,
      "step": 540
    },
    {
      "epoch": 0.86,
      "learning_rate": 2.7905444723949765e-07,
      "logits/chosen": 0.24068915843963623,
      "logits/rejected": 0.36839014291763306,
      "logps/chosen": -1065.5458984375,
      "logps/rejected": -1131.9764404296875,
      "loss": 218209.65,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.3374769687652588,
      "rewards/margins": 0.04803069308400154,
      "rewards/rejected": -0.3855076730251312,
      "step": 550
    },
    {
      "epoch": 0.88,
      "learning_rate": 2.1949373582475065e-07,
      "logits/chosen": 0.30096447467803955,
      "logits/rejected": 0.34244808554649353,
      "logps/chosen": -962.107421875,
      "logps/rejected": -1130.420166015625,
      "loss": 203465.5125,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.3097023069858551,
      "rewards/margins": 0.073573537170887,
      "rewards/rejected": -0.3832758069038391,
      "step": 560
    },
    {
      "epoch": 0.89,
      "learning_rate": 1.667865332469379e-07,
      "logits/chosen": 0.22145691514015198,
      "logits/rejected": 0.3323192000389099,
      "logps/chosen": -997.6228637695312,
      "logps/rejected": -1113.1171875,
      "loss": 211145.9,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.3235357105731964,
      "rewards/margins": 0.06192871183156967,
      "rewards/rejected": -0.3854644000530243,
      "step": 570
    },
    {
      "epoch": 0.91,
      "learning_rate": 1.2109123822844653e-07,
      "logits/chosen": 0.2121209353208542,
      "logits/rejected": 0.32750505208969116,
      "logps/chosen": -1062.212158203125,
      "logps/rejected": -1173.4722900390625,
      "loss": 216943.15,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.3464076817035675,
      "rewards/margins": 0.06313388049602509,
      "rewards/rejected": -0.409541517496109,
      "step": 580
    },
    {
      "epoch": 0.93,
      "learning_rate": 8.254517690300946e-08,
      "logits/chosen": 0.2600761651992798,
      "logits/rejected": 0.277840793132782,
      "logps/chosen": -1001.9027099609375,
      "logps/rejected": -1103.707763671875,
      "loss": 212477.1,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.321969211101532,
      "rewards/margins": 0.06838768720626831,
      "rewards/rejected": -0.3903568983078003,
      "step": 590
    },
    {
      "epoch": 0.94,
      "learning_rate": 5.126419011529993e-08,
      "logits/chosen": 0.2812707722187042,
      "logits/rejected": 0.3223172128200531,
      "logps/chosen": -1003.4280395507812,
      "logps/rejected": -1109.736083984375,
      "loss": 207337.8,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.3305527865886688,
      "rewards/margins": 0.05715988203883171,
      "rewards/rejected": -0.38771265745162964,
      "step": 600
    },
    {
      "epoch": 0.96,
      "learning_rate": 2.734228528934679e-08,
      "logits/chosen": 0.25758108496665955,
      "logits/rejected": 0.26223865151405334,
      "logps/chosen": -1068.127197265625,
      "logps/rejected": -1201.6923828125,
      "loss": 207598.25,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.34323567152023315,
      "rewards/margins": 0.07567054033279419,
      "rewards/rejected": -0.41890621185302734,
      "step": 610
    },
    {
      "epoch": 0.97,
      "learning_rate": 1.0851353912008644e-08,
      "logits/chosen": 0.24180451035499573,
      "logits/rejected": 0.30353251099586487,
      "logps/chosen": -1058.5758056640625,
      "logps/rejected": -1153.9122314453125,
      "loss": 209104.875,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.3343828320503235,
      "rewards/margins": 0.06873960793018341,
      "rewards/rejected": -0.4031224846839905,
      "step": 620
    },
    {
      "epoch": 0.99,
      "learning_rate": 1.8409554805329243e-09,
      "logits/chosen": 0.22301486134529114,
      "logits/rejected": 0.2677055895328522,
      "logps/chosen": -1038.4554443359375,
      "logps/rejected": -1139.900634765625,
      "loss": 215125.55,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.3383429944515228,
      "rewards/margins": 0.06525392830371857,
      "rewards/rejected": -0.403596967458725,
      "step": 630
    },
    {
      "epoch": 1.0,
      "step": 637,
      "total_flos": 0.0,
      "train_loss": 219463.76923076922,
      "train_runtime": 8069.1141,
      "train_samples_per_second": 2.525,
      "train_steps_per_second": 0.079
    }
  ],
  "logging_steps": 10,
  "max_steps": 637,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 100,
  "total_flos": 0.0,
  "train_batch_size": 4,
  "trial_name": null,
  "trial_params": null
}