{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 2.9865871833084947,
  "eval_steps": 500,
  "global_step": 501,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.029806259314456036,
      "grad_norm": 1802.2630615234375,
      "learning_rate": 3.75e-07,
      "log_odds_chosen": -0.22044436633586884,
      "log_odds_ratio": -1.0070970058441162,
      "logits/chosen": 204.41079711914062,
      "logits/rejected": 203.1068878173828,
      "logps/chosen": -14.805323600769043,
      "logps/rejected": -14.584878921508789,
      "loss": 15.3769,
      "nll_loss": 14.526931762695312,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -7.4026618003845215,
      "rewards/margins": -0.11022226512432098,
      "rewards/rejected": -7.2924394607543945,
      "step": 5
    },
    {
      "epoch": 0.05961251862891207,
      "grad_norm": 964.4558715820312,
      "learning_rate": 7.5e-07,
      "log_odds_chosen": 0.22153806686401367,
      "log_odds_ratio": -0.7704828977584839,
      "logits/chosen": 229.1735382080078,
      "logits/rejected": 234.7494659423828,
      "logps/chosen": -10.87135124206543,
      "logps/rejected": -11.092044830322266,
      "loss": 11.644,
      "nll_loss": 10.984331130981445,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -5.435675621032715,
      "rewards/margins": 0.1103477030992508,
      "rewards/rejected": -5.546022415161133,
      "step": 10
    },
    {
      "epoch": 0.08941877794336811,
      "grad_norm": 325.32958984375,
      "learning_rate": 1.125e-06,
      "log_odds_chosen": 0.0398867093026638,
      "log_odds_ratio": -0.7268143892288208,
      "logits/chosen": 288.3202209472656,
      "logits/rejected": 268.65325927734375,
      "logps/chosen": -6.712647914886475,
      "logps/rejected": -6.752583980560303,
      "loss": 7.2706,
      "nll_loss": 6.687516689300537,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -3.3563239574432373,
      "rewards/margins": 0.019967829808592796,
      "rewards/rejected": -3.3762919902801514,
      "step": 15
    },
    {
      "epoch": 0.11922503725782414,
      "grad_norm": 153.8790740966797,
      "learning_rate": 1.5e-06,
      "log_odds_chosen": 0.005181239452213049,
      "log_odds_ratio": -0.8698928952217102,
      "logits/chosen": 279.0413513183594,
      "logits/rejected": 273.7066955566406,
      "logps/chosen": -4.006048202514648,
      "logps/rejected": -4.012642860412598,
      "loss": 4.5269,
      "nll_loss": 4.109046459197998,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -2.003024101257324,
      "rewards/margins": 0.003297415329143405,
      "rewards/rejected": -2.006321430206299,
      "step": 20
    },
    {
      "epoch": 0.14903129657228018,
      "grad_norm": 110.71721649169922,
      "learning_rate": 1.875e-06,
      "log_odds_chosen": -0.0372297465801239,
      "log_odds_ratio": -0.8627766370773315,
      "logits/chosen": 340.18084716796875,
      "logits/rejected": 349.11151123046875,
      "logps/chosen": -2.547694206237793,
      "logps/rejected": -2.4806361198425293,
      "loss": 3.1921,
      "nll_loss": 2.749494791030884,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.2738471031188965,
      "rewards/margins": -0.03352922201156616,
      "rewards/rejected": -1.2403180599212646,
      "step": 25
    },
    {
      "epoch": 0.17883755588673622,
      "grad_norm": 75.32969665527344,
      "learning_rate": 2.25e-06,
      "log_odds_chosen": 0.018188167363405228,
      "log_odds_ratio": -0.8359094858169556,
      "logits/chosen": 373.6573791503906,
      "logits/rejected": 400.1236572265625,
      "logps/chosen": -2.330045223236084,
      "logps/rejected": -2.3124964237213135,
      "loss": 2.6112,
      "nll_loss": 2.3758959770202637,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -1.165022611618042,
      "rewards/margins": -0.008774325251579285,
      "rewards/rejected": -1.1562482118606567,
      "step": 30
    },
    {
      "epoch": 0.20864381520119224,
      "grad_norm": 45.19425964355469,
      "learning_rate": 2.625e-06,
      "log_odds_chosen": 0.25173941254615784,
      "log_odds_ratio": -0.7035487294197083,
      "logits/chosen": 391.4384765625,
      "logits/rejected": 380.1146545410156,
      "logps/chosen": -1.7061666250228882,
      "logps/rejected": -1.8836677074432373,
      "loss": 2.3484,
      "nll_loss": 1.9240200519561768,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.8530833125114441,
      "rewards/margins": 0.08875055611133575,
      "rewards/rejected": -0.9418338537216187,
      "step": 35
    },
    {
      "epoch": 0.23845007451564829,
      "grad_norm": 49.53385925292969,
      "learning_rate": 3e-06,
      "log_odds_chosen": 0.2747120261192322,
      "log_odds_ratio": -0.6599891185760498,
      "logits/chosen": 381.4653625488281,
      "logits/rejected": 379.7186279296875,
      "logps/chosen": -1.598753571510315,
      "logps/rejected": -1.8254070281982422,
      "loss": 2.2488,
      "nll_loss": 2.010526418685913,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.7993767857551575,
      "rewards/margins": 0.11332666873931885,
      "rewards/rejected": -0.9127035140991211,
      "step": 40
    },
    {
      "epoch": 0.26825633383010433,
      "grad_norm": 68.84819030761719,
      "learning_rate": 3.3750000000000003e-06,
      "log_odds_chosen": 0.29411372542381287,
      "log_odds_ratio": -0.6252076625823975,
      "logits/chosen": 395.29290771484375,
      "logits/rejected": 404.8909912109375,
      "logps/chosen": -1.651572823524475,
      "logps/rejected": -1.8901698589324951,
      "loss": 2.1795,
      "nll_loss": 1.7518342733383179,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.8257864117622375,
      "rewards/margins": 0.11929843574762344,
      "rewards/rejected": -0.9450849294662476,
      "step": 45
    },
    {
      "epoch": 0.29806259314456035,
      "grad_norm": 47.519283294677734,
      "learning_rate": 3.75e-06,
      "log_odds_chosen": 0.5501509308815002,
      "log_odds_ratio": -0.5450042486190796,
      "logits/chosen": 403.5310363769531,
      "logits/rejected": 424.2149963378906,
      "logps/chosen": -1.5113065242767334,
      "logps/rejected": -1.964956521987915,
      "loss": 2.1594,
      "nll_loss": 1.8876510858535767,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.7556532621383667,
      "rewards/margins": 0.22682484984397888,
      "rewards/rejected": -0.9824782609939575,
      "step": 50
    },
    {
      "epoch": 0.32786885245901637,
      "grad_norm": 36.98734664916992,
      "learning_rate": 4.125e-06,
      "log_odds_chosen": 0.26053518056869507,
      "log_odds_ratio": -0.6607739925384521,
      "logits/chosen": 388.755859375,
      "logits/rejected": 381.29827880859375,
      "logps/chosen": -1.498511552810669,
      "logps/rejected": -1.7075812816619873,
      "loss": 2.1979,
      "nll_loss": 1.9259532690048218,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.7492557764053345,
      "rewards/margins": 0.1045350432395935,
      "rewards/rejected": -0.8537906408309937,
      "step": 55
    },
    {
      "epoch": 0.35767511177347244,
      "grad_norm": 63.64994430541992,
      "learning_rate": 4.5e-06,
      "log_odds_chosen": 0.23358798027038574,
      "log_odds_ratio": -0.6189488172531128,
      "logits/chosen": 396.7557678222656,
      "logits/rejected": 388.23681640625,
      "logps/chosen": -1.4873348474502563,
      "logps/rejected": -1.6665433645248413,
      "loss": 2.0244,
      "nll_loss": 1.826873540878296,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7436674237251282,
      "rewards/margins": 0.08960425853729248,
      "rewards/rejected": -0.8332716822624207,
      "step": 60
    },
    {
      "epoch": 0.38748137108792846,
      "grad_norm": 27.614044189453125,
      "learning_rate": 4.875e-06,
      "log_odds_chosen": 0.3461737036705017,
      "log_odds_ratio": -0.6412385106086731,
      "logits/chosen": 397.4202575683594,
      "logits/rejected": 397.5688171386719,
      "logps/chosen": -1.3429419994354248,
      "logps/rejected": -1.6044105291366577,
      "loss": 1.945,
      "nll_loss": 1.5645616054534912,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.6714709997177124,
      "rewards/margins": 0.13073426485061646,
      "rewards/rejected": -0.8022052645683289,
      "step": 65
    },
    {
      "epoch": 0.4172876304023845,
      "grad_norm": 86.65094757080078,
      "learning_rate": 5.25e-06,
      "log_odds_chosen": 0.41045960783958435,
      "log_odds_ratio": -0.5924609303474426,
      "logits/chosen": 376.8070983886719,
      "logits/rejected": 384.87994384765625,
      "logps/chosen": -1.5336923599243164,
      "logps/rejected": -1.8701730966567993,
      "loss": 1.997,
      "nll_loss": 1.7713584899902344,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.7668461799621582,
      "rewards/margins": 0.16824033856391907,
      "rewards/rejected": -0.9350865483283997,
      "step": 70
    },
    {
      "epoch": 0.44709388971684055,
      "grad_norm": 75.32307434082031,
      "learning_rate": 5.625e-06,
      "log_odds_chosen": 0.5093179941177368,
      "log_odds_ratio": -0.5533502101898193,
      "logits/chosen": 403.20697021484375,
      "logits/rejected": 389.50067138671875,
      "logps/chosen": -1.303440809249878,
      "logps/rejected": -1.7079664468765259,
      "loss": 1.901,
      "nll_loss": 1.5844974517822266,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.651720404624939,
      "rewards/margins": 0.20226280391216278,
      "rewards/rejected": -0.8539832234382629,
      "step": 75
    },
    {
      "epoch": 0.47690014903129657,
      "grad_norm": 52.309078216552734,
      "learning_rate": 6e-06,
      "log_odds_chosen": 0.5146449208259583,
      "log_odds_ratio": -0.5773543119430542,
      "logits/chosen": 392.9078063964844,
      "logits/rejected": 413.81451416015625,
      "logps/chosen": -1.4215633869171143,
      "logps/rejected": -1.8292858600616455,
      "loss": 1.9712,
      "nll_loss": 1.699523687362671,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.7107816934585571,
      "rewards/margins": 0.20386135578155518,
      "rewards/rejected": -0.9146429300308228,
      "step": 80
    },
    {
      "epoch": 0.5067064083457526,
      "grad_norm": 665.8851318359375,
      "learning_rate": 6.375e-06,
      "log_odds_chosen": 0.4371088147163391,
      "log_odds_ratio": -0.5891958475112915,
      "logits/chosen": 411.912109375,
      "logits/rejected": 395.8421325683594,
      "logps/chosen": -1.437474012374878,
      "logps/rejected": -1.776432991027832,
      "loss": 2.0743,
      "nll_loss": 1.8243602514266968,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.718737006187439,
      "rewards/margins": 0.16947945952415466,
      "rewards/rejected": -0.888216495513916,
      "step": 85
    },
    {
      "epoch": 0.5365126676602087,
      "grad_norm": 57.34864044189453,
      "learning_rate": 6.750000000000001e-06,
      "log_odds_chosen": 0.9314719438552856,
      "log_odds_ratio": -0.49316421151161194,
      "logits/chosen": 410.67156982421875,
      "logits/rejected": 423.86431884765625,
      "logps/chosen": -1.415283441543579,
      "logps/rejected": -2.1992759704589844,
      "loss": 1.8288,
      "nll_loss": 1.5969065427780151,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.7076417207717896,
      "rewards/margins": 0.39199626445770264,
      "rewards/rejected": -1.0996379852294922,
      "step": 90
    },
    {
      "epoch": 0.5663189269746647,
      "grad_norm": 65.98050689697266,
      "learning_rate": 7.1249999999999995e-06,
      "log_odds_chosen": 0.38506197929382324,
      "log_odds_ratio": -0.5869079828262329,
      "logits/chosen": 378.6953430175781,
      "logits/rejected": 390.01422119140625,
      "logps/chosen": -1.212702989578247,
      "logps/rejected": -1.4467040300369263,
      "loss": 1.8818,
      "nll_loss": 1.4865398406982422,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.6063514947891235,
      "rewards/margins": 0.1170005351305008,
      "rewards/rejected": -0.7233520150184631,
      "step": 95
    },
    {
      "epoch": 0.5961251862891207,
      "grad_norm": 42.52804183959961,
      "learning_rate": 7.5e-06,
      "log_odds_chosen": 0.6413298845291138,
      "log_odds_ratio": -0.538486659526825,
      "logits/chosen": 412.3399353027344,
      "logits/rejected": 447.61865234375,
      "logps/chosen": -1.3371002674102783,
      "logps/rejected": -1.868131399154663,
      "loss": 1.9035,
      "nll_loss": 1.6383569240570068,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.6685501337051392,
      "rewards/margins": 0.2655157446861267,
      "rewards/rejected": -0.9340656995773315,
      "step": 100
    },
    {
      "epoch": 0.6259314456035767,
      "grad_norm": 48.187232971191406,
      "learning_rate": 7.319250547113999e-06,
      "log_odds_chosen": 0.7124954462051392,
      "log_odds_ratio": -0.5506631731987,
      "logits/chosen": 403.2848205566406,
      "logits/rejected": 415.0179138183594,
      "logps/chosen": -1.253138780593872,
      "logps/rejected": -1.768273115158081,
      "loss": 1.8545,
      "nll_loss": 1.6297533512115479,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.626569390296936,
      "rewards/margins": 0.25756722688674927,
      "rewards/rejected": -0.8841365575790405,
      "step": 105
    },
    {
      "epoch": 0.6557377049180327,
      "grad_norm": 195.05877685546875,
      "learning_rate": 7.150969419341942e-06,
      "log_odds_chosen": 0.4548686444759369,
      "log_odds_ratio": -0.569240927696228,
      "logits/chosen": 389.09381103515625,
      "logits/rejected": 386.32513427734375,
      "logps/chosen": -1.3020490407943726,
      "logps/rejected": -1.6546863317489624,
      "loss": 1.8713,
      "nll_loss": 1.6556203365325928,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.6510245203971863,
      "rewards/margins": 0.1763187050819397,
      "rewards/rejected": -0.8273431658744812,
      "step": 110
    },
    {
      "epoch": 0.6855439642324889,
      "grad_norm": 53.028961181640625,
      "learning_rate": 6.993786061802354e-06,
      "log_odds_chosen": 0.8375183939933777,
      "log_odds_ratio": -0.4603613018989563,
      "logits/chosen": 408.5062561035156,
      "logits/rejected": 414.22332763671875,
      "logps/chosen": -1.220641851425171,
      "logps/rejected": -1.8461415767669678,
      "loss": 1.8867,
      "nll_loss": 1.583599328994751,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.6103209257125854,
      "rewards/margins": 0.3127499222755432,
      "rewards/rejected": -0.9230707883834839,
      "step": 115
    },
    {
      "epoch": 0.7153502235469449,
      "grad_norm": 49.91960144042969,
      "learning_rate": 6.846531968814577e-06,
      "log_odds_chosen": 0.6342799663543701,
      "log_odds_ratio": -0.5294886827468872,
      "logits/chosen": 393.26812744140625,
      "logits/rejected": 391.69677734375,
      "logps/chosen": -1.1840012073516846,
      "logps/rejected": -1.6264234781265259,
      "loss": 1.9171,
      "nll_loss": 1.6122639179229736,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.5920006036758423,
      "rewards/margins": 0.22121115028858185,
      "rewards/rejected": -0.8132117390632629,
      "step": 120
    },
    {
      "epoch": 0.7451564828614009,
      "grad_norm": 26.490934371948242,
      "learning_rate": 6.708203932499369e-06,
      "log_odds_chosen": 0.3495193421840668,
      "log_odds_ratio": -0.6338833570480347,
      "logits/chosen": 394.9533386230469,
      "logits/rejected": 406.28662109375,
      "logps/chosen": -1.23225998878479,
      "logps/rejected": -1.4777755737304688,
      "loss": 1.8239,
      "nll_loss": 1.5472067594528198,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.616129994392395,
      "rewards/margins": 0.12275775521993637,
      "rewards/rejected": -0.7388877868652344,
      "step": 125
    },
    {
      "epoch": 0.7749627421758569,
      "grad_norm": 45.540184020996094,
      "learning_rate": 6.577935144802719e-06,
      "log_odds_chosen": 0.4649319052696228,
      "log_odds_ratio": -0.5916207432746887,
      "logits/chosen": 417.3863830566406,
      "logits/rejected": 405.1355895996094,
      "logps/chosen": -1.2279858589172363,
      "logps/rejected": -1.5856168270111084,
      "loss": 1.8861,
      "nll_loss": 1.565514326095581,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.6139929294586182,
      "rewards/margins": 0.17881537973880768,
      "rewards/rejected": -0.7928084135055542,
      "step": 130
    },
    {
      "epoch": 0.8047690014903129,
      "grad_norm": 42.16865921020508,
      "learning_rate": 6.454972243679028e-06,
      "log_odds_chosen": 0.46650758385658264,
      "log_odds_ratio": -0.6158651113510132,
      "logits/chosen": 421.04730224609375,
      "logits/rejected": 423.00775146484375,
      "logps/chosen": -1.509341835975647,
      "logps/rejected": -1.8879953622817993,
      "loss": 1.9281,
      "nll_loss": 1.723570466041565,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.7546709179878235,
      "rewards/margins": 0.18932664394378662,
      "rewards/rejected": -0.9439976811408997,
      "step": 135
    },
    {
      "epoch": 0.834575260804769,
      "grad_norm": 41.30878448486328,
      "learning_rate": 6.338656910463875e-06,
      "log_odds_chosen": 0.27211299538612366,
      "log_odds_ratio": -0.680214524269104,
      "logits/chosen": 404.1061096191406,
      "logits/rejected": 409.54559326171875,
      "logps/chosen": -1.4520971775054932,
      "logps/rejected": -1.670898199081421,
      "loss": 1.9311,
      "nll_loss": 1.7876545190811157,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.7260485887527466,
      "rewards/margins": 0.10940053313970566,
      "rewards/rejected": -0.8354490995407104,
      "step": 140
    },
    {
      "epoch": 0.8643815201192251,
      "grad_norm": 58.79216003417969,
      "learning_rate": 6.228410989030498e-06,
      "log_odds_chosen": 0.13751578330993652,
      "log_odds_ratio": -0.6719902753829956,
      "logits/chosen": 402.82342529296875,
      "logits/rejected": 405.1291809082031,
      "logps/chosen": -1.3680763244628906,
      "logps/rejected": -1.4593302011489868,
      "loss": 1.9064,
      "nll_loss": 1.6308927536010742,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.6840381622314453,
      "rewards/margins": 0.045626964420080185,
      "rewards/rejected": -0.7296651005744934,
      "step": 145
    },
    {
      "epoch": 0.8941877794336811,
      "grad_norm": 28.582611083984375,
      "learning_rate": 6.123724356957946e-06,
      "log_odds_chosen": 0.8705786466598511,
      "log_odds_ratio": -0.5025649666786194,
      "logits/chosen": 407.1871643066406,
      "logits/rejected": 423.8992614746094,
      "logps/chosen": -1.2447984218597412,
      "logps/rejected": -1.9607397317886353,
      "loss": 1.8397,
      "nll_loss": 1.4880900382995605,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6223992109298706,
      "rewards/margins": 0.357970654964447,
      "rewards/rejected": -0.9803698658943176,
      "step": 150
    },
    {
      "epoch": 0.9239940387481371,
      "grad_norm": 56.87172317504883,
      "learning_rate": 6.024144966768742e-06,
      "log_odds_chosen": 0.32878491282463074,
      "log_odds_ratio": -0.6313835382461548,
      "logits/chosen": 388.4563293457031,
      "logits/rejected": 402.23834228515625,
      "logps/chosen": -1.365476131439209,
      "logps/rejected": -1.6416184902191162,
      "loss": 1.8579,
      "nll_loss": 1.458609938621521,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.6827380657196045,
      "rewards/margins": 0.13807126879692078,
      "rewards/rejected": -0.8208092451095581,
      "step": 155
    },
    {
      "epoch": 0.9538002980625931,
      "grad_norm": 52.100013732910156,
      "learning_rate": 5.929270612815711e-06,
      "log_odds_chosen": 0.2496444433927536,
      "log_odds_ratio": -0.6904889345169067,
      "logits/chosen": 397.15887451171875,
      "logits/rejected": 440.45587158203125,
      "logps/chosen": -1.2235276699066162,
      "logps/rejected": -1.4282524585723877,
      "loss": 1.8144,
      "nll_loss": 1.502598762512207,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.6117638349533081,
      "rewards/margins": 0.10236239433288574,
      "rewards/rejected": -0.7141262292861938,
      "step": 160
    },
    {
      "epoch": 0.9836065573770492,
      "grad_norm": 62.34568786621094,
      "learning_rate": 5.838742081211422e-06,
      "log_odds_chosen": 0.25045713782310486,
      "log_odds_ratio": -0.6064125895500183,
      "logits/chosen": 407.72943115234375,
      "logits/rejected": 420.569580078125,
      "logps/chosen": -1.2549211978912354,
      "logps/rejected": -1.4284086227416992,
      "loss": 1.8398,
      "nll_loss": 1.5022889375686646,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6274605989456177,
      "rewards/margins": 0.0867437794804573,
      "rewards/rejected": -0.7142043113708496,
      "step": 165
    },
    {
      "epoch": 0.9955290611028316,
      "eval_log_odds_chosen": 0.25424009561538696,
      "eval_log_odds_ratio": -0.6749479174613953,
      "eval_logits/chosen": 331.1551513671875,
      "eval_logits/rejected": 302.9094543457031,
      "eval_logps/chosen": -1.1524564027786255,
      "eval_logps/rejected": -1.343080997467041,
      "eval_loss": 1.8982083797454834,
      "eval_nll_loss": 1.5454045534133911,
      "eval_rewards/accuracies": 0.5179855823516846,
      "eval_rewards/chosen": -0.5762282013893127,
      "eval_rewards/margins": 0.09531236439943314,
      "eval_rewards/rejected": -0.6715404987335205,
      "eval_runtime": 378.1633,
      "eval_samples_per_second": 1.462,
      "eval_steps_per_second": 0.368,
      "step": 167
    },
    {
      "epoch": 1.0134128166915053,
      "grad_norm": 19.05902862548828,
      "learning_rate": 5.752237416355279e-06,
      "log_odds_chosen": 0.8472374081611633,
      "log_odds_ratio": -0.5489901304244995,
      "logits/chosen": 389.7673645019531,
      "logits/rejected": 412.4244079589844,
      "logps/chosen": -1.1339842081069946,
      "logps/rejected": -1.749389410018921,
      "loss": 1.6458,
      "nll_loss": 1.2897634506225586,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.5669921040534973,
      "rewards/margins": 0.3077026307582855,
      "rewards/rejected": -0.8746947050094604,
      "step": 170
    },
    {
      "epoch": 1.0432190760059612,
      "grad_norm": 39.458702087402344,
      "learning_rate": 5.669467095138408e-06,
      "log_odds_chosen": 1.5121970176696777,
      "log_odds_ratio": -0.30622434616088867,
      "logits/chosen": 365.8326416015625,
      "logits/rejected": 406.28411865234375,
      "logps/chosen": -0.7569233775138855,
      "logps/rejected": -1.806539535522461,
      "loss": 1.2547,
      "nll_loss": 1.183380365371704,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.37846168875694275,
      "rewards/margins": 0.5248081088066101,
      "rewards/rejected": -0.9032697677612305,
      "step": 175
    },
    {
      "epoch": 1.0730253353204173,
      "grad_norm": 31.42799949645996,
      "learning_rate": 5.5901699437494744e-06,
      "log_odds_chosen": 1.5321556329727173,
      "log_odds_ratio": -0.2848857045173645,
      "logits/chosen": 364.0169677734375,
      "logits/rejected": 339.5393371582031,
      "logps/chosen": -0.8578670620918274,
      "logps/rejected": -1.9486223459243774,
      "loss": 1.3431,
      "nll_loss": 1.3115805387496948,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.4289335310459137,
      "rewards/margins": 0.5453776717185974,
      "rewards/rejected": -0.9743111729621887,
      "step": 180
    },
    {
      "epoch": 1.1028315946348732,
      "grad_norm": 35.6085090637207,
      "learning_rate": 5.514109665703559e-06,
      "log_odds_chosen": 2.223374843597412,
      "log_odds_ratio": -0.2399306744337082,
      "logits/chosen": 396.5127868652344,
      "logits/rejected": 386.45831298828125,
      "logps/chosen": -0.8754903078079224,
      "logps/rejected": -2.4870636463165283,
      "loss": 1.2172,
      "nll_loss": 1.070143222808838,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.4377451539039612,
      "rewards/margins": 0.8057867288589478,
      "rewards/rejected": -1.2435318231582642,
      "step": 185
    },
    {
      "epoch": 1.1326378539493294,
      "grad_norm": 22.422922134399414,
      "learning_rate": 5.441071875825088e-06,
      "log_odds_chosen": 1.2998098134994507,
      "log_odds_ratio": -0.32330313324928284,
      "logits/chosen": 364.8489074707031,
      "logits/rejected": 408.52972412109375,
      "logps/chosen": -0.870254397392273,
      "logps/rejected": -1.7498779296875,
      "loss": 1.2807,
      "nll_loss": 1.1046603918075562,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4351271986961365,
      "rewards/margins": 0.4398117661476135,
      "rewards/rejected": -0.87493896484375,
      "step": 190
    },
    {
      "epoch": 1.1624441132637853,
      "grad_norm": 25.98566246032715,
      "learning_rate": 5.370861555295747e-06,
      "log_odds_chosen": 2.002765417098999,
      "log_odds_ratio": -0.24665918946266174,
      "logits/chosen": 384.2296142578125,
      "logits/rejected": 398.67205810546875,
      "logps/chosen": -0.8954353332519531,
      "logps/rejected": -2.385274648666382,
      "loss": 1.2781,
      "nll_loss": 1.1818829774856567,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.44771766662597656,
      "rewards/margins": 0.7449195981025696,
      "rewards/rejected": -1.192637324333191,
      "step": 195
    },
    {
      "epoch": 1.1922503725782414,
      "grad_norm": 23.273193359375,
      "learning_rate": 5.303300858899106e-06,
      "log_odds_chosen": 2.1724612712860107,
      "log_odds_ratio": -0.26438647508621216,
      "logits/chosen": 413.26361083984375,
      "logits/rejected": 396.8680419921875,
      "logps/chosen": -0.730914831161499,
      "logps/rejected": -2.3943963050842285,
      "loss": 1.2803,
      "nll_loss": 1.0805118083953857,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.3654574155807495,
      "rewards/margins": 0.83174067735672,
      "rewards/rejected": -1.1971981525421143,
      "step": 200
    },
    {
      "epoch": 1.2220566318926975,
      "grad_norm": 27.806596755981445,
      "learning_rate": 5.238227218271837e-06,
      "log_odds_chosen": 2.4484591484069824,
      "log_odds_ratio": -0.22746221721172333,
      "logits/chosen": 365.72119140625,
      "logits/rejected": 359.6267395019531,
      "logps/chosen": -0.758979320526123,
      "logps/rejected": -2.573237180709839,
      "loss": 1.2398,
      "nll_loss": 1.1106595993041992,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.3794896602630615,
      "rewards/margins": 0.9071289896965027,
      "rewards/rejected": -1.2866185903549194,
      "step": 205
    },
    {
      "epoch": 1.2518628912071534,
      "grad_norm": 22.43109130859375,
      "learning_rate": 5.175491695067656e-06,
      "log_odds_chosen": 2.793255567550659,
      "log_odds_ratio": -0.23325839638710022,
      "logits/chosen": 377.73443603515625,
      "logits/rejected": 404.66644287109375,
      "logps/chosen": -0.7964418530464172,
      "logps/rejected": -3.0091164112091064,
      "loss": 1.2371,
      "nll_loss": 1.0737245082855225,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.3982209265232086,
      "rewards/margins": 1.106337308883667,
      "rewards/rejected": -1.5045582056045532,
      "step": 210
    },
    {
      "epoch": 1.2816691505216096,
      "grad_norm": 49.615081787109375,
      "learning_rate": 5.114957546028551e-06,
      "log_odds_chosen": 2.5913193225860596,
      "log_odds_ratio": -0.23048730194568634,
      "logits/chosen": 401.5857238769531,
      "logits/rejected": 382.7472229003906,
      "logps/chosen": -0.7830225229263306,
      "logps/rejected": -2.782843828201294,
      "loss": 1.2555,
      "nll_loss": 1.1490776538848877,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.3915112614631653,
      "rewards/margins": 0.9999106526374817,
      "rewards/rejected": -1.391421914100647,
      "step": 215
    },
    {
      "epoch": 1.3114754098360657,
      "grad_norm": 37.056453704833984,
      "learning_rate": 5.056498968474315e-06,
      "log_odds_chosen": 1.4505621194839478,
      "log_odds_ratio": -0.3243867754936218,
      "logits/chosen": 393.34063720703125,
      "logits/rejected": 387.152099609375,
      "logps/chosen": -0.8490570783615112,
      "logps/rejected": -1.8110103607177734,
      "loss": 1.2897,
      "nll_loss": 1.1027252674102783,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.4245285391807556,
      "rewards/margins": 0.48097652196884155,
      "rewards/rejected": -0.9055051803588867,
      "step": 220
    },
    {
      "epoch": 1.3412816691505216,
      "grad_norm": 22.3747615814209,
      "learning_rate": 4.9999999999999996e-06,
      "log_odds_chosen": 1.0405768156051636,
      "log_odds_ratio": -0.37774857878685,
      "logits/chosen": 391.2264099121094,
      "logits/rejected": 382.2864685058594,
      "logps/chosen": -1.0447379350662231,
      "logps/rejected": -1.8071935176849365,
      "loss": 1.2781,
      "nll_loss": 1.2478437423706055,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5223689675331116,
      "rewards/margins": 0.38122788071632385,
      "rewards/rejected": -0.9035967588424683,
      "step": 225
    },
    {
      "epoch": 1.3710879284649775,
      "grad_norm": 22.041149139404297,
      "learning_rate": 4.945353550468404e-06,
      "log_odds_chosen": 2.0077407360076904,
      "log_odds_ratio": -0.25288665294647217,
      "logits/chosen": 370.07696533203125,
      "logits/rejected": 385.3541564941406,
      "logps/chosen": -0.90089350938797,
      "logps/rejected": -2.3661916255950928,
      "loss": 1.2539,
      "nll_loss": 0.989953339099884,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.450446754693985,
      "rewards/margins": 0.7326492667198181,
      "rewards/rejected": -1.1830958127975464,
      "step": 230
    },
    {
      "epoch": 1.4008941877794336,
      "grad_norm": 33.446224212646484,
      "learning_rate": 4.8924605479008165e-06,
      "log_odds_chosen": 1.8689000606536865,
      "log_odds_ratio": -0.24617977440357208,
      "logits/chosen": 391.69732666015625,
      "logits/rejected": 402.88836669921875,
      "logps/chosen": -0.8436087369918823,
      "logps/rejected": -2.213329553604126,
      "loss": 1.2797,
      "nll_loss": 1.1102927923202515,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.42180436849594116,
      "rewards/margins": 0.684860348701477,
      "rewards/rejected": -1.106664776802063,
      "step": 235
    },
    {
      "epoch": 1.4307004470938898,
      "grad_norm": 19.861835479736328,
      "learning_rate": 4.8412291827592715e-06,
      "log_odds_chosen": 1.848902940750122,
      "log_odds_ratio": -0.3047091066837311,
      "logits/chosen": 415.73486328125,
      "logits/rejected": 420.25909423828125,
      "logps/chosen": -0.8824371099472046,
      "logps/rejected": -2.3095035552978516,
      "loss": 1.2891,
      "nll_loss": 1.1334781646728516,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.4412185549736023,
      "rewards/margins": 0.7135332226753235,
      "rewards/rejected": -1.1547517776489258,
      "step": 240
    },
    {
      "epoch": 1.4605067064083457,
      "grad_norm": 31.12167739868164,
      "learning_rate": 4.79157423749955e-06,
      "log_odds_chosen": 1.9949026107788086,
      "log_odds_ratio": -0.24574021995067596,
      "logits/chosen": 409.1098937988281,
      "logits/rejected": 394.39593505859375,
      "logps/chosen": -0.7639743089675903,
      "logps/rejected": -2.1869406700134277,
      "loss": 1.312,
      "nll_loss": 1.2122286558151245,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.38198715448379517,
      "rewards/margins": 0.7114832997322083,
      "rewards/rejected": -1.0934703350067139,
      "step": 245
    },
    {
      "epoch": 1.4903129657228018,
      "grad_norm": 20.75017547607422,
      "learning_rate": 4.74341649025257e-06,
      "log_odds_chosen": 2.6609864234924316,
      "log_odds_ratio": -0.25564372539520264,
      "logits/chosen": 380.4820251464844,
      "logits/rejected": 385.7266845703125,
      "logps/chosen": -0.804089367389679,
      "logps/rejected": -2.9035263061523438,
      "loss": 1.2332,
      "nll_loss": 1.0138413906097412,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4020446836948395,
      "rewards/margins": 1.0497183799743652,
      "rewards/rejected": -1.4517631530761719,
      "step": 250
    },
    {
      "epoch": 1.520119225037258,
      "grad_norm": 26.518312454223633,
      "learning_rate": 4.696682183138622e-06,
      "log_odds_chosen": 2.0447866916656494,
      "log_odds_ratio": -0.22285214066505432,
      "logits/chosen": 399.64971923828125,
      "logits/rejected": 400.3153076171875,
      "logps/chosen": -0.8079016804695129,
      "logps/rejected": -2.2692599296569824,
      "loss": 1.2551,
      "nll_loss": 1.0453466176986694,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.40395084023475647,
      "rewards/margins": 0.7306792140007019,
      "rewards/rejected": -1.1346299648284912,
      "step": 255
    },
    {
      "epoch": 1.5499254843517138,
      "grad_norm": 18.25040626525879,
      "learning_rate": 4.6513025470953165e-06,
      "log_odds_chosen": 1.7279441356658936,
      "log_odds_ratio": -0.3020802140235901,
      "logits/chosen": 380.86590576171875,
      "logits/rejected": 420.64007568359375,
      "logps/chosen": -0.8852326273918152,
      "logps/rejected": -2.209996461868286,
      "loss": 1.2132,
      "nll_loss": 1.1014916896820068,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.4426163136959076,
      "rewards/margins": 0.6623818278312683,
      "rewards/rejected": -1.104998230934143,
      "step": 260
    },
    {
      "epoch": 1.5797317436661698,
      "grad_norm": 17.494176864624023,
      "learning_rate": 4.607213376254634e-06,
      "log_odds_chosen": 1.6819546222686768,
      "log_odds_ratio": -0.3340403437614441,
      "logits/chosen": 388.51068115234375,
      "logits/rejected": 409.89971923828125,
      "logps/chosen": -0.8424839973449707,
      "logps/rejected": -2.041083574295044,
      "loss": 1.3572,
      "nll_loss": 1.1232020854949951,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.42124199867248535,
      "rewards/margins": 0.5992996692657471,
      "rewards/rejected": -1.020541787147522,
      "step": 265
    },
    {
      "epoch": 1.6095380029806259,
      "grad_norm": 34.47447967529297,
      "learning_rate": 4.564354645876384e-06,
      "log_odds_chosen": 1.859960913658142,
      "log_odds_ratio": -0.2928032875061035,
      "logits/chosen": 372.35052490234375,
      "logits/rejected": 382.39117431640625,
      "logps/chosen": -0.9743008613586426,
      "logps/rejected": -2.432216167449951,
      "loss": 1.3166,
      "nll_loss": 1.270026683807373,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.4871504306793213,
      "rewards/margins": 0.7289576530456543,
      "rewards/rejected": -1.2161080837249756,
      "step": 270
    },
    {
      "epoch": 1.639344262295082,
      "grad_norm": 21.639741897583008,
      "learning_rate": 4.522670168666455e-06,
      "log_odds_chosen": 1.463616967201233,
      "log_odds_ratio": -0.3027709126472473,
      "logits/chosen": 362.27044677734375,
      "logits/rejected": 350.851318359375,
      "logps/chosen": -0.7592640519142151,
      "logps/rejected": -1.6749483346939087,
      "loss": 1.2876,
      "nll_loss": 1.0598416328430176,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.37963202595710754,
      "rewards/margins": 0.45784205198287964,
      "rewards/rejected": -0.8374741673469543,
      "step": 275
    },
    {
      "epoch": 1.669150521609538,
      "grad_norm": 32.02434158325195,
      "learning_rate": 4.482107285003976e-06,
      "log_odds_chosen": 1.598225474357605,
      "log_odds_ratio": -0.3603498339653015,
      "logits/chosen": 410.98687744140625,
      "logits/rejected": 397.2725830078125,
      "logps/chosen": -0.8709872364997864,
      "logps/rejected": -1.9590165615081787,
      "loss": 1.285,
      "nll_loss": 1.1194555759429932,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.4354936182498932,
      "rewards/margins": 0.5440146923065186,
      "rewards/rejected": -0.9795082807540894,
      "step": 280
    },
    {
      "epoch": 1.698956780923994,
      "grad_norm": 18.135896682739258,
      "learning_rate": 4.442616583193192e-06,
      "log_odds_chosen": 2.131542921066284,
      "log_odds_ratio": -0.2719998061656952,
      "logits/chosen": 375.759033203125,
      "logits/rejected": 377.7268981933594,
      "logps/chosen": -0.7118241786956787,
      "logps/rejected": -2.2590417861938477,
      "loss": 1.2113,
      "nll_loss": 1.054491400718689,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.35591208934783936,
      "rewards/margins": 0.7736088037490845,
      "rewards/rejected": -1.1295208930969238,
      "step": 285
    },
    {
      "epoch": 1.7287630402384502,
      "grad_norm": 16.36552619934082,
      "learning_rate": 4.404151646360276e-06,
      "log_odds_chosen": 2.539419651031494,
      "log_odds_ratio": -0.2885150611400604,
      "logits/chosen": 395.09832763671875,
      "logits/rejected": 429.5597229003906,
      "logps/chosen": -0.8963809013366699,
      "logps/rejected": -2.95475697517395,
      "loss": 1.2765,
      "nll_loss": 1.1113842725753784,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.44819045066833496,
      "rewards/margins": 1.0291881561279297,
      "rewards/rejected": -1.477378487586975,
      "step": 290
    },
    {
      "epoch": 1.758569299552906,
      "grad_norm": 20.132436752319336,
      "learning_rate": 4.366668823046865e-06,
      "log_odds_chosen": 1.553693413734436,
      "log_odds_ratio": -0.3133796155452728,
      "logits/chosen": 371.56475830078125,
      "logits/rejected": 413.505615234375,
      "logps/chosen": -0.9781205058097839,
      "logps/rejected": -2.0690226554870605,
      "loss": 1.2963,
      "nll_loss": 1.1234586238861084,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.48906025290489197,
      "rewards/margins": 0.5454510450363159,
      "rewards/rejected": -1.0345113277435303,
      "step": 295
    },
    {
      "epoch": 1.788375558867362,
      "grad_norm": 26.103137969970703,
      "learning_rate": 4.330127018922194e-06,
      "log_odds_chosen": 2.1462044715881348,
      "log_odds_ratio": -0.24198386073112488,
      "logits/chosen": 409.97161865234375,
      "logits/rejected": 383.2239990234375,
      "logps/chosen": -0.7521153688430786,
      "logps/rejected": -2.3262906074523926,
      "loss": 1.1963,
      "nll_loss": 1.022304654121399,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.3760576844215393,
      "rewards/margins": 0.7870877385139465,
      "rewards/rejected": -1.1631453037261963,
      "step": 300
    },
    {
      "epoch": 1.8181818181818183,
      "grad_norm": 20.291587829589844,
      "learning_rate": 4.294487507354011e-06,
      "log_odds_chosen": 1.8153495788574219,
      "log_odds_ratio": -0.3603833019733429,
      "logits/chosen": 399.9996032714844,
      "logits/rejected": 405.7534484863281,
      "logps/chosen": -1.029065728187561,
      "logps/rejected": -2.513669490814209,
      "loss": 1.2618,
      "nll_loss": 1.2645294666290283,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5145328640937805,
      "rewards/margins": 0.742301881313324,
      "rewards/rejected": -1.2568347454071045,
      "step": 305
    },
    {
      "epoch": 1.8479880774962743,
      "grad_norm": 19.601377487182617,
      "learning_rate": 4.259713756852986e-06,
      "log_odds_chosen": 3.238518238067627,
      "log_odds_ratio": -0.22569267451763153,
      "logits/chosen": 380.0532531738281,
      "logits/rejected": 386.49920654296875,
      "logps/chosen": -0.8288024067878723,
      "logps/rejected": -3.505189895629883,
      "loss": 1.3118,
      "nll_loss": 1.1829016208648682,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.41440120339393616,
      "rewards/margins": 1.338193655014038,
      "rewards/rejected": -1.7525949478149414,
      "step": 310
    },
    {
      "epoch": 1.8777943368107302,
      "grad_norm": 19.882904052734375,
      "learning_rate": 4.225771273642583e-06,
      "log_odds_chosen": 1.2986266613006592,
      "log_odds_ratio": -0.3742119073867798,
      "logits/chosen": 354.4183044433594,
      "logits/rejected": 373.57757568359375,
      "logps/chosen": -1.0900706052780151,
      "logps/rejected": -2.065725326538086,
      "loss": 1.3407,
      "nll_loss": 1.3649649620056152,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.5450353026390076,
      "rewards/margins": 0.4878273606300354,
      "rewards/rejected": -1.032862663269043,
      "step": 315
    },
    {
      "epoch": 1.9076005961251863,
      "grad_norm": 21.79350471496582,
      "learning_rate": 4.192627457812106e-06,
      "log_odds_chosen": 2.3978288173675537,
      "log_odds_ratio": -0.24583116173744202,
      "logits/chosen": 379.4220886230469,
      "logits/rejected": 400.0596618652344,
      "logps/chosen": -0.7697328329086304,
      "logps/rejected": -2.6053688526153564,
      "loss": 1.1966,
      "nll_loss": 1.0758590698242188,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.3848664164543152,
      "rewards/margins": 0.9178179502487183,
      "rewards/rejected": -1.3026844263076782,
      "step": 320
    },
    {
      "epoch": 1.9374068554396424,
      "grad_norm": 25.364540100097656,
      "learning_rate": 4.1602514716892185e-06,
      "log_odds_chosen": 2.511842727661133,
      "log_odds_ratio": -0.21811020374298096,
      "logits/chosen": 377.62139892578125,
      "logits/rejected": 409.9249572753906,
      "logps/chosen": -0.8184257745742798,
      "logps/rejected": -2.717911720275879,
      "loss": 1.2025,
      "nll_loss": 1.0009828805923462,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.4092128872871399,
      "rewards/margins": 0.9497429728507996,
      "rewards/rejected": -1.3589558601379395,
      "step": 325
    },
    {
      "epoch": 1.9672131147540983,
      "grad_norm": 24.96152114868164,
      "learning_rate": 4.128614119223853e-06,
      "log_odds_chosen": 1.425945520401001,
      "log_odds_ratio": -0.2635498642921448,
      "logits/chosen": 370.4853515625,
      "logits/rejected": 376.5760192871094,
      "logps/chosen": -0.8058730959892273,
      "logps/rejected": -1.770816445350647,
      "loss": 1.257,
      "nll_loss": 0.9558019638061523,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.40293654799461365,
      "rewards/margins": 0.48247161507606506,
      "rewards/rejected": -0.8854082226753235,
      "step": 330
    },
    {
      "epoch": 1.9970193740685542,
      "grad_norm": 17.690673828125,
      "learning_rate": 4.097687735311148e-06,
      "log_odds_chosen": 1.0468049049377441,
      "log_odds_ratio": -0.43120455741882324,
      "logits/chosen": 390.3360900878906,
      "logits/rejected": 377.1321105957031,
      "logps/chosen": -1.1694064140319824,
      "logps/rejected": -1.9213879108428955,
      "loss": 1.299,
      "nll_loss": 1.3618862628936768,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5847032070159912,
      "rewards/margins": 0.3759908080101013,
      "rewards/rejected": -0.9606939554214478,
      "step": 335
    },
    {
      "epoch": 1.9970193740685542,
      "eval_log_odds_chosen": 0.30739960074424744,
      "eval_log_odds_ratio": -0.6665536761283875,
      "eval_logits/chosen": 313.4974365234375,
      "eval_logits/rejected": 282.3182678222656,
      "eval_logps/chosen": -1.1013935804367065,
      "eval_logps/rejected": -1.3020689487457275,
      "eval_loss": 1.8186225891113281,
      "eval_nll_loss": 1.4682071208953857,
      "eval_rewards/accuracies": 0.5395683646202087,
      "eval_rewards/chosen": -0.5506967902183533,
      "eval_rewards/margins": 0.10033769905567169,
      "eval_rewards/rejected": -0.6510344743728638,
      "eval_runtime": 378.4505,
      "eval_samples_per_second": 1.461,
      "eval_steps_per_second": 0.367,
      "step": 335
    },
    {
      "epoch": 2.0268256333830106,
      "grad_norm": 19.424360275268555,
      "learning_rate": 4.0674460840998035e-06,
      "log_odds_chosen": 3.000514507293701,
      "log_odds_ratio": -0.10731367021799088,
      "logits/chosen": 394.8528747558594,
      "logits/rejected": 368.841796875,
      "logps/chosen": -0.4828576147556305,
      "logps/rejected": -2.412517547607422,
      "loss": 0.7587,
      "nll_loss": 0.8271492123603821,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.24142880737781525,
      "rewards/margins": 0.9648300409317017,
      "rewards/rejected": -1.206258773803711,
      "step": 340
    },
    {
      "epoch": 2.0566318926974665,
      "grad_norm": 13.576729774475098,
      "learning_rate": 4.037864265436242e-06,
      "log_odds_chosen": 3.940922498703003,
      "log_odds_ratio": -0.07164354622364044,
      "logits/chosen": 342.024658203125,
      "logits/rejected": 342.9552307128906,
      "logps/chosen": -0.48198527097702026,
      "logps/rejected": -3.1786067485809326,
      "loss": 0.6501,
      "nll_loss": 0.6376822590827942,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.24099263548851013,
      "rewards/margins": 1.3483107089996338,
      "rewards/rejected": -1.5893033742904663,
      "step": 345
    },
    {
      "epoch": 2.0864381520119224,
      "grad_norm": 13.920299530029297,
      "learning_rate": 4.008918628686366e-06,
      "log_odds_chosen": 3.9218780994415283,
      "log_odds_ratio": -0.04757709056138992,
      "logits/chosen": 334.36187744140625,
      "logits/rejected": 374.3761291503906,
      "logps/chosen": -0.4404733180999756,
      "logps/rejected": -3.2984976768493652,
      "loss": 0.6334,
      "nll_loss": 0.6365537643432617,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2202366590499878,
      "rewards/margins": 1.4290122985839844,
      "rewards/rejected": -1.6492488384246826,
      "step": 350
    },
    {
      "epoch": 2.1162444113263787,
      "grad_norm": 12.827689170837402,
      "learning_rate": 3.980586693257065e-06,
      "log_odds_chosen": 4.072226047515869,
      "log_odds_ratio": -0.05848418548703194,
      "logits/chosen": 343.1537170410156,
      "logits/rejected": 358.5604248046875,
      "logps/chosen": -0.41472020745277405,
      "logps/rejected": -3.3263726234436035,
      "loss": 0.6337,
      "nll_loss": 0.6818060278892517,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20736010372638702,
      "rewards/margins": 1.4558260440826416,
      "rewards/rejected": -1.6631863117218018,
      "step": 355
    },
    {
      "epoch": 2.1460506706408347,
      "grad_norm": 14.782392501831055,
      "learning_rate": 3.952847075210474e-06,
      "log_odds_chosen": 4.195507049560547,
      "log_odds_ratio": -0.03957627713680267,
      "logits/chosen": 346.2444152832031,
      "logits/rejected": 393.8316955566406,
      "logps/chosen": -0.4012891352176666,
      "logps/rejected": -3.2268481254577637,
      "loss": 0.62,
      "nll_loss": 0.5778239965438843,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2006445676088333,
      "rewards/margins": 1.4127795696258545,
      "rewards/rejected": -1.6134240627288818,
      "step": 360
    },
    {
      "epoch": 2.1758569299552906,
      "grad_norm": 13.0780029296875,
      "learning_rate": 3.925679419426603e-06,
      "log_odds_chosen": 3.8077125549316406,
      "log_odds_ratio": -0.06930537521839142,
      "logits/chosen": 345.62933349609375,
      "logits/rejected": 324.48724365234375,
      "logps/chosen": -0.416290283203125,
      "logps/rejected": -3.0446524620056152,
      "loss": 0.6211,
      "nll_loss": 0.5697866678237915,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.2081451416015625,
      "rewards/margins": 1.3141810894012451,
      "rewards/rejected": -1.5223262310028076,
      "step": 365
    },
    {
      "epoch": 2.2056631892697465,
      "grad_norm": 13.796915054321289,
      "learning_rate": 3.8990643368252725e-06,
      "log_odds_chosen": 3.758181095123291,
      "log_odds_ratio": -0.08863835036754608,
      "logits/chosen": 309.7760009765625,
      "logits/rejected": 327.28875732421875,
      "logps/chosen": -0.4810813367366791,
      "logps/rejected": -3.211531400680542,
      "loss": 0.6805,
      "nll_loss": 0.6794728636741638,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.24054066836833954,
      "rewards/margins": 1.365225076675415,
      "rewards/rejected": -1.605765700340271,
      "step": 370
    },
    {
      "epoch": 2.235469448584203,
      "grad_norm": 12.518247604370117,
      "learning_rate": 3.872983346207417e-06,
      "log_odds_chosen": 3.9930922985076904,
      "log_odds_ratio": -0.07049717009067535,
      "logits/chosen": 364.5960388183594,
      "logits/rejected": 381.1838684082031,
      "logps/chosen": -0.47075748443603516,
      "logps/rejected": -3.3296546936035156,
      "loss": 0.6191,
      "nll_loss": 0.6227518320083618,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.23537874221801758,
      "rewards/margins": 1.4294484853744507,
      "rewards/rejected": -1.6648273468017578,
      "step": 375
    },
    {
      "epoch": 2.2652757078986587,
      "grad_norm": 23.583436965942383,
      "learning_rate": 3.847418820319329e-06,
      "log_odds_chosen": 3.55949330329895,
      "log_odds_ratio": -0.048723362386226654,
      "logits/chosen": 364.0585632324219,
      "logits/rejected": 337.66424560546875,
      "logps/chosen": -0.41810473799705505,
      "logps/rejected": -2.7338831424713135,
      "loss": 0.6367,
      "nll_loss": 0.6199051737785339,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20905236899852753,
      "rewards/margins": 1.1578892469406128,
      "rewards/rejected": -1.3669415712356567,
      "step": 380
    },
    {
      "epoch": 2.2950819672131146,
      "grad_norm": 12.274104118347168,
      "learning_rate": 3.822353935782191e-06,
      "log_odds_chosen": 4.226048946380615,
      "log_odds_ratio": -0.04617181420326233,
      "logits/chosen": 359.08154296875,
      "logits/rejected": 344.7435607910156,
      "logps/chosen": -0.36728304624557495,
      "logps/rejected": -3.1797947883605957,
      "loss": 0.5982,
      "nll_loss": 0.536335289478302,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.18364152312278748,
      "rewards/margins": 1.4062559604644775,
      "rewards/rejected": -1.5898973941802979,
      "step": 385
    },
    {
      "epoch": 2.3248882265275705,
      "grad_norm": 15.543499946594238,
      "learning_rate": 3.79777262656375e-06,
      "log_odds_chosen": 3.8094513416290283,
      "log_odds_ratio": -0.03931097313761711,
      "logits/chosen": 373.632568359375,
      "logits/rejected": 393.21295166015625,
      "logps/chosen": -0.46378618478775024,
      "logps/rejected": -3.2327797412872314,
      "loss": 0.667,
      "nll_loss": 0.6460134387016296,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.23189309239387512,
      "rewards/margins": 1.3844965696334839,
      "rewards/rejected": -1.6163898706436157,
      "step": 390
    },
    {
      "epoch": 2.354694485842027,
      "grad_norm": 13.162529945373535,
      "learning_rate": 3.773659540699707e-06,
      "log_odds_chosen": 3.9874942302703857,
      "log_odds_ratio": -0.05443572998046875,
      "logits/chosen": 368.2042236328125,
      "logits/rejected": 346.73590087890625,
      "logps/chosen": -0.3340614438056946,
      "logps/rejected": -2.9790287017822266,
      "loss": 0.5966,
      "nll_loss": 0.5552054643630981,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1670307219028473,
      "rewards/margins": 1.3224836587905884,
      "rewards/rejected": -1.4895143508911133,
      "step": 395
    },
    {
      "epoch": 2.384500745156483,
      "grad_norm": 11.184009552001953,
      "learning_rate": 3.75e-06,
      "log_odds_chosen": 4.703558444976807,
      "log_odds_ratio": -0.06566660106182098,
      "logits/chosen": 368.7803955078125,
      "logits/rejected": 383.49188232421875,
      "logps/chosen": -0.4126485288143158,
      "logps/rejected": -3.8882851600646973,
      "loss": 0.619,
      "nll_loss": 0.5621041059494019,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.2063242644071579,
      "rewards/margins": 1.7378184795379639,
      "rewards/rejected": -1.9441425800323486,
      "step": 400
    },
    {
      "epoch": 2.4143070044709387,
      "grad_norm": 12.980825424194336,
      "learning_rate": 3.72677996249965e-06,
      "log_odds_chosen": 4.317046165466309,
      "log_odds_ratio": -0.07360969483852386,
      "logits/chosen": 422.6468200683594,
      "logits/rejected": 380.22998046875,
      "logps/chosen": -0.43779826164245605,
      "logps/rejected": -3.444517135620117,
      "loss": 0.6456,
      "nll_loss": 0.6048913598060608,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.21889913082122803,
      "rewards/margins": 1.5033594369888306,
      "rewards/rejected": -1.7222585678100586,
      "step": 405
    },
    {
      "epoch": 2.444113263785395,
      "grad_norm": 11.386286735534668,
      "learning_rate": 3.7039859874359616e-06,
      "log_odds_chosen": 3.6852588653564453,
      "log_odds_ratio": -0.06955203413963318,
      "logits/chosen": 365.75726318359375,
      "logits/rejected": 373.08935546875,
      "logps/chosen": -0.4394207000732422,
      "logps/rejected": -2.8892974853515625,
      "loss": 0.6307,
      "nll_loss": 0.6479864120483398,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.2197103500366211,
      "rewards/margins": 1.2249382734298706,
      "rewards/rejected": -1.4446487426757812,
      "step": 410
    },
    {
      "epoch": 2.473919523099851,
      "grad_norm": 18.22783088684082,
      "learning_rate": 3.68160520255362e-06,
      "log_odds_chosen": 4.396167755126953,
      "log_odds_ratio": -0.04461110755801201,
      "logits/chosen": 371.11077880859375,
      "logits/rejected": 393.236328125,
      "logps/chosen": -0.4007198214530945,
      "logps/rejected": -3.539832353591919,
      "loss": 0.6134,
      "nll_loss": 0.5493707656860352,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20035991072654724,
      "rewards/margins": 1.5695561170578003,
      "rewards/rejected": -1.7699161767959595,
      "step": 415
    },
    {
      "epoch": 2.503725782414307,
      "grad_norm": 12.961064338684082,
      "learning_rate": 3.6596252735569993e-06,
      "log_odds_chosen": 3.7051539421081543,
      "log_odds_ratio": -0.06346116960048676,
      "logits/chosen": 368.0414123535156,
      "logits/rejected": 338.7422180175781,
      "logps/chosen": -0.38283970952033997,
      "logps/rejected": -2.9152991771698,
      "loss": 0.6314,
      "nll_loss": 0.6913596987724304,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.19141985476016998,
      "rewards/margins": 1.2662298679351807,
      "rewards/rejected": -1.4576495885849,
      "step": 420
    },
    {
      "epoch": 2.533532041728763,
      "grad_norm": 14.195425033569336,
      "learning_rate": 3.6380343755449946e-06,
      "log_odds_chosen": 4.560431480407715,
      "log_odds_ratio": -0.03129279986023903,
      "logits/chosen": 362.96746826171875,
      "logits/rejected": 349.57958984375,
      "logps/chosen": -0.4239065647125244,
      "logps/rejected": -3.7524616718292236,
      "loss": 0.6005,
      "nll_loss": 0.6241891980171204,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2119532823562622,
      "rewards/margins": 1.66427743434906,
      "rewards/rejected": -1.8762308359146118,
      "step": 425
    },
    {
      "epoch": 2.563338301043219,
      "grad_norm": 13.435834884643555,
      "learning_rate": 3.6168211662780912e-06,
      "log_odds_chosen": 4.4234299659729,
      "log_odds_ratio": -0.03766634315252304,
      "logits/chosen": 361.0891418457031,
      "logits/rejected": 362.5020446777344,
      "logps/chosen": -0.3212960362434387,
      "logps/rejected": -3.3532252311706543,
      "loss": 0.6194,
      "nll_loss": 0.5339005589485168,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16064801812171936,
      "rewards/margins": 1.5159647464752197,
      "rewards/rejected": -1.6766126155853271,
      "step": 430
    },
    {
      "epoch": 2.593144560357675,
      "grad_norm": 13.10000228881836,
      "learning_rate": 3.5959747611403815e-06,
      "log_odds_chosen": 3.667933702468872,
      "log_odds_ratio": -0.049441076815128326,
      "logits/chosen": 370.7209777832031,
      "logits/rejected": 329.81048583984375,
      "logps/chosen": -0.3787091374397278,
      "logps/rejected": -2.830317735671997,
      "loss": 0.6284,
      "nll_loss": 0.5626553297042847,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1893545687198639,
      "rewards/margins": 1.2258042097091675,
      "rewards/rejected": -1.4151588678359985,
      "step": 435
    },
    {
      "epoch": 2.6229508196721314,
      "grad_norm": 16.336698532104492,
      "learning_rate": 3.575484709670971e-06,
      "log_odds_chosen": 4.056857585906982,
      "log_odds_ratio": -0.035340312868356705,
      "logits/chosen": 350.6004333496094,
      "logits/rejected": 356.38323974609375,
      "logps/chosen": -0.3078530430793762,
      "logps/rejected": -2.8841094970703125,
      "loss": 0.6605,
      "nll_loss": 0.6005590558052063,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1539265215396881,
      "rewards/margins": 1.288128137588501,
      "rewards/rejected": -1.4420547485351562,
      "step": 440
    },
    {
      "epoch": 2.6527570789865873,
      "grad_norm": 16.911279678344727,
      "learning_rate": 3.5553409735498294e-06,
      "log_odds_chosen": 4.540930271148682,
      "log_odds_ratio": -0.021845344454050064,
      "logits/chosen": 333.2252197265625,
      "logits/rejected": 360.4376525878906,
      "logps/chosen": -0.3848707973957062,
      "logps/rejected": -3.6115005016326904,
      "loss": 0.6305,
      "nll_loss": 0.616925060749054,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1924353986978531,
      "rewards/margins": 1.6133148670196533,
      "rewards/rejected": -1.8057502508163452,
      "step": 445
    },
    {
      "epoch": 2.682563338301043,
      "grad_norm": 12.66332721710205,
      "learning_rate": 3.535533905932738e-06,
      "log_odds_chosen": 3.8172874450683594,
      "log_odds_ratio": -0.04109153896570206,
      "logits/chosen": 363.5258483886719,
      "logits/rejected": 355.9951171875,
      "logps/chosen": -0.442394495010376,
      "logps/rejected": -3.0551743507385254,
      "loss": 0.6594,
      "nll_loss": 0.6155456304550171,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.221197247505188,
      "rewards/margins": 1.3063898086547852,
      "rewards/rejected": -1.5275871753692627,
      "step": 450
    },
    {
      "epoch": 2.712369597615499,
      "grad_norm": 11.626214981079102,
      "learning_rate": 3.516054232038716e-06,
      "log_odds_chosen": 3.7958292961120605,
      "log_odds_ratio": -0.05676658824086189,
      "logits/chosen": 394.8970947265625,
      "logits/rejected": 370.40692138671875,
      "logps/chosen": -0.49928539991378784,
      "logps/rejected": -3.0073413848876953,
      "loss": 0.6971,
      "nll_loss": 0.6259102821350098,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.24964269995689392,
      "rewards/margins": 1.2540279626846313,
      "rewards/rejected": -1.5036706924438477,
      "step": 455
    },
    {
      "epoch": 2.742175856929955,
      "grad_norm": 12.148904800415039,
      "learning_rate": 3.496893030901177e-06,
      "log_odds_chosen": 4.385091781616211,
      "log_odds_ratio": -0.0362153984606266,
      "logits/chosen": 390.84942626953125,
      "logits/rejected": 403.558837890625,
      "logps/chosen": -0.35050949454307556,
      "logps/rejected": -3.5067138671875,
      "loss": 0.64,
      "nll_loss": 0.6797647476196289,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17525474727153778,
      "rewards/margins": 1.5781023502349854,
      "rewards/rejected": -1.75335693359375,
      "step": 460
    },
    {
      "epoch": 2.7719821162444114,
      "grad_norm": 16.0198974609375,
      "learning_rate": 3.478041718201262e-06,
      "log_odds_chosen": 4.281471252441406,
      "log_odds_ratio": -0.04437633976340294,
      "logits/chosen": 379.65142822265625,
      "logits/rejected": 373.5622863769531,
      "logps/chosen": -0.3376389443874359,
      "logps/rejected": -3.2862961292266846,
      "loss": 0.6475,
      "nll_loss": 0.6045721769332886,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16881947219371796,
      "rewards/margins": 1.4743287563323975,
      "rewards/rejected": -1.6431480646133423,
      "step": 465
    },
    {
      "epoch": 2.8017883755588673,
      "grad_norm": 20.6240291595459,
      "learning_rate": 3.459492030108319e-06,
      "log_odds_chosen": 3.6953399181365967,
      "log_odds_ratio": -0.07698026299476624,
      "logits/chosen": 384.45758056640625,
      "logits/rejected": 344.5814208984375,
      "logps/chosen": -0.43422994017601013,
      "logps/rejected": -2.9317374229431152,
      "loss": 0.6539,
      "nll_loss": 0.6051168441772461,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.21711497008800507,
      "rewards/margins": 1.2487537860870361,
      "rewards/rejected": -1.4658687114715576,
      "step": 470
    },
    {
      "epoch": 2.8315946348733236,
      "grad_norm": 16.241731643676758,
      "learning_rate": 3.441236008058426e-06,
      "log_odds_chosen": 4.172859191894531,
      "log_odds_ratio": -0.03734353557229042,
      "logits/chosen": 359.6179504394531,
      "logits/rejected": 372.67559814453125,
      "logps/chosen": -0.3877125382423401,
      "logps/rejected": -3.2986769676208496,
      "loss": 0.6017,
      "nll_loss": 0.5468435287475586,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19385626912117004,
      "rewards/margins": 1.4554822444915771,
      "rewards/rejected": -1.6493384838104248,
      "step": 475
    },
    {
      "epoch": 2.8614008941877795,
      "grad_norm": 13.836338996887207,
      "learning_rate": 3.4232659844072883e-06,
      "log_odds_chosen": 5.078559875488281,
      "log_odds_ratio": -0.040774669498205185,
      "logits/chosen": 369.23388671875,
      "logits/rejected": 398.89654541015625,
      "logps/chosen": -0.4116280674934387,
      "logps/rejected": -4.2671427726745605,
      "loss": 0.5823,
      "nll_loss": 0.5545191764831543,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20581403374671936,
      "rewards/margins": 1.9277572631835938,
      "rewards/rejected": -2.1335713863372803,
      "step": 480
    },
    {
      "epoch": 2.8912071535022354,
      "grad_norm": 13.629563331604004,
      "learning_rate": 3.4055745688987482e-06,
      "log_odds_chosen": 3.756863832473755,
      "log_odds_ratio": -0.05358726903796196,
      "logits/chosen": 360.4635925292969,
      "logits/rejected": 379.5230407714844,
      "logps/chosen": -0.4683550000190735,
      "logps/rejected": -3.0478286743164062,
      "loss": 0.6356,
      "nll_loss": 0.5991590023040771,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.23417750000953674,
      "rewards/margins": 1.2897369861602783,
      "rewards/rejected": -1.5239143371582031,
      "step": 485
    },
    {
      "epoch": 2.9210134128166914,
      "grad_norm": 11.519225120544434,
      "learning_rate": 3.3881546358946924e-06,
      "log_odds_chosen": 4.434014320373535,
      "log_odds_ratio": -0.048062875866889954,
      "logits/chosen": 385.712158203125,
      "logits/rejected": 336.68524169921875,
      "logps/chosen": -0.45937252044677734,
      "logps/rejected": -3.5976157188415527,
      "loss": 0.6968,
      "nll_loss": 0.5958720445632935,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.22968626022338867,
      "rewards/margins": 1.5691215991973877,
      "rewards/rejected": -1.7988078594207764,
      "step": 490
    },
    {
      "epoch": 2.9508196721311473,
      "grad_norm": 16.139705657958984,
      "learning_rate": 3.37099931231621e-06,
      "log_odds_chosen": 4.680273532867432,
      "log_odds_ratio": -0.062451474368572235,
      "logits/chosen": 375.66119384765625,
      "logits/rejected": 394.88690185546875,
      "logps/chosen": -0.4953223764896393,
      "logps/rejected": -3.950305938720703,
      "loss": 0.6257,
      "nll_loss": 0.628521203994751,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.24766118824481964,
      "rewards/margins": 1.7274919748306274,
      "rewards/rejected": -1.9751529693603516,
      "step": 495
    },
    {
      "epoch": 2.9806259314456036,
      "grad_norm": 15.226314544677734,
      "learning_rate": 3.3541019662496845e-06,
      "log_odds_chosen": 4.290003776550293,
      "log_odds_ratio": -0.04303581267595291,
      "logits/chosen": 379.34765625,
      "logits/rejected": 365.36773681640625,
      "logps/chosen": -0.3512369692325592,
      "logps/rejected": -3.2438788414001465,
      "loss": 0.6379,
      "nll_loss": 0.5231742858886719,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1756184846162796,
      "rewards/margins": 1.44632089138031,
      "rewards/rejected": -1.6219394207000732,
      "step": 500
    },
    {
      "epoch": 2.9865871833084947,
      "eval_log_odds_chosen": 0.33574143052101135,
      "eval_log_odds_ratio": -0.6769095063209534,
      "eval_logits/chosen": 292.74676513671875,
      "eval_logits/rejected": 259.2041320800781,
      "eval_logps/chosen": -1.2910676002502441,
      "eval_logps/rejected": -1.5239651203155518,
      "eval_loss": 2.003471851348877,
      "eval_nll_loss": 1.6439613103866577,
      "eval_rewards/accuracies": 0.6115108132362366,
      "eval_rewards/chosen": -0.6455338001251221,
      "eval_rewards/margins": 0.1164487823843956,
      "eval_rewards/rejected": -0.7619825601577759,
      "eval_runtime": 378.0521,
      "eval_samples_per_second": 1.463,
      "eval_steps_per_second": 0.368,
      "step": 501
    },
    {
      "epoch": 2.9865871833084947,
      "step": 501,
      "total_flos": 0.0,
      "train_loss": 1.6206341998543805,
      "train_runtime": 33576.7022,
      "train_samples_per_second": 0.479,
      "train_steps_per_second": 0.015
    }
  ],
  "logging_steps": 5,
  "max_steps": 501,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 3,
  "save_steps": 500,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": false,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 2,
  "trial_name": null,
  "trial_params": null
}