{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 2.992914501653283,
  "eval_steps": 50,
  "global_step": 1056,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "dpo_loss": 0.6931471824645996,
      "epoch": 0.002834199338686821,
      "grad_norm": 17675.58033930074,
      "learning_rate": 4.716981132075472e-09,
      "logits": -1.2867579460144043,
      "logps": -84.34933471679688,
      "loss": 169.5214,
      "objective": 153.4677734375,
      "ranking_idealized": 0.6458333134651184,
      "ranking_idealized_expo": 0.5833333134651184,
      "ranking_simple": 0.5833333134651184,
      "regularize": 0.3618059456348419,
      "step": 1,
      "wo_beta": 14.83154582977295
    },
    {
      "dpo_loss": 0.6930959224700928,
      "epoch": 0.014170996693434105,
      "grad_norm": 16812.231100839916,
      "learning_rate": 2.3584905660377358e-08,
      "logits": -1.4290882349014282,
      "logps": -83.8636474609375,
      "loss": 181.7078,
      "objective": 168.5659942626953,
      "ranking_idealized": 0.5625,
      "ranking_idealized_expo": 0.4895833432674408,
      "ranking_simple": 0.4895833432674408,
      "regularize": 0.40367603302001953,
      "step": 5,
      "wo_beta": 16.679981231689453
    },
    {
      "dpo_loss": 0.6930798292160034,
      "epoch": 0.02834199338686821,
      "grad_norm": 18597.19901899509,
      "learning_rate": 4.7169811320754715e-08,
      "logits": -1.4008352756500244,
      "logps": -84.84938049316406,
      "loss": 177.1073,
      "objective": 170.35797119140625,
      "ranking_idealized": 0.6708333492279053,
      "ranking_idealized_expo": 0.5833333134651184,
      "ranking_simple": 0.5791666507720947,
      "regularize": 0.4039422273635864,
      "step": 10,
      "wo_beta": 15.222180366516113
    },
    {
      "dpo_loss": 0.6921038031578064,
      "epoch": 0.042512990080302314,
      "grad_norm": 17855.275799007577,
      "learning_rate": 7.075471698113207e-08,
      "logits": -1.538023829460144,
      "logps": -84.5517578125,
      "loss": 178.9814,
      "objective": 187.4513702392578,
      "ranking_idealized": 0.6499999761581421,
      "ranking_idealized_expo": 0.5708333253860474,
      "ranking_simple": 0.5666666626930237,
      "regularize": 0.4422657787799835,
      "step": 15,
      "wo_beta": 15.718367576599121
    },
    {
      "dpo_loss": 0.6917796730995178,
      "epoch": 0.05668398677373642,
      "grad_norm": 17564.60110673315,
      "learning_rate": 9.433962264150943e-08,
      "logits": -1.3617039918899536,
      "logps": -83.66792297363281,
      "loss": 185.8199,
      "objective": 204.0640411376953,
      "ranking_idealized": 0.5833333134651184,
      "ranking_idealized_expo": 0.4833333194255829,
      "ranking_simple": 0.48750001192092896,
      "regularize": 0.44205835461616516,
      "step": 20,
      "wo_beta": 16.52640151977539
    },
    {
      "dpo_loss": 0.6927011013031006,
      "epoch": 0.07085498346717052,
      "grad_norm": 16991.775752313566,
      "learning_rate": 1.1792452830188679e-07,
      "logits": -1.3692513704299927,
      "logps": -83.765869140625,
      "loss": 182.1115,
      "objective": 173.06422424316406,
      "ranking_idealized": 0.6458333134651184,
      "ranking_idealized_expo": 0.5583333373069763,
      "ranking_simple": 0.5666666626930237,
      "regularize": 0.40760377049446106,
      "step": 25,
      "wo_beta": 15.608158111572266
    },
    {
      "dpo_loss": 0.6904457211494446,
      "epoch": 0.08502598016060463,
      "grad_norm": 14856.204222337537,
      "learning_rate": 1.4150943396226414e-07,
      "logits": -1.4308700561523438,
      "logps": -83.563232421875,
      "loss": 181.7541,
      "objective": 176.98880004882812,
      "ranking_idealized": 0.5541666746139526,
      "ranking_idealized_expo": 0.4833333194255829,
      "ranking_simple": 0.48750001192092896,
      "regularize": 0.43005380034446716,
      "step": 30,
      "wo_beta": 17.01230812072754
    },
    {
      "dpo_loss": 0.6906622648239136,
      "epoch": 0.09919697685403873,
      "grad_norm": 16081.157641472842,
      "learning_rate": 1.650943396226415e-07,
      "logits": -1.4087789058685303,
      "logps": -82.7640151977539,
      "loss": 184.1344,
      "objective": 172.8912811279297,
      "ranking_idealized": 0.6291666626930237,
      "ranking_idealized_expo": 0.5375000238418579,
      "ranking_simple": 0.5416666865348816,
      "regularize": 0.37934428453445435,
      "step": 35,
      "wo_beta": 16.152484893798828
    },
    {
      "dpo_loss": 0.6896480917930603,
      "epoch": 0.11336797354747284,
      "grad_norm": 17056.964984105944,
      "learning_rate": 1.8867924528301886e-07,
      "logits": -1.4006307125091553,
      "logps": -83.35142517089844,
      "loss": 188.1977,
      "objective": 182.53704833984375,
      "ranking_idealized": 0.5708333253860474,
      "ranking_idealized_expo": 0.5166666507720947,
      "ranking_simple": 0.5166666507720947,
      "regularize": 0.40673011541366577,
      "step": 40,
      "wo_beta": 14.254140853881836
    },
    {
      "dpo_loss": 0.6860460638999939,
      "epoch": 0.12753897024090693,
      "grad_norm": 15541.887109298903,
      "learning_rate": 2.1226415094339622e-07,
      "logits": -1.4170690774917603,
      "logps": -83.82962799072266,
      "loss": 172.0023,
      "objective": 184.33473205566406,
      "ranking_idealized": 0.5958333611488342,
      "ranking_idealized_expo": 0.5083333253860474,
      "ranking_simple": 0.512499988079071,
      "regularize": 0.41498348116874695,
      "step": 45,
      "wo_beta": 14.2799711227417
    },
    {
      "dpo_loss": 0.6840464472770691,
      "epoch": 0.14170996693434104,
      "grad_norm": 16674.096437377164,
      "learning_rate": 2.3584905660377358e-07,
      "logits": -1.4327392578125,
      "logps": -84.8567123413086,
      "loss": 182.5182,
      "objective": 187.45941162109375,
      "ranking_idealized": 0.5916666388511658,
      "ranking_idealized_expo": 0.5,
      "ranking_simple": 0.5041666626930237,
      "regularize": 0.43751442432403564,
      "step": 50,
      "wo_beta": 15.616755485534668
    },
    {
      "epoch": 0.14170996693434104,
      "eval_dpo_loss": 0.689544677734375,
      "eval_logits": -1.4199916124343872,
      "eval_logps": -90.85165405273438,
      "eval_loss": 182.50025939941406,
      "eval_objective": 180.4892578125,
      "eval_ranking_idealized": 0.6024844646453857,
      "eval_ranking_idealized_expo": 0.5232919454574585,
      "eval_ranking_simple": 0.5248447060585022,
      "eval_regularize": 0.4092595875263214,
      "eval_runtime": 369.2017,
      "eval_samples_per_second": 15.682,
      "eval_steps_per_second": 1.308,
      "eval_wo_beta": 16.310007095336914,
      "step": 50
    },
    {
      "dpo_loss": 0.6828119158744812,
      "epoch": 0.15588096362777515,
      "grad_norm": 17241.419986006367,
      "learning_rate": 2.5943396226415094e-07,
      "logits": -1.3938590288162231,
      "logps": -84.56362915039062,
      "loss": 171.7727,
      "objective": 174.47201538085938,
      "ranking_idealized": 0.5583333373069763,
      "ranking_idealized_expo": 0.49166667461395264,
      "ranking_simple": 0.4833333194255829,
      "regularize": 0.3751158118247986,
      "step": 55,
      "wo_beta": 15.145721435546875
    },
    {
      "dpo_loss": 0.6828226447105408,
      "epoch": 0.17005196032120926,
      "grad_norm": 16193.112384702756,
      "learning_rate": 2.830188679245283e-07,
      "logits": -1.325377345085144,
      "logps": -85.11466217041016,
      "loss": 175.0018,
      "objective": 174.82723999023438,
      "ranking_idealized": 0.6000000238418579,
      "ranking_idealized_expo": 0.5166666507720947,
      "ranking_simple": 0.5208333134651184,
      "regularize": 0.41120079159736633,
      "step": 60,
      "wo_beta": 14.497312545776367
    },
    {
      "dpo_loss": 0.673675537109375,
      "epoch": 0.18422295701464336,
      "grad_norm": 16474.473591772632,
      "learning_rate": 3.066037735849056e-07,
      "logits": -1.4237332344055176,
      "logps": -82.10260772705078,
      "loss": 174.1747,
      "objective": 163.51272583007812,
      "ranking_idealized": 0.637499988079071,
      "ranking_idealized_expo": 0.5249999761581421,
      "ranking_simple": 0.5333333611488342,
      "regularize": 0.3746616542339325,
      "step": 65,
      "wo_beta": 15.076056480407715
    },
    {
      "dpo_loss": 0.6760156750679016,
      "epoch": 0.19839395370807747,
      "grad_norm": 15655.300421670066,
      "learning_rate": 3.30188679245283e-07,
      "logits": -1.4630695581436157,
      "logps": -84.45524597167969,
      "loss": 175.2515,
      "objective": 174.8110809326172,
      "ranking_idealized": 0.5916666388511658,
      "ranking_idealized_expo": 0.5333333611488342,
      "ranking_simple": 0.5333333611488342,
      "regularize": 0.40165895223617554,
      "step": 70,
      "wo_beta": 15.362497329711914
    },
    {
      "dpo_loss": 0.6764008402824402,
      "epoch": 0.21256495040151158,
      "grad_norm": 18669.073461411434,
      "learning_rate": 3.5377358490566033e-07,
      "logits": -1.3853403329849243,
      "logps": -84.13139343261719,
      "loss": 176.1206,
      "objective": 166.9723663330078,
      "ranking_idealized": 0.6083333492279053,
      "ranking_idealized_expo": 0.5166666507720947,
      "ranking_simple": 0.5208333134651184,
      "regularize": 0.38751351833343506,
      "step": 75,
      "wo_beta": 15.094878196716309
    },
    {
      "dpo_loss": 0.6740989089012146,
      "epoch": 0.22673594709494568,
      "grad_norm": 19007.32032313182,
      "learning_rate": 3.773584905660377e-07,
      "logits": -1.481835126876831,
      "logps": -83.50402069091797,
      "loss": 170.5001,
      "objective": 178.72813415527344,
      "ranking_idealized": 0.6291666626930237,
      "ranking_idealized_expo": 0.5291666388511658,
      "ranking_simple": 0.5458333492279053,
      "regularize": 0.42763322591781616,
      "step": 80,
      "wo_beta": 16.335308074951172
    },
    {
      "dpo_loss": 0.6696261167526245,
      "epoch": 0.2409069437883798,
      "grad_norm": 15826.33154406391,
      "learning_rate": 4.009433962264151e-07,
      "logits": -1.4629038572311401,
      "logps": -83.72789001464844,
      "loss": 173.4106,
      "objective": 184.8107452392578,
      "ranking_idealized": 0.625,
      "ranking_idealized_expo": 0.550000011920929,
      "ranking_simple": 0.550000011920929,
      "regularize": 0.41807958483695984,
      "step": 85,
      "wo_beta": 15.131178855895996
    },
    {
      "dpo_loss": 0.6652686595916748,
      "epoch": 0.25507794048181387,
      "grad_norm": 16251.074571263394,
      "learning_rate": 4.2452830188679244e-07,
      "logits": -1.562590479850769,
      "logps": -84.50687408447266,
      "loss": 170.0469,
      "objective": 178.0680694580078,
      "ranking_idealized": 0.612500011920929,
      "ranking_idealized_expo": 0.5291666388511658,
      "ranking_simple": 0.5541666746139526,
      "regularize": 0.40833932161331177,
      "step": 90,
      "wo_beta": 15.540740966796875
    },
    {
      "dpo_loss": 0.6606998443603516,
      "epoch": 0.269248937175248,
      "grad_norm": 17104.36417405731,
      "learning_rate": 4.481132075471698e-07,
      "logits": -1.4817092418670654,
      "logps": -84.26000213623047,
      "loss": 172.5874,
      "objective": 179.69664001464844,
      "ranking_idealized": 0.5583333373069763,
      "ranking_idealized_expo": 0.4791666567325592,
      "ranking_simple": 0.512499988079071,
      "regularize": 0.412168025970459,
      "step": 95,
      "wo_beta": 16.397871017456055
    },
    {
      "dpo_loss": 0.6613048315048218,
      "epoch": 0.2834199338686821,
      "grad_norm": 16520.717626446203,
      "learning_rate": 4.7169811320754717e-07,
      "logits": -1.3626132011413574,
      "logps": -83.24072265625,
      "loss": 159.305,
      "objective": 164.95997619628906,
      "ranking_idealized": 0.5958333611488342,
      "ranking_idealized_expo": 0.4833333194255829,
      "ranking_simple": 0.5,
      "regularize": 0.37101998925209045,
      "step": 100,
      "wo_beta": 15.463290214538574
    },
    {
      "epoch": 0.2834199338686821,
      "eval_dpo_loss": 0.6859607100486755,
      "eval_logits": -1.4621983766555786,
      "eval_logps": -91.35309600830078,
      "eval_loss": 182.15220642089844,
      "eval_objective": 180.52191162109375,
      "eval_ranking_idealized": 0.6024844646453857,
      "eval_ranking_idealized_expo": 0.5232919454574585,
      "eval_ranking_simple": 0.5310559272766113,
      "eval_regularize": 0.41025590896606445,
      "eval_runtime": 368.4707,
      "eval_samples_per_second": 15.714,
      "eval_steps_per_second": 1.311,
      "eval_wo_beta": 16.38188362121582,
      "step": 100
    },
    {
      "dpo_loss": 0.6732772588729858,
      "epoch": 0.2975909305621162,
      "grad_norm": 20087.528218167263,
      "learning_rate": 4.952830188679246e-07,
      "logits": -1.5680618286132812,
      "logps": -86.15119934082031,
      "loss": 163.5086,
      "objective": 165.89260864257812,
      "ranking_idealized": 0.625,
      "ranking_idealized_expo": 0.5291666388511658,
      "ranking_simple": 0.5375000238418579,
      "regularize": 0.38661062717437744,
      "step": 105,
      "wo_beta": 14.799639701843262
    },
    {
      "dpo_loss": 0.6610164046287537,
      "epoch": 0.3117619272555503,
      "grad_norm": 16022.782640878671,
      "learning_rate": 4.999781286194085e-07,
      "logits": -1.470965027809143,
      "logps": -85.03189849853516,
      "loss": 162.0049,
      "objective": 163.46115112304688,
      "ranking_idealized": 0.5874999761581421,
      "ranking_idealized_expo": 0.5041666626930237,
      "ranking_simple": 0.5166666507720947,
      "regularize": 0.38891494274139404,
      "step": 110,
      "wo_beta": 15.151239395141602
    },
    {
      "dpo_loss": 0.6605216860771179,
      "epoch": 0.32593292394898443,
      "grad_norm": 17332.290004559494,
      "learning_rate": 4.998892826944417e-07,
      "logits": -1.4446924924850464,
      "logps": -83.6183090209961,
      "loss": 151.5536,
      "objective": 154.1279296875,
      "ranking_idealized": 0.6208333373069763,
      "ranking_idealized_expo": 0.5249999761581421,
      "ranking_simple": 0.5416666865348816,
      "regularize": 0.37217938899993896,
      "step": 115,
      "wo_beta": 16.30828094482422
    },
    {
      "dpo_loss": 0.6518290042877197,
      "epoch": 0.3401039206424185,
      "grad_norm": 15444.050796566442,
      "learning_rate": 4.997321195347154e-07,
      "logits": -1.4417078495025635,
      "logps": -83.37753295898438,
      "loss": 159.0179,
      "objective": 166.70437622070312,
      "ranking_idealized": 0.5916666388511658,
      "ranking_idealized_expo": 0.512499988079071,
      "ranking_simple": 0.5416666865348816,
      "regularize": 0.36605674028396606,
      "step": 120,
      "wo_beta": 15.73963451385498
    },
    {
      "dpo_loss": 0.655017614364624,
      "epoch": 0.35427491733585265,
      "grad_norm": 16163.2780124362,
      "learning_rate": 4.995066821070679e-07,
      "logits": -1.479369044303894,
      "logps": -86.81266021728516,
      "loss": 152.8657,
      "objective": 143.82176208496094,
      "ranking_idealized": 0.5583333373069763,
      "ranking_idealized_expo": 0.48750001192092896,
      "ranking_simple": 0.5083333253860474,
      "regularize": 0.3394821584224701,
      "step": 125,
      "wo_beta": 16.155017852783203
    },
    {
      "dpo_loss": 0.6490565538406372,
      "epoch": 0.3684459140292867,
      "grad_norm": 21066.462923637613,
      "learning_rate": 4.99213032043841e-07,
      "logits": -1.4559980630874634,
      "logps": -84.79336547851562,
      "loss": 159.2873,
      "objective": 164.24014282226562,
      "ranking_idealized": 0.6458333134651184,
      "ranking_idealized_expo": 0.5916666388511658,
      "ranking_simple": 0.5791666507720947,
      "regularize": 0.36314311623573303,
      "step": 130,
      "wo_beta": 16.631132125854492
    },
    {
      "dpo_loss": 0.6596164703369141,
      "epoch": 0.3826169107227208,
      "grad_norm": 25406.331880366477,
      "learning_rate": 4.988512496260301e-07,
      "logits": -1.4966251850128174,
      "logps": -85.71400451660156,
      "loss": 162.2932,
      "objective": 171.63385009765625,
      "ranking_idealized": 0.574999988079071,
      "ranking_idealized_expo": 0.4958333373069763,
      "ranking_simple": 0.5,
      "regularize": 0.3589702546596527,
      "step": 135,
      "wo_beta": 14.497623443603516
    },
    {
      "dpo_loss": 0.6495281457901001,
      "epoch": 0.39678790741615494,
      "grad_norm": 18698.7190702899,
      "learning_rate": 4.984214337613357e-07,
      "logits": -1.4679287672042847,
      "logps": -85.32872772216797,
      "loss": 145.3416,
      "objective": 162.12405395507812,
      "ranking_idealized": 0.612500011920929,
      "ranking_idealized_expo": 0.5416666865348816,
      "ranking_simple": 0.5583333373069763,
      "regularize": 0.37301716208457947,
      "step": 140,
      "wo_beta": 14.43948745727539
    },
    {
      "dpo_loss": 0.6436702013015747,
      "epoch": 0.410958904109589,
      "grad_norm": 17254.43790008681,
      "learning_rate": 4.979237019571234e-07,
      "logits": -1.4821332693099976,
      "logps": -85.37076568603516,
      "loss": 150.1815,
      "objective": 147.49313354492188,
      "ranking_idealized": 0.6666666865348816,
      "ranking_idealized_expo": 0.5583333373069763,
      "ranking_simple": 0.5958333611488342,
      "regularize": 0.35633403062820435,
      "step": 145,
      "wo_beta": 14.047316551208496
    },
    {
      "dpo_loss": 0.6469600796699524,
      "epoch": 0.42512990080302315,
      "grad_norm": 15151.951735697437,
      "learning_rate": 4.973581902882989e-07,
      "logits": -1.507290005683899,
      "logps": -84.8727035522461,
      "loss": 150.2379,
      "objective": 155.10818481445312,
      "ranking_idealized": 0.5958333611488342,
      "ranking_idealized_expo": 0.5541666746139526,
      "ranking_simple": 0.5625,
      "regularize": 0.35707762837409973,
      "step": 150,
      "wo_beta": 17.33567237854004
    },
    {
      "epoch": 0.42512990080302315,
      "eval_dpo_loss": 0.6805834770202637,
      "eval_logits": -1.4576332569122314,
      "eval_logps": -90.24694061279297,
      "eval_loss": 180.0574951171875,
      "eval_objective": 177.15777587890625,
      "eval_ranking_idealized": 0.6024844646453857,
      "eval_ranking_idealized_expo": 0.5232919454574585,
      "eval_ranking_simple": 0.5331262946128845,
      "eval_regularize": 0.40097880363464355,
      "eval_runtime": 369.0413,
      "eval_samples_per_second": 15.689,
      "eval_steps_per_second": 1.309,
      "eval_wo_beta": 16.610729217529297,
      "step": 150
    },
    {
      "dpo_loss": 0.645554780960083,
      "epoch": 0.43930089749645723,
      "grad_norm": 16201.615637764453,
      "learning_rate": 4.967250533601059e-07,
      "logits": -1.5539363622665405,
      "logps": -83.4765396118164,
      "loss": 146.8879,
      "objective": 153.79454040527344,
      "ranking_idealized": 0.5666666626930237,
      "ranking_idealized_expo": 0.48750001192092896,
      "ranking_simple": 0.49166667461395264,
      "regularize": 0.3449983596801758,
      "step": 155,
      "wo_beta": 15.733673095703125
    },
    {
      "dpo_loss": 0.6385053396224976,
      "epoch": 0.45347189418989137,
      "grad_norm": 15493.137210302475,
      "learning_rate": 4.960244642658585e-07,
      "logits": -1.4331082105636597,
      "logps": -84.22053527832031,
      "loss": 152.5499,
      "objective": 151.0172882080078,
      "ranking_idealized": 0.5916666388511658,
      "ranking_idealized_expo": 0.5333333611488342,
      "ranking_simple": 0.5625,
      "regularize": 0.34484514594078064,
      "step": 160,
      "wo_beta": 14.795166015625
    },
    {
      "dpo_loss": 0.6392009854316711,
      "epoch": 0.46764289088332545,
      "grad_norm": 17120.620589579255,
      "learning_rate": 4.952566145396196e-07,
      "logits": -1.5298134088516235,
      "logps": -85.76438903808594,
      "loss": 144.1923,
      "objective": 138.38796997070312,
      "ranking_idealized": 0.6083333492279053,
      "ranking_idealized_expo": 0.5249999761581421,
      "ranking_simple": 0.550000011920929,
      "regularize": 0.32356587052345276,
      "step": 165,
      "wo_beta": 15.664267539978027
    },
    {
      "dpo_loss": 0.6427010893821716,
      "epoch": 0.4818138875767596,
      "grad_norm": 16261.789384415662,
      "learning_rate": 4.944217141038378e-07,
      "logits": -1.5661406517028809,
      "logps": -85.18688201904297,
      "loss": 146.9829,
      "objective": 135.14251708984375,
      "ranking_idealized": 0.5583333373069763,
      "ranking_idealized_expo": 0.5041666626930237,
      "ranking_simple": 0.5166666507720947,
      "regularize": 0.32022854685783386,
      "step": 170,
      "wo_beta": 16.11832046508789
    },
    {
      "dpo_loss": 0.6393853425979614,
      "epoch": 0.49598488427019366,
      "grad_norm": 17356.348275057364,
      "learning_rate": 4.935199912119557e-07,
      "logits": -1.4016886949539185,
      "logps": -86.42796325683594,
      "loss": 138.7378,
      "objective": 130.81809997558594,
      "ranking_idealized": 0.574999988079071,
      "ranking_idealized_expo": 0.5249999761581421,
      "ranking_simple": 0.550000011920929,
      "regularize": 0.3065473735332489,
      "step": 175,
      "wo_beta": 18.01983070373535
    },
    {
      "dpo_loss": 0.6324561238288879,
      "epoch": 0.5101558809636277,
      "grad_norm": 16878.815105316982,
      "learning_rate": 4.925516923860082e-07,
      "logits": -1.387779951095581,
      "logps": -85.71736907958984,
      "loss": 143.5962,
      "objective": 155.15878295898438,
      "ranking_idealized": 0.5708333253860474,
      "ranking_idealized_expo": 0.48750001192092896,
      "ranking_simple": 0.5083333253860474,
      "regularize": 0.34209319949150085,
      "step": 180,
      "wo_beta": 15.11577033996582
    },
    {
      "dpo_loss": 0.6253044605255127,
      "epoch": 0.5243268776570619,
      "grad_norm": 18146.789309631047,
      "learning_rate": 4.91517082349226e-07,
      "logits": -1.4047019481658936,
      "logps": -84.99198913574219,
      "loss": 137.7836,
      "objective": 136.2899627685547,
      "ranking_idealized": 0.5791666507720947,
      "ranking_idealized_expo": 0.4833333194255829,
      "ranking_simple": 0.5291666388511658,
      "regularize": 0.32172784209251404,
      "step": 185,
      "wo_beta": 14.294845581054688
    },
    {
      "dpo_loss": 0.618080735206604,
      "epoch": 0.538497874350496,
      "grad_norm": 16414.68768978514,
      "learning_rate": 4.904164439536626e-07,
      "logits": -1.438673496246338,
      "logps": -84.07417297363281,
      "loss": 134.8449,
      "objective": 134.37405395507812,
      "ranking_idealized": 0.6625000238418579,
      "ranking_idealized_expo": 0.5625,
      "ranking_simple": 0.6083333492279053,
      "regularize": 0.30226340889930725,
      "step": 190,
      "wo_beta": 17.166841506958008
    },
    {
      "dpo_loss": 0.6151688694953918,
      "epoch": 0.5526688710439301,
      "grad_norm": 17479.85306665603,
      "learning_rate": 4.892500781028655e-07,
      "logits": -1.4530678987503052,
      "logps": -84.28058624267578,
      "loss": 143.4298,
      "objective": 148.5701141357422,
      "ranking_idealized": 0.625,
      "ranking_idealized_expo": 0.5375000238418579,
      "ranking_simple": 0.5666666626930237,
      "regularize": 0.32925572991371155,
      "step": 195,
      "wo_beta": 15.276497840881348
    },
    {
      "dpo_loss": 0.6147686243057251,
      "epoch": 0.5668398677373642,
      "grad_norm": 17426.518013700184,
      "learning_rate": 4.880183036696122e-07,
      "logits": -1.461132287979126,
      "logps": -84.65553283691406,
      "loss": 135.925,
      "objective": 143.74124145507812,
      "ranking_idealized": 0.5833333134651184,
      "ranking_idealized_expo": 0.512499988079071,
      "ranking_simple": 0.5291666388511658,
      "regularize": 0.3410433530807495,
      "step": 200,
      "wo_beta": 16.30559539794922
    },
    {
      "epoch": 0.5668398677373642,
      "eval_dpo_loss": 0.6795096397399902,
      "eval_logits": -1.445342779159546,
      "eval_logps": -91.12489318847656,
      "eval_loss": 179.97398376464844,
      "eval_objective": 177.04129028320312,
      "eval_ranking_idealized": 0.6024844646453857,
      "eval_ranking_idealized_expo": 0.5232919454574585,
      "eval_ranking_simple": 0.5305383205413818,
      "eval_regularize": 0.40064504742622375,
      "eval_runtime": 371.8339,
      "eval_samples_per_second": 15.571,
      "eval_steps_per_second": 1.299,
      "eval_wo_beta": 16.26874351501465,
      "step": 200
    },
    {
      "dpo_loss": 0.6258116960525513,
      "epoch": 0.5810108644307983,
      "grad_norm": 16576.414881735825,
      "learning_rate": 4.867214574087337e-07,
      "logits": -1.3335182666778564,
      "logps": -85.45887756347656,
      "loss": 131.7019,
      "objective": 124.74952697753906,
      "ranking_idealized": 0.574999988079071,
      "ranking_idealized_expo": 0.5041666626930237,
      "ranking_simple": 0.5208333134651184,
      "regularize": 0.30992764234542847,
      "step": 205,
      "wo_beta": 16.508281707763672
    },
    {
      "dpo_loss": 0.6121302247047424,
      "epoch": 0.5951818611242324,
      "grad_norm": 19803.851938298896,
      "learning_rate": 4.853598938650486e-07,
      "logits": -1.3813374042510986,
      "logps": -85.58969116210938,
      "loss": 142.8564,
      "objective": 147.2575225830078,
      "ranking_idealized": 0.6041666865348816,
      "ranking_idealized_expo": 0.5208333134651184,
      "ranking_simple": 0.5458333492279053,
      "regularize": 0.34093615412712097,
      "step": 210,
      "wo_beta": 15.581511497497559
    },
    {
      "dpo_loss": 0.6104889512062073,
      "epoch": 0.6093528578176665,
      "grad_norm": 16635.20929245541,
      "learning_rate": 4.839339852764349e-07,
      "logits": -1.4747200012207031,
      "logps": -84.081298828125,
      "loss": 132.2213,
      "objective": 143.8260040283203,
      "ranking_idealized": 0.5791666507720947,
      "ranking_idealized_expo": 0.4749999940395355,
      "ranking_simple": 0.5249999761581421,
      "regularize": 0.3345707654953003,
      "step": 215,
      "wo_beta": 16.507150650024414
    },
    {
      "dpo_loss": 0.6056095957756042,
      "epoch": 0.6235238545111006,
      "grad_norm": 17892.305863583017,
      "learning_rate": 4.824441214720628e-07,
      "logits": -1.4685624837875366,
      "logps": -84.3477783203125,
      "loss": 136.6061,
      "objective": 134.5189971923828,
      "ranking_idealized": 0.574999988079071,
      "ranking_idealized_expo": 0.4791666567325592,
      "ranking_simple": 0.512499988079071,
      "regularize": 0.3130161464214325,
      "step": 220,
      "wo_beta": 16.86256217956543
    },
    {
      "dpo_loss": 0.6179187297821045,
      "epoch": 0.6376948512045347,
      "grad_norm": 15080.021415971216,
      "learning_rate": 4.808907097658205e-07,
      "logits": -1.5259219408035278,
      "logps": -85.83920288085938,
      "loss": 132.0525,
      "objective": 135.1264190673828,
      "ranking_idealized": 0.6000000238418579,
      "ranking_idealized_expo": 0.5041666626930237,
      "ranking_simple": 0.512499988079071,
      "regularize": 0.310780793428421,
      "step": 225,
      "wo_beta": 15.415994644165039
    },
    {
      "dpo_loss": 0.613567590713501,
      "epoch": 0.6518658478979689,
      "grad_norm": 15927.895183583649,
      "learning_rate": 4.792741748449574e-07,
      "logits": -1.4311482906341553,
      "logps": -85.6706314086914,
      "loss": 121.739,
      "objective": 121.26396179199219,
      "ranking_idealized": 0.637499988079071,
      "ranking_idealized_expo": 0.5666666626930237,
      "ranking_simple": 0.5916666388511658,
      "regularize": 0.29853278398513794,
      "step": 230,
      "wo_beta": 16.95390510559082
    },
    {
      "dpo_loss": 0.6123810410499573,
      "epoch": 0.6660368445914029,
      "grad_norm": 18377.873397463725,
      "learning_rate": 4.775949586539803e-07,
      "logits": -1.3708454370498657,
      "logps": -86.44990539550781,
      "loss": 121.6467,
      "objective": 109.16976165771484,
      "ranking_idealized": 0.5874999761581421,
      "ranking_idealized_expo": 0.5416666865348816,
      "ranking_simple": 0.5583333373069763,
      "regularize": 0.2606847584247589,
      "step": 235,
      "wo_beta": 16.56985092163086
    },
    {
      "dpo_loss": 0.6006901264190674,
      "epoch": 0.680207841284837,
      "grad_norm": 20176.56902743112,
      "learning_rate": 4.758535202738287e-07,
      "logits": -1.5398815870285034,
      "logps": -86.04439544677734,
      "loss": 135.6713,
      "objective": 137.5254364013672,
      "ranking_idealized": 0.6041666865348816,
      "ranking_idealized_expo": 0.5083333253860474,
      "ranking_simple": 0.5416666865348816,
      "regularize": 0.29497501254081726,
      "step": 240,
      "wo_beta": 16.538175582885742
    },
    {
      "dpo_loss": 0.6231993436813354,
      "epoch": 0.6943788379782712,
      "grad_norm": 16269.619431330953,
      "learning_rate": 4.7405033579636755e-07,
      "logits": -1.5562618970870972,
      "logps": -86.00993347167969,
      "loss": 122.2342,
      "objective": 110.85368347167969,
      "ranking_idealized": 0.6166666746139526,
      "ranking_idealized_expo": 0.5458333492279053,
      "ranking_simple": 0.574999988079071,
      "regularize": 0.2635067105293274,
      "step": 245,
      "wo_beta": 16.003578186035156
    },
    {
      "dpo_loss": 0.608769953250885,
      "epoch": 0.7085498346717053,
      "grad_norm": 16883.673846830992,
      "learning_rate": 4.721858981942284e-07,
      "logits": -1.3197777271270752,
      "logps": -84.4737319946289,
      "loss": 130.7065,
      "objective": 131.6678924560547,
      "ranking_idealized": 0.612500011920929,
      "ranking_idealized_expo": 0.5291666388511658,
      "ranking_simple": 0.5708333253860474,
      "regularize": 0.29679057002067566,
      "step": 250,
      "wo_beta": 15.271538734436035
    },
    {
      "epoch": 0.7085498346717053,
      "eval_dpo_loss": 0.680046558380127,
      "eval_logits": -1.5061429738998413,
      "eval_logps": -91.61782836914062,
      "eval_loss": 181.50924682617188,
      "eval_objective": 178.27838134765625,
      "eval_ranking_idealized": 0.6024844646453857,
      "eval_ranking_idealized_expo": 0.5232919454574585,
      "eval_ranking_simple": 0.5305383205413818,
      "eval_regularize": 0.404918372631073,
      "eval_runtime": 368.2121,
      "eval_samples_per_second": 15.725,
      "eval_steps_per_second": 1.312,
      "eval_wo_beta": 16.640703201293945,
      "step": 250
    },
    {
      "dpo_loss": 0.6077960729598999,
      "epoch": 0.7227208313651393,
      "grad_norm": 16400.742832285967,
      "learning_rate": 4.702607171860353e-07,
      "logits": -1.582943320274353,
      "logps": -83.8155746459961,
      "loss": 128.7728,
      "objective": 116.49504852294922,
      "ranking_idealized": 0.5708333253860474,
      "ranking_idealized_expo": 0.42916667461395264,
      "ranking_simple": 0.4833333194255829,
      "regularize": 0.28917694091796875,
      "step": 255,
      "wo_beta": 15.234207153320312
    },
    {
      "dpo_loss": 0.6078327298164368,
      "epoch": 0.7368918280585735,
      "grad_norm": 19074.785460432806,
      "learning_rate": 4.6827531909705327e-07,
      "logits": -1.5513815879821777,
      "logps": -85.73532104492188,
      "loss": 120.8491,
      "objective": 135.26841735839844,
      "ranking_idealized": 0.5416666865348816,
      "ranking_idealized_expo": 0.4625000059604645,
      "ranking_simple": 0.512499988079071,
      "regularize": 0.3036416172981262,
      "step": 260,
      "wo_beta": 15.789148330688477
    },
    {
      "dpo_loss": 0.6069940328598022,
      "epoch": 0.7510628247520076,
      "grad_norm": 14839.46399025929,
      "learning_rate": 4.662302467152955e-07,
      "logits": -1.4743403196334839,
      "logps": -85.0496597290039,
      "loss": 115.1093,
      "objective": 120.47551727294922,
      "ranking_idealized": 0.6625000238418579,
      "ranking_idealized_expo": 0.5666666626930237,
      "ranking_simple": 0.6208333373069763,
      "regularize": 0.29132476449012756,
      "step": 265,
      "wo_beta": 16.472087860107422
    },
    {
      "dpo_loss": 0.6180242300033569,
      "epoch": 0.7652338214454416,
      "grad_norm": 17448.69545980003,
      "learning_rate": 4.6412605914313143e-07,
      "logits": -1.5716725587844849,
      "logps": -85.8649673461914,
      "loss": 116.6119,
      "objective": 116.45634460449219,
      "ranking_idealized": 0.5249999761581421,
      "ranking_idealized_expo": 0.4583333432674408,
      "ranking_simple": 0.4791666567325592,
      "regularize": 0.25854194164276123,
      "step": 270,
      "wo_beta": 14.43771743774414
    },
    {
      "dpo_loss": 0.6129618287086487,
      "epoch": 0.7794048181388757,
      "grad_norm": 16560.36826913483,
      "learning_rate": 4.619633316444329e-07,
      "logits": -1.4034606218338013,
      "logps": -84.92838287353516,
      "loss": 111.1039,
      "objective": 110.11347961425781,
      "ranking_idealized": 0.574999988079071,
      "ranking_idealized_expo": 0.5041666626930237,
      "ranking_simple": 0.5208333134651184,
      "regularize": 0.2595362663269043,
      "step": 275,
      "wo_beta": 15.505229949951172
    },
    {
      "dpo_loss": 0.6037231087684631,
      "epoch": 0.7935758148323099,
      "grad_norm": 18592.99967397487,
      "learning_rate": 4.597426554873036e-07,
      "logits": -1.5048367977142334,
      "logps": -85.48095703125,
      "loss": 124.5191,
      "objective": 130.4561767578125,
      "ranking_idealized": 0.5874999761581421,
      "ranking_idealized_expo": 0.5208333134651184,
      "ranking_simple": 0.5583333373069763,
      "regularize": 0.27942749857902527,
      "step": 280,
      "wo_beta": 15.601365089416504
    },
    {
      "dpo_loss": 0.6074939370155334,
      "epoch": 0.807746811525744,
      "grad_norm": 16296.64397951243,
      "learning_rate": 4.574646377824315e-07,
      "logits": -1.4947975873947144,
      "logps": -87.23796844482422,
      "loss": 116.6867,
      "objective": 114.05181121826172,
      "ranking_idealized": 0.5666666626930237,
      "ranking_idealized_expo": 0.4791666567325592,
      "ranking_simple": 0.5083333253860474,
      "regularize": 0.2666790187358856,
      "step": 285,
      "wo_beta": 17.003517150878906
    },
    {
      "dpo_loss": 0.5908948183059692,
      "epoch": 0.821917808219178,
      "grad_norm": 16844.100126093494,
      "learning_rate": 4.551299013171111e-07,
      "logits": -1.500679850578308,
      "logps": -86.47892761230469,
      "loss": 113.1195,
      "objective": 110.57820892333984,
      "ranking_idealized": 0.5874999761581421,
      "ranking_idealized_expo": 0.5041666626930237,
      "ranking_simple": 0.5333333611488342,
      "regularize": 0.25748953223228455,
      "step": 290,
      "wo_beta": 16.26107406616211
    },
    {
      "dpo_loss": 0.602837860584259,
      "epoch": 0.8360888049126122,
      "grad_norm": 15918.83667114978,
      "learning_rate": 4.5273908438498e-07,
      "logits": -1.488081693649292,
      "logps": -86.3355484008789,
      "loss": 112.7178,
      "objective": 110.10317993164062,
      "ranking_idealized": 0.550000011920929,
      "ranking_idealized_expo": 0.4958333373069763,
      "ranking_simple": 0.5041666626930237,
      "regularize": 0.2576940059661865,
      "step": 295,
      "wo_beta": 16.481365203857422
    },
    {
      "dpo_loss": 0.6130139231681824,
      "epoch": 0.8502598016060463,
      "grad_norm": 16024.42965961574,
      "learning_rate": 4.502928406115152e-07,
      "logits": -1.4704513549804688,
      "logps": -84.77509307861328,
      "loss": 109.74,
      "objective": 110.3909683227539,
      "ranking_idealized": 0.625,
      "ranking_idealized_expo": 0.550000011920929,
      "ranking_simple": 0.5791666507720947,
      "regularize": 0.25806304812431335,
      "step": 300,
      "wo_beta": 14.735386848449707
    },
    {
      "epoch": 0.8502598016060463,
      "eval_dpo_loss": 0.6815229654312134,
      "eval_logits": -1.476037859916687,
      "eval_logps": -92.42357635498047,
      "eval_loss": 180.4923553466797,
      "eval_objective": 178.13650512695312,
      "eval_ranking_idealized": 0.6024844646453857,
      "eval_ranking_idealized_expo": 0.5232919454574585,
      "eval_ranking_simple": 0.5305383205413818,
      "eval_regularize": 0.40466198325157166,
      "eval_runtime": 369.1363,
      "eval_samples_per_second": 15.685,
      "eval_steps_per_second": 1.308,
      "eval_wo_beta": 16.49808120727539,
      "step": 300
    },
    {
      "dpo_loss": 0.6006699204444885,
      "epoch": 0.8644307982994804,
      "grad_norm": 16503.480891595664,
      "learning_rate": 4.4779183877533877e-07,
      "logits": -1.4632763862609863,
      "logps": -85.11184692382812,
      "loss": 110.0493,
      "objective": 114.68462371826172,
      "ranking_idealized": 0.6166666746139526,
      "ranking_idealized_expo": 0.5208333134651184,
      "ranking_simple": 0.5833333134651184,
      "regularize": 0.25510621070861816,
      "step": 305,
      "wo_beta": 15.895527839660645
    },
    {
      "dpo_loss": 0.6042004227638245,
      "epoch": 0.8786017949929145,
      "grad_norm": 15793.814439937723,
      "learning_rate": 4.4523676262538045e-07,
      "logits": -1.47891104221344,
      "logps": -84.36697387695312,
      "loss": 112.7694,
      "objective": 122.6783676147461,
      "ranking_idealized": 0.5958333611488342,
      "ranking_idealized_expo": 0.4833333194255829,
      "ranking_simple": 0.5291666388511658,
      "regularize": 0.2812657058238983,
      "step": 310,
      "wo_beta": 15.347722053527832
    },
    {
      "dpo_loss": 0.591303825378418,
      "epoch": 0.8927727916863486,
      "grad_norm": 17495.093435377945,
      "learning_rate": 4.426283106939473e-07,
      "logits": -1.4973819255828857,
      "logps": -84.14984893798828,
      "loss": 105.8584,
      "objective": 102.11656188964844,
      "ranking_idealized": 0.5541666746139526,
      "ranking_idealized_expo": 0.47083333134651184,
      "ranking_simple": 0.512499988079071,
      "regularize": 0.23703667521476746,
      "step": 315,
      "wo_beta": 15.412622451782227
    },
    {
      "dpo_loss": 0.5901548862457275,
      "epoch": 0.9069437883797827,
      "grad_norm": 16776.49496984533,
      "learning_rate": 4.3996719610575215e-07,
      "logits": -1.4549332857131958,
      "logps": -86.48863983154297,
      "loss": 101.9659,
      "objective": 103.75129699707031,
      "ranking_idealized": 0.6208333373069763,
      "ranking_idealized_expo": 0.5583333373069763,
      "ranking_simple": 0.6000000238418579,
      "regularize": 0.2400914579629898,
      "step": 320,
      "wo_beta": 15.67490005493164
    },
    {
      "dpo_loss": 0.5979679226875305,
      "epoch": 0.9211147850732169,
      "grad_norm": 16212.553406478031,
      "learning_rate": 4.372541463829523e-07,
      "logits": -1.543658971786499,
      "logps": -87.07477569580078,
      "loss": 110.2578,
      "objective": 94.58499908447266,
      "ranking_idealized": 0.6083333492279053,
      "ranking_idealized_expo": 0.5,
      "ranking_simple": 0.5583333373069763,
      "regularize": 0.23497775197029114,
      "step": 325,
      "wo_beta": 15.544549942016602
    },
    {
      "dpo_loss": 0.5859458446502686,
      "epoch": 0.9352857817666509,
      "grad_norm": 17177.10951815794,
      "learning_rate": 4.344899032462524e-07,
      "logits": -1.3802608251571655,
      "logps": -86.10081481933594,
      "loss": 113.3699,
      "objective": 119.8874740600586,
      "ranking_idealized": 0.6166666746139526,
      "ranking_idealized_expo": 0.5291666388511658,
      "ranking_simple": 0.574999988079071,
      "regularize": 0.2620205283164978,
      "step": 330,
      "wo_beta": 16.999446868896484
    },
    {
      "dpo_loss": 0.5934227108955383,
      "epoch": 0.949456778460085,
      "grad_norm": 17661.288172623317,
      "learning_rate": 4.316752224121252e-07,
      "logits": -1.4096896648406982,
      "logps": -85.97354125976562,
      "loss": 112.801,
      "objective": 102.87708282470703,
      "ranking_idealized": 0.5874999761581421,
      "ranking_idealized_expo": 0.5208333134651184,
      "ranking_simple": 0.5708333253860474,
      "regularize": 0.226911723613739,
      "step": 335,
      "wo_beta": 15.912822723388672
    },
    {
      "dpo_loss": 0.5984110236167908,
      "epoch": 0.9636277751535192,
      "grad_norm": 16330.575056246405,
      "learning_rate": 4.2881087338620634e-07,
      "logits": -1.4624823331832886,
      "logps": -85.42594909667969,
      "loss": 112.5159,
      "objective": 105.5082015991211,
      "ranking_idealized": 0.6041666865348816,
      "ranking_idealized_expo": 0.5458333492279053,
      "ranking_simple": 0.5916666388511658,
      "regularize": 0.24777108430862427,
      "step": 340,
      "wo_beta": 16.43703269958496
    },
    {
      "dpo_loss": 0.5881261825561523,
      "epoch": 0.9777987718469532,
      "grad_norm": 16460.598983396474,
      "learning_rate": 4.258976392529192e-07,
      "logits": -1.5221667289733887,
      "logps": -84.57250213623047,
      "loss": 105.3708,
      "objective": 96.2402114868164,
      "ranking_idealized": 0.6458333134651184,
      "ranking_idealized_expo": 0.5833333134651184,
      "ranking_simple": 0.625,
      "regularize": 0.22673317790031433,
      "step": 345,
      "wo_beta": 15.497628211975098
    },
    {
      "dpo_loss": 0.582562267780304,
      "epoch": 0.9919697685403873,
      "grad_norm": 15979.907994389112,
      "learning_rate": 4.2293631646138735e-07,
      "logits": -1.4198105335235596,
      "logps": -87.27174377441406,
      "loss": 104.2663,
      "objective": 106.56012725830078,
      "ranking_idealized": 0.6583333611488342,
      "ranking_idealized_expo": 0.574999988079071,
      "ranking_simple": 0.6208333373069763,
      "regularize": 0.2592408061027527,
      "step": 350,
      "wo_beta": 15.726160049438477
    },
    {
      "epoch": 0.9919697685403873,
      "eval_dpo_loss": 0.6808353066444397,
      "eval_logits": -1.5066107511520386,
      "eval_logps": -92.80049133300781,
      "eval_loss": 182.25906372070312,
      "eval_objective": 178.8644256591797,
      "eval_ranking_idealized": 0.6024844646453857,
      "eval_ranking_idealized_expo": 0.5232919454574585,
      "eval_ranking_simple": 0.5289855003356934,
      "eval_regularize": 0.4057510197162628,
      "eval_runtime": 368.5888,
      "eval_samples_per_second": 15.709,
      "eval_steps_per_second": 1.31,
      "eval_wo_beta": 16.569448471069336,
      "step": 350
    },
    {
      "dpo_loss": 0.5861319899559021,
      "epoch": 1.0061407652338215,
      "grad_norm": 16794.52051588047,
      "learning_rate": 4.1992771460769325e-07,
      "logits": -1.6238858699798584,
      "logps": -84.81636810302734,
      "loss": 101.4044,
      "objective": 99.22207641601562,
      "ranking_idealized": 0.6083333492279053,
      "ranking_idealized_expo": 0.5375000238418579,
      "ranking_simple": 0.574999988079071,
      "regularize": 0.2406620979309082,
      "step": 355,
      "wo_beta": 16.778457641601562
    },
    {
      "dpo_loss": 0.5623802542686462,
      "epoch": 1.0203117619272555,
      "grad_norm": 17512.309592114732,
      "learning_rate": 4.168726562135431e-07,
      "logits": -1.4817250967025757,
      "logps": -85.78034973144531,
      "loss": 89.2678,
      "objective": 87.2276840209961,
      "ranking_idealized": 0.574999988079071,
      "ranking_idealized_expo": 0.5083333253860474,
      "ranking_simple": 0.5708333253860474,
      "regularize": 0.20162057876586914,
      "step": 360,
      "wo_beta": 16.540082931518555
    },
    {
      "dpo_loss": 0.5711230635643005,
      "epoch": 1.0344827586206897,
      "grad_norm": 17722.39127835427,
      "learning_rate": 4.1377197650139734e-07,
      "logits": -1.5016947984695435,
      "logps": -86.203369140625,
      "loss": 94.5137,
      "objective": 95.95963287353516,
      "ranking_idealized": 0.5833333134651184,
      "ranking_idealized_expo": 0.5,
      "ranking_simple": 0.5625,
      "regularize": 0.21429090201854706,
      "step": 365,
      "wo_beta": 15.065950393676758
    },
    {
      "dpo_loss": 0.5723836421966553,
      "epoch": 1.0486537553141237,
      "grad_norm": 16746.74468565218,
      "learning_rate": 4.106265231661291e-07,
      "logits": -1.4276858568191528,
      "logps": -84.18301391601562,
      "loss": 87.9015,
      "objective": 87.47245788574219,
      "ranking_idealized": 0.5958333611488342,
      "ranking_idealized_expo": 0.5166666507720947,
      "ranking_simple": 0.5666666626930237,
      "regularize": 0.20792974531650543,
      "step": 370,
      "wo_beta": 15.806612968444824
    },
    {
      "dpo_loss": 0.575459897518158,
      "epoch": 1.0628247520075578,
      "grad_norm": 16711.49471758267,
      "learning_rate": 4.0743715614327314e-07,
      "logits": -1.4709128141403198,
      "logps": -84.51998901367188,
      "loss": 81.3317,
      "objective": 80.16383361816406,
      "ranking_idealized": 0.6083333492279053,
      "ranking_idealized_expo": 0.5166666507720947,
      "ranking_simple": 0.5666666626930237,
      "regularize": 0.19415231049060822,
      "step": 375,
      "wo_beta": 16.286664962768555
    },
    {
      "dpo_loss": 0.5680096745491028,
      "epoch": 1.076995748700992,
      "grad_norm": 15938.178621987885,
      "learning_rate": 4.042047473739277e-07,
      "logits": -1.4488080739974976,
      "logps": -86.38304138183594,
      "loss": 88.4379,
      "objective": 90.77100372314453,
      "ranking_idealized": 0.6625000238418579,
      "ranking_idealized_expo": 0.574999988079071,
      "ranking_simple": 0.6333333253860474,
      "regularize": 0.21614128351211548,
      "step": 380,
      "wo_beta": 15.816045761108398
    },
    {
      "dpo_loss": 0.5828992128372192,
      "epoch": 1.091166745394426,
      "grad_norm": 15573.908569824505,
      "learning_rate": 4.009301805663752e-07,
      "logits": -1.4298585653305054,
      "logps": -85.34860229492188,
      "loss": 90.2727,
      "objective": 100.38928985595703,
      "ranking_idealized": 0.6000000238418579,
      "ranking_idealized_expo": 0.5583333373069763,
      "ranking_simple": 0.5791666507720947,
      "regularize": 0.21903865039348602,
      "step": 385,
      "wo_beta": 15.969101905822754
    },
    {
      "dpo_loss": 0.5651105046272278,
      "epoch": 1.10533774208786,
      "grad_norm": 15903.36624651428,
      "learning_rate": 3.9761435095448424e-07,
      "logits": -1.386973261833191,
      "logps": -86.8327865600586,
      "loss": 85.9698,
      "objective": 81.47636413574219,
      "ranking_idealized": 0.637499988079071,
      "ranking_idealized_expo": 0.5625,
      "ranking_simple": 0.612500011920929,
      "regularize": 0.1953142136335373,
      "step": 390,
      "wo_beta": 16.701154708862305
    },
    {
      "dpo_loss": 0.5567125082015991,
      "epoch": 1.1195087387812943,
      "grad_norm": 17073.21610205935,
      "learning_rate": 3.942581650529625e-07,
      "logits": -1.4661533832550049,
      "logps": -86.22716522216797,
      "loss": 78.9786,
      "objective": 86.99171447753906,
      "ranking_idealized": 0.5791666507720947,
      "ranking_idealized_expo": 0.5041666626930237,
      "ranking_simple": 0.5458333492279053,
      "regularize": 0.19182702898979187,
      "step": 395,
      "wo_beta": 15.073732376098633
    },
    {
      "dpo_loss": 0.5746569037437439,
      "epoch": 1.1336797354747283,
      "grad_norm": 17100.977239850836,
      "learning_rate": 3.908625404095242e-07,
      "logits": -1.542074203491211,
      "logps": -84.60574340820312,
      "loss": 91.3585,
      "objective": 91.53578186035156,
      "ranking_idealized": 0.6291666626930237,
      "ranking_idealized_expo": 0.5416666865348816,
      "ranking_simple": 0.5833333134651184,
      "regularize": 0.2116149365901947,
      "step": 400,
      "wo_beta": 15.816818237304688
    },
    {
      "epoch": 1.1336797354747283,
      "eval_dpo_loss": 0.6799505352973938,
      "eval_logits": -1.478875756263733,
      "eval_logps": -92.38536071777344,
      "eval_loss": 180.029541015625,
      "eval_objective": 177.71481323242188,
      "eval_ranking_idealized": 0.6024844646453857,
      "eval_ranking_idealized_expo": 0.5232919454574585,
      "eval_ranking_simple": 0.5279502868652344,
      "eval_regularize": 0.402423620223999,
      "eval_runtime": 374.7281,
      "eval_samples_per_second": 15.451,
      "eval_steps_per_second": 1.289,
      "eval_wo_beta": 16.585174560546875,
      "step": 400
    },
    {
      "dpo_loss": 0.5667446255683899,
      "epoch": 1.1478507321681626,
      "grad_norm": 17235.250314074532,
      "learning_rate": 3.874284053540415e-07,
      "logits": -1.506400227546692,
      "logps": -86.09246063232422,
      "loss": 87.7881,
      "objective": 99.79354095458984,
      "ranking_idealized": 0.5666666626930237,
      "ranking_idealized_expo": 0.4625000059604645,
      "ranking_simple": 0.5375000238418579,
      "regularize": 0.21635954082012177,
      "step": 405,
      "wo_beta": 17.256864547729492
    },
    {
      "dpo_loss": 0.5707606673240662,
      "epoch": 1.1620217288615966,
      "grad_norm": 16725.889726383095,
      "learning_rate": 3.839566987447491e-07,
      "logits": -1.5043673515319824,
      "logps": -85.6620864868164,
      "loss": 88.1003,
      "objective": 87.76438903808594,
      "ranking_idealized": 0.6000000238418579,
      "ranking_idealized_expo": 0.5208333134651184,
      "ranking_simple": 0.5666666626930237,
      "regularize": 0.19931498169898987,
      "step": 410,
      "wo_beta": 16.657556533813477
    },
    {
      "dpo_loss": 0.5724092721939087,
      "epoch": 1.1761927255550306,
      "grad_norm": 14812.855448630145,
      "learning_rate": 3.804483697115693e-07,
      "logits": -1.420817255973816,
      "logps": -85.20552062988281,
      "loss": 80.2264,
      "objective": 89.5212173461914,
      "ranking_idealized": 0.612500011920929,
      "ranking_idealized_expo": 0.5166666507720947,
      "ranking_simple": 0.5874999761581421,
      "regularize": 0.21374522149562836,
      "step": 415,
      "wo_beta": 16.42493438720703
    },
    {
      "dpo_loss": 0.5684979557991028,
      "epoch": 1.1903637222484649,
      "grad_norm": 16758.534351302478,
      "learning_rate": 3.769043773966292e-07,
      "logits": -1.3999756574630737,
      "logps": -86.63607788085938,
      "loss": 86.6386,
      "objective": 74.90924835205078,
      "ranking_idealized": 0.6000000238418579,
      "ranking_idealized_expo": 0.4958333373069763,
      "ranking_simple": 0.5708333253860474,
      "regularize": 0.17667636275291443,
      "step": 420,
      "wo_beta": 16.390174865722656
    },
    {
      "dpo_loss": 0.5639453530311584,
      "epoch": 1.204534718941899,
      "grad_norm": 16337.276317485204,
      "learning_rate": 3.733256906920412e-07,
      "logits": -1.4687834978103638,
      "logps": -85.14289093017578,
      "loss": 88.4628,
      "objective": 91.70726013183594,
      "ranking_idealized": 0.5958333611488342,
      "ranking_idealized_expo": 0.512499988079071,
      "ranking_simple": 0.574999988079071,
      "regularize": 0.21049639582633972,
      "step": 425,
      "wo_beta": 15.31130599975586
    },
    {
      "dpo_loss": 0.5734551548957825,
      "epoch": 1.2187057156353331,
      "grad_norm": 15248.100441462713,
      "learning_rate": 3.6971328797501735e-07,
      "logits": -1.448046088218689,
      "logps": -86.40514373779297,
      "loss": 77.2816,
      "objective": 79.60990142822266,
      "ranking_idealized": 0.5625,
      "ranking_idealized_expo": 0.4958333373069763,
      "ranking_simple": 0.5458333492279053,
      "regularize": 0.17995795607566833,
      "step": 430,
      "wo_beta": 16.629831314086914
    },
    {
      "dpo_loss": 0.5668199062347412,
      "epoch": 1.2328767123287672,
      "grad_norm": 16306.828228688702,
      "learning_rate": 3.660681568403909e-07,
      "logits": -1.4072421789169312,
      "logps": -85.66299438476562,
      "loss": 87.0271,
      "objective": 85.58671569824219,
      "ranking_idealized": 0.6458333134651184,
      "ranking_idealized_expo": 0.5958333611488342,
      "ranking_simple": 0.6208333373069763,
      "regularize": 0.19756914675235748,
      "step": 435,
      "wo_beta": 17.663021087646484
    },
    {
      "dpo_loss": 0.5658931732177734,
      "epoch": 1.2470477090222012,
      "grad_norm": 20345.22767033362,
      "learning_rate": 3.623912938306176e-07,
      "logits": -1.4035090208053589,
      "logps": -85.33922576904297,
      "loss": 79.1586,
      "objective": 74.25531768798828,
      "ranking_idealized": 0.612500011920929,
      "ranking_idealized_expo": 0.5541666746139526,
      "ranking_simple": 0.5791666507720947,
      "regularize": 0.17811766266822815,
      "step": 440,
      "wo_beta": 15.194890975952148
    },
    {
      "dpo_loss": 0.5541914105415344,
      "epoch": 1.2612187057156352,
      "grad_norm": 17279.48326071237,
      "learning_rate": 3.5868370416333116e-07,
      "logits": -1.4238730669021606,
      "logps": -87.22127532958984,
      "loss": 85.5554,
      "objective": 88.88020324707031,
      "ranking_idealized": 0.637499988079071,
      "ranking_idealized_expo": 0.5333333611488342,
      "ranking_simple": 0.6166666746139526,
      "regularize": 0.21330490708351135,
      "step": 445,
      "wo_beta": 17.520837783813477
    },
    {
      "dpo_loss": 0.5635745525360107,
      "epoch": 1.2753897024090695,
      "grad_norm": 15953.8185394536,
      "learning_rate": 3.549464014565264e-07,
      "logits": -1.5866882801055908,
      "logps": -86.32205963134766,
      "loss": 77.8925,
      "objective": 74.42088317871094,
      "ranking_idealized": 0.5541666746139526,
      "ranking_idealized_expo": 0.5,
      "ranking_simple": 0.5666666626930237,
      "regularize": 0.1866002231836319,
      "step": 450,
      "wo_beta": 14.815221786499023
    },
    {
      "epoch": 1.2753897024090695,
      "eval_dpo_loss": 0.679172158241272,
      "eval_logits": -1.4745829105377197,
      "eval_logps": -92.70618438720703,
      "eval_loss": 179.24407958984375,
      "eval_objective": 175.84751892089844,
      "eval_ranking_idealized": 0.6024844646453857,
      "eval_ranking_idealized_expo": 0.5232919454574585,
      "eval_ranking_simple": 0.5274327397346497,
      "eval_regularize": 0.39890119433403015,
      "eval_runtime": 369.0794,
      "eval_samples_per_second": 15.688,
      "eval_steps_per_second": 1.309,
      "eval_wo_beta": 16.526927947998047,
      "step": 450
    },
    {
      "dpo_loss": 0.5642288327217102,
      "epoch": 1.2895606991025035,
      "grad_norm": 15540.821203538826,
      "learning_rate": 3.511804074514468e-07,
      "logits": -1.3849934339523315,
      "logps": -86.52748107910156,
      "loss": 78.8058,
      "objective": 79.82011413574219,
      "ranking_idealized": 0.5458333492279053,
      "ranking_idealized_expo": 0.49166667461395264,
      "ranking_simple": 0.512499988079071,
      "regularize": 0.18714289367198944,
      "step": 455,
      "wo_beta": 15.67411994934082
    },
    {
      "dpo_loss": 0.5722388625144958,
      "epoch": 1.3037316957959377,
      "grad_norm": 17894.656202069662,
      "learning_rate": 3.4738675173325007e-07,
      "logits": -1.5175042152404785,
      "logps": -86.22012329101562,
      "loss": 80.887,
      "objective": 79.5283432006836,
      "ranking_idealized": 0.5583333373069763,
      "ranking_idealized_expo": 0.4749999940395355,
      "ranking_simple": 0.5291666388511658,
      "regularize": 0.18461202085018158,
      "step": 460,
      "wo_beta": 16.33133316040039
    },
    {
      "dpo_loss": 0.5584803819656372,
      "epoch": 1.3179026924893718,
      "grad_norm": 15804.388771620323,
      "learning_rate": 3.4356647144953003e-07,
      "logits": -1.4979623556137085,
      "logps": -84.8365478515625,
      "loss": 78.4995,
      "objective": 80.0457763671875,
      "ranking_idealized": 0.5541666746139526,
      "ranking_idealized_expo": 0.4749999940395355,
      "ranking_simple": 0.5291666388511658,
      "regularize": 0.19293205440044403,
      "step": 465,
      "wo_beta": 14.855480194091797
    },
    {
      "dpo_loss": 0.5635024309158325,
      "epoch": 1.3320736891828058,
      "grad_norm": 15763.736551798547,
      "learning_rate": 3.3972061102677124e-07,
      "logits": -1.5794017314910889,
      "logps": -83.71866607666016,
      "loss": 80.6678,
      "objective": 83.6802749633789,
      "ranking_idealized": 0.612500011920929,
      "ranking_idealized_expo": 0.5,
      "ranking_simple": 0.5916666388511658,
      "regularize": 0.18187165260314941,
      "step": 470,
      "wo_beta": 14.902677536010742
    },
    {
      "dpo_loss": 0.5684855580329895,
      "epoch": 1.34624468587624,
      "grad_norm": 16747.760130279752,
      "learning_rate": 3.3585022188481246e-07,
      "logits": -1.45767343044281,
      "logps": -85.98019409179688,
      "loss": 70.8929,
      "objective": 75.84042358398438,
      "ranking_idealized": 0.6416666507720947,
      "ranking_idealized_expo": 0.574999988079071,
      "ranking_simple": 0.6208333373069763,
      "regularize": 0.19827060401439667,
      "step": 475,
      "wo_beta": 16.622520446777344
    },
    {
      "dpo_loss": 0.5594444274902344,
      "epoch": 1.360415682569674,
      "grad_norm": 15233.336787247383,
      "learning_rate": 3.3195636214939935e-07,
      "logits": -1.5256932973861694,
      "logps": -86.3495101928711,
      "loss": 72.525,
      "objective": 74.22408294677734,
      "ranking_idealized": 0.5666666626930237,
      "ranking_idealized_expo": 0.4791666567325592,
      "ranking_simple": 0.5416666865348816,
      "regularize": 0.1702665388584137,
      "step": 480,
      "wo_beta": 17.96377182006836
    },
    {
      "dpo_loss": 0.5718420743942261,
      "epoch": 1.3745866792631083,
      "grad_norm": 16631.30941131418,
      "learning_rate": 3.2804009636290396e-07,
      "logits": -1.5204293727874756,
      "logps": -84.49634552001953,
      "loss": 76.2635,
      "objective": 74.72454071044922,
      "ranking_idealized": 0.574999988079071,
      "ranking_idealized_expo": 0.4958333373069763,
      "ranking_simple": 0.5375000238418579,
      "regularize": 0.18596205115318298,
      "step": 485,
      "wo_beta": 16.379438400268555
    },
    {
      "dpo_loss": 0.5699793696403503,
      "epoch": 1.3887576759565423,
      "grad_norm": 18558.655175644297,
      "learning_rate": 3.241024951932884e-07,
      "logits": -1.4783555269241333,
      "logps": -86.9972915649414,
      "loss": 75.5371,
      "objective": 76.1517105102539,
      "ranking_idealized": 0.574999988079071,
      "ranking_idealized_expo": 0.5333333611488342,
      "ranking_simple": 0.5666666626930237,
      "regularize": 0.18451951444149017,
      "step": 490,
      "wo_beta": 15.841702461242676
    },
    {
      "dpo_loss": 0.5529130101203918,
      "epoch": 1.4029286726499763,
      "grad_norm": 20634.943800946956,
      "learning_rate": 3.201446351413958e-07,
      "logits": -1.4992899894714355,
      "logps": -87.463623046875,
      "loss": 79.5849,
      "objective": 71.89769744873047,
      "ranking_idealized": 0.5583333373069763,
      "ranking_idealized_expo": 0.4791666567325592,
      "ranking_simple": 0.5416666865348816,
      "regularize": 0.17148242890834808,
      "step": 495,
      "wo_beta": 15.643444061279297
    },
    {
      "dpo_loss": 0.5567610859870911,
      "epoch": 1.4170996693434104,
      "grad_norm": 17182.106354079515,
      "learning_rate": 3.161675982466454e-07,
      "logits": -1.4364333152770996,
      "logps": -86.2386703491211,
      "loss": 73.5844,
      "objective": 70.71806335449219,
      "ranking_idealized": 0.6000000238418579,
      "ranking_idealized_expo": 0.4833333194255829,
      "ranking_simple": 0.5625,
      "regularize": 0.16988083720207214,
      "step": 500,
      "wo_beta": 15.247078895568848
    },
    {
      "epoch": 1.4170996693434104,
      "eval_dpo_loss": 0.6786399483680725,
      "eval_logits": -1.4849432706832886,
      "eval_logps": -93.26954650878906,
      "eval_loss": 180.36428833007812,
      "eval_objective": 176.233154296875,
      "eval_ranking_idealized": 0.6024844646453857,
      "eval_ranking_idealized_expo": 0.5232919454574585,
      "eval_ranking_simple": 0.5305383205413818,
      "eval_regularize": 0.3994362950325012,
      "eval_runtime": 373.0338,
      "eval_samples_per_second": 15.521,
      "eval_steps_per_second": 1.295,
      "eval_wo_beta": 16.500259399414062,
      "step": 500
    },
    {
      "dpo_loss": 0.5469445586204529,
      "epoch": 1.4312706660368446,
      "grad_norm": 16377.98808062248,
      "learning_rate": 3.121724717912138e-07,
      "logits": -1.507896900177002,
      "logps": -86.27371978759766,
      "loss": 81.0237,
      "objective": 78.97586822509766,
      "ranking_idealized": 0.625,
      "ranking_idealized_expo": 0.5249999761581421,
      "ranking_simple": 0.5916666388511658,
      "regularize": 0.17829085886478424,
      "step": 505,
      "wo_beta": 16.57958984375
    },
    {
      "dpo_loss": 0.5666388273239136,
      "epoch": 1.4454416627302786,
      "grad_norm": 16987.7779441374,
      "learning_rate": 3.081603480027826e-07,
      "logits": -1.5229469537734985,
      "logps": -86.77935028076172,
      "loss": 72.4223,
      "objective": 68.7424087524414,
      "ranking_idealized": 0.5874999761581421,
      "ranking_idealized_expo": 0.5208333134651184,
      "ranking_simple": 0.5791666507720947,
      "regularize": 0.16074414551258087,
      "step": 510,
      "wo_beta": 15.368119239807129
    },
    {
      "dpo_loss": 0.5596444606781006,
      "epoch": 1.4596126594237129,
      "grad_norm": 18192.200248438854,
      "learning_rate": 3.0413232375593494e-07,
      "logits": -1.581657886505127,
      "logps": -87.4643783569336,
      "loss": 75.4207,
      "objective": 79.91267395019531,
      "ranking_idealized": 0.550000011920929,
      "ranking_idealized_expo": 0.4625000059604645,
      "ranking_simple": 0.5249999761581421,
      "regularize": 0.19809895753860474,
      "step": 515,
      "wo_beta": 16.213024139404297
    },
    {
      "dpo_loss": 0.5609327554702759,
      "epoch": 1.473783656117147,
      "grad_norm": 17307.001345557863,
      "learning_rate": 3.000895002722803e-07,
      "logits": -1.472069501876831,
      "logps": -84.53739929199219,
      "loss": 73.445,
      "objective": 75.21688079833984,
      "ranking_idealized": 0.574999988079071,
      "ranking_idealized_expo": 0.49166667461395264,
      "ranking_simple": 0.5333333611488342,
      "regularize": 0.17492397129535675,
      "step": 520,
      "wo_beta": 15.897907257080078
    },
    {
      "dpo_loss": 0.5437408685684204,
      "epoch": 1.487954652810581,
      "grad_norm": 16071.775164367858,
      "learning_rate": 2.960329828193918e-07,
      "logits": -1.3647209405899048,
      "logps": -83.83393859863281,
      "loss": 80.3729,
      "objective": 83.74950408935547,
      "ranking_idealized": 0.6000000238418579,
      "ranking_idealized_expo": 0.5,
      "ranking_simple": 0.5708333253860474,
      "regularize": 0.1921227127313614,
      "step": 525,
      "wo_beta": 15.854673385620117
    },
    {
      "dpo_loss": 0.5658655762672424,
      "epoch": 1.5021256495040152,
      "grad_norm": 16145.860622049926,
      "learning_rate": 2.919638804086369e-07,
      "logits": -1.5306588411331177,
      "logps": -86.52985382080078,
      "loss": 72.6051,
      "objective": 74.99629211425781,
      "ranking_idealized": 0.574999988079071,
      "ranking_idealized_expo": 0.5,
      "ranking_simple": 0.5583333373069763,
      "regularize": 0.16594384610652924,
      "step": 530,
      "wo_beta": 15.360599517822266
    },
    {
      "dpo_loss": 0.5535832643508911,
      "epoch": 1.5162966461974492,
      "grad_norm": 16070.421212133448,
      "learning_rate": 2.878833054919851e-07,
      "logits": -1.5020090341567993,
      "logps": -85.10139465332031,
      "loss": 76.8492,
      "objective": 71.94368743896484,
      "ranking_idealized": 0.5833333134651184,
      "ranking_idealized_expo": 0.5249999761581421,
      "ranking_simple": 0.5708333253860474,
      "regularize": 0.17868037521839142,
      "step": 535,
      "wo_beta": 15.670897483825684
    },
    {
      "dpo_loss": 0.5716097354888916,
      "epoch": 1.5304676428908834,
      "grad_norm": 17086.3623131816,
      "learning_rate": 2.8379237365787425e-07,
      "logits": -1.4154467582702637,
      "logps": -85.44342803955078,
      "loss": 72.6152,
      "objective": 71.88426208496094,
      "ranking_idealized": 0.6291666626930237,
      "ranking_idealized_expo": 0.5249999761581421,
      "ranking_simple": 0.5833333134651184,
      "regularize": 0.16814741492271423,
      "step": 540,
      "wo_beta": 15.268050193786621
    },
    {
      "dpo_loss": 0.5665360689163208,
      "epoch": 1.5446386395843175,
      "grad_norm": 15014.903555178817,
      "learning_rate": 2.7969220332622e-07,
      "logits": -1.5325461626052856,
      "logps": -86.13166046142578,
      "loss": 71.8279,
      "objective": 66.47465515136719,
      "ranking_idealized": 0.6583333611488342,
      "ranking_idealized_expo": 0.5874999761581421,
      "ranking_simple": 0.6333333253860474,
      "regularize": 0.15991279482841492,
      "step": 545,
      "wo_beta": 15.164175033569336
    },
    {
      "dpo_loss": 0.5597227811813354,
      "epoch": 1.5588096362777515,
      "grad_norm": 15652.670268373007,
      "learning_rate": 2.7558391544265126e-07,
      "logits": -1.5225752592086792,
      "logps": -86.28731536865234,
      "loss": 74.752,
      "objective": 75.39783477783203,
      "ranking_idealized": 0.5625,
      "ranking_idealized_expo": 0.46666666865348816,
      "ranking_simple": 0.5249999761581421,
      "regularize": 0.1811082661151886,
      "step": 550,
      "wo_beta": 15.459908485412598
    },
    {
      "epoch": 1.5588096362777515,
      "eval_dpo_loss": 0.6795263886451721,
      "eval_logits": -1.4832454919815063,
      "eval_logps": -92.88924407958984,
      "eval_loss": 181.36459350585938,
      "eval_objective": 177.22666931152344,
      "eval_ranking_idealized": 0.6024844646453857,
      "eval_ranking_idealized_expo": 0.5232919454574585,
      "eval_ranking_simple": 0.5274327397346497,
      "eval_regularize": 0.40197721123695374,
      "eval_runtime": 369.092,
      "eval_samples_per_second": 15.687,
      "eval_steps_per_second": 1.309,
      "eval_wo_beta": 16.55462646484375,
      "step": 550
    },
    {
      "dpo_loss": 0.5626943707466125,
      "epoch": 1.5729806329711855,
      "grad_norm": 18195.881077955793,
      "learning_rate": 2.7146863317205425e-07,
      "logits": -1.3379462957382202,
      "logps": -86.09709930419922,
      "loss": 77.0693,
      "objective": 73.5696792602539,
      "ranking_idealized": 0.6000000238418579,
      "ranking_idealized_expo": 0.5208333134651184,
      "ranking_simple": 0.5791666507720947,
      "regularize": 0.1626834124326706,
      "step": 555,
      "wo_beta": 14.696831703186035
    },
    {
      "dpo_loss": 0.552921712398529,
      "epoch": 1.5871516296646198,
      "grad_norm": 17118.412791156054,
      "learning_rate": 2.67347481591511e-07,
      "logits": -1.4245628118515015,
      "logps": -84.61250305175781,
      "loss": 71.0964,
      "objective": 72.3261489868164,
      "ranking_idealized": 0.6083333492279053,
      "ranking_idealized_expo": 0.5249999761581421,
      "ranking_simple": 0.5833333134651184,
      "regularize": 0.1769203245639801,
      "step": 560,
      "wo_beta": 16.116395950317383
    },
    {
      "dpo_loss": 0.5479990243911743,
      "epoch": 1.601322626358054,
      "grad_norm": 17861.734640053382,
      "learning_rate": 2.6322158738271414e-07,
      "logits": -1.3948200941085815,
      "logps": -85.63233184814453,
      "loss": 69.8501,
      "objective": 66.04268646240234,
      "ranking_idealized": 0.5458333492279053,
      "ranking_idealized_expo": 0.44999998807907104,
      "ranking_simple": 0.5333333611488342,
      "regularize": 0.1456415057182312,
      "step": 565,
      "wo_beta": 15.080788612365723
    },
    {
      "dpo_loss": 0.5674367547035217,
      "epoch": 1.615493623051488,
      "grad_norm": 17710.317360917932,
      "learning_rate": 2.590920785239436e-07,
      "logits": -1.5569151639938354,
      "logps": -86.57015228271484,
      "loss": 67.9067,
      "objective": 68.46839904785156,
      "ranking_idealized": 0.6000000238418579,
      "ranking_idealized_expo": 0.5166666507720947,
      "ranking_simple": 0.5708333253860474,
      "regularize": 0.16210322082042694,
      "step": 570,
      "wo_beta": 15.696702003479004
    },
    {
      "dpo_loss": 0.5638484358787537,
      "epoch": 1.629664619744922,
      "grad_norm": 18073.751646088458,
      "learning_rate": 2.549600839816884e-07,
      "logits": -1.5135074853897095,
      "logps": -84.736328125,
      "loss": 72.0534,
      "objective": 84.00183868408203,
      "ranking_idealized": 0.637499988079071,
      "ranking_idealized_expo": 0.5583333373069763,
      "ranking_simple": 0.6041666865348816,
      "regularize": 0.1912422776222229,
      "step": 575,
      "wo_beta": 16.294113159179688
    },
    {
      "dpo_loss": 0.5543667078018188,
      "epoch": 1.643835616438356,
      "grad_norm": 16118.144699864974,
      "learning_rate": 2.508267334019988e-07,
      "logits": -1.5415210723876953,
      "logps": -84.3241195678711,
      "loss": 67.4175,
      "objective": 66.42247009277344,
      "ranking_idealized": 0.6083333492279053,
      "ranking_idealized_expo": 0.5333333611488342,
      "ranking_simple": 0.5833333134651184,
      "regularize": 0.1573016196489334,
      "step": 580,
      "wo_beta": 14.898622512817383
    },
    {
      "dpo_loss": 0.5663701295852661,
      "epoch": 1.6580066131317903,
      "grad_norm": 17666.121211791866,
      "learning_rate": 2.4669315680165195e-07,
      "logits": -1.3956176042556763,
      "logps": -84.12041473388672,
      "loss": 66.3727,
      "objective": 61.66792678833008,
      "ranking_idealized": 0.5333333611488342,
      "ranking_idealized_expo": 0.4625000059604645,
      "ranking_simple": 0.5083333253860474,
      "regularize": 0.15171830356121063,
      "step": 585,
      "wo_beta": 15.958919525146484
    },
    {
      "dpo_loss": 0.558569610118866,
      "epoch": 1.6721776098252243,
      "grad_norm": 16850.29970069498,
      "learning_rate": 2.425604842592169e-07,
      "logits": -1.4625413417816162,
      "logps": -84.04219055175781,
      "loss": 74.8738,
      "objective": 81.01467895507812,
      "ranking_idealized": 0.6208333373069763,
      "ranking_idealized_expo": 0.5083333253860474,
      "ranking_simple": 0.5874999761581421,
      "regularize": 0.18167801201343536,
      "step": 590,
      "wo_beta": 13.437705039978027
    },
    {
      "dpo_loss": 0.5554817914962769,
      "epoch": 1.6863486065186586,
      "grad_norm": 17084.24865418485,
      "learning_rate": 2.384298456061022e-07,
      "logits": -1.4085568189620972,
      "logps": -84.96867370605469,
      "loss": 69.8306,
      "objective": 80.21109008789062,
      "ranking_idealized": 0.6166666746139526,
      "ranking_idealized_expo": 0.5375000238418579,
      "ranking_simple": 0.5833333134651184,
      "regularize": 0.18683354556560516,
      "step": 595,
      "wo_beta": 17.37420654296875
    },
    {
      "dpo_loss": 0.5489537119865417,
      "epoch": 1.7005196032120926,
      "grad_norm": 15218.152775914608,
      "learning_rate": 2.3430237011767164e-07,
      "logits": -1.4603925943374634,
      "logps": -85.71869659423828,
      "loss": 66.606,
      "objective": 76.13477325439453,
      "ranking_idealized": 0.6333333253860474,
      "ranking_idealized_expo": 0.550000011920929,
      "ranking_simple": 0.6166666746139526,
      "regularize": 0.17256483435630798,
      "step": 600,
      "wo_beta": 16.145771026611328
    },
    {
      "epoch": 1.7005196032120926,
      "eval_dpo_loss": 0.6788755655288696,
      "eval_logits": -1.4675469398498535,
      "eval_logps": -91.61579895019531,
      "eval_loss": 179.49525451660156,
      "eval_objective": 176.2792510986328,
      "eval_ranking_idealized": 0.6024844646453857,
      "eval_ranking_idealized_expo": 0.5232919454574585,
      "eval_ranking_simple": 0.5310559272766113,
      "eval_regularize": 0.3999297320842743,
      "eval_runtime": 368.9049,
      "eval_samples_per_second": 15.695,
      "eval_steps_per_second": 1.309,
      "eval_wo_beta": 16.6182804107666,
      "step": 600
    },
    {
      "dpo_loss": 0.564985454082489,
      "epoch": 1.7146905999055266,
      "grad_norm": 16714.450551200658,
      "learning_rate": 2.30179186204511e-07,
      "logits": -1.4053993225097656,
      "logps": -86.0546646118164,
      "loss": 69.0228,
      "objective": 73.44601440429688,
      "ranking_idealized": 0.5833333134651184,
      "ranking_idealized_expo": 0.5291666388511658,
      "ranking_simple": 0.5666666626930237,
      "regularize": 0.17200763523578644,
      "step": 605,
      "wo_beta": 18.306556701660156
    },
    {
      "dpo_loss": 0.5740829706192017,
      "epoch": 1.7288615965989607,
      "grad_norm": 18689.625067588473,
      "learning_rate": 2.2606142110393245e-07,
      "logits": -1.4901000261306763,
      "logps": -84.50035858154297,
      "loss": 69.9038,
      "objective": 70.67005920410156,
      "ranking_idealized": 0.6291666626930237,
      "ranking_idealized_expo": 0.5708333253860474,
      "ranking_simple": 0.625,
      "regularize": 0.1648392379283905,
      "step": 610,
      "wo_beta": 15.994599342346191
    },
    {
      "dpo_loss": 0.5698901414871216,
      "epoch": 1.743032593292395,
      "grad_norm": 17284.824092388248,
      "learning_rate": 2.2195020057179894e-07,
      "logits": -1.4990768432617188,
      "logps": -84.69489288330078,
      "loss": 69.1858,
      "objective": 62.69770431518555,
      "ranking_idealized": 0.5833333134651184,
      "ranking_idealized_expo": 0.5291666388511658,
      "ranking_simple": 0.5791666507720947,
      "regularize": 0.1527046412229538,
      "step": 615,
      "wo_beta": 15.638397216796875
    },
    {
      "dpo_loss": 0.553841233253479,
      "epoch": 1.7572035899858292,
      "grad_norm": 16362.729415329737,
      "learning_rate": 2.1784664857475352e-07,
      "logits": -1.54779851436615,
      "logps": -84.9485092163086,
      "loss": 69.826,
      "objective": 70.27579498291016,
      "ranking_idealized": 0.6208333373069763,
      "ranking_idealized_expo": 0.5416666865348816,
      "ranking_simple": 0.5833333134651184,
      "regularize": 0.1771712750196457,
      "step": 620,
      "wo_beta": 16.527475357055664
    },
    {
      "dpo_loss": 0.5640650987625122,
      "epoch": 1.7713745866792632,
      "grad_norm": 16519.00259049995,
      "learning_rate": 2.1375188698293854e-07,
      "logits": -1.456007480621338,
      "logps": -83.91544342041016,
      "loss": 67.7564,
      "objective": 73.47472381591797,
      "ranking_idealized": 0.6083333492279053,
      "ranking_idealized_expo": 0.5249999761581421,
      "ranking_simple": 0.5791666507720947,
      "regularize": 0.16373471915721893,
      "step": 625,
      "wo_beta": 15.541656494140625
    },
    {
      "dpo_loss": 0.5638567805290222,
      "epoch": 1.7855455833726972,
      "grad_norm": 16003.013277681921,
      "learning_rate": 2.0966703526328726e-07,
      "logits": -1.4914802312850952,
      "logps": -85.642822265625,
      "loss": 68.6768,
      "objective": 65.9126205444336,
      "ranking_idealized": 0.574999988079071,
      "ranking_idealized_expo": 0.49166667461395264,
      "ranking_simple": 0.5541666746139526,
      "regularize": 0.15695922076702118,
      "step": 630,
      "wo_beta": 16.262632369995117
    },
    {
      "dpo_loss": 0.5648781657218933,
      "epoch": 1.7997165800661312,
      "grad_norm": 17168.862791414114,
      "learning_rate": 2.0559321017347282e-07,
      "logits": -1.5868287086486816,
      "logps": -84.44722747802734,
      "loss": 63.1038,
      "objective": 59.76757049560547,
      "ranking_idealized": 0.637499988079071,
      "ranking_idealized_expo": 0.5291666388511658,
      "ranking_simple": 0.5916666388511658,
      "regularize": 0.15224316716194153,
      "step": 635,
      "wo_beta": 17.01032829284668
    },
    {
      "dpo_loss": 0.5457783341407776,
      "epoch": 1.8138875767595655,
      "grad_norm": 17993.04524523219,
      "learning_rate": 2.0153152545659796e-07,
      "logits": -1.5063692331314087,
      "logps": -85.88375091552734,
      "loss": 66.3807,
      "objective": 64.99014282226562,
      "ranking_idealized": 0.5666666626930237,
      "ranking_idealized_expo": 0.5083333253860474,
      "ranking_simple": 0.550000011920929,
      "regularize": 0.15310163795948029,
      "step": 640,
      "wo_beta": 17.52164649963379
    },
    {
      "dpo_loss": 0.5509151220321655,
      "epoch": 1.8280585734529995,
      "grad_norm": 19625.243857455473,
      "learning_rate": 1.9748309153670856e-07,
      "logits": -1.5516611337661743,
      "logps": -86.286376953125,
      "loss": 65.9614,
      "objective": 64.1622543334961,
      "ranking_idealized": 0.6416666507720947,
      "ranking_idealized_expo": 0.550000011920929,
      "ranking_simple": 0.6333333253860474,
      "regularize": 0.1483081430196762,
      "step": 645,
      "wo_beta": 16.739274978637695
    },
    {
      "dpo_loss": 0.5534842014312744,
      "epoch": 1.8422295701464337,
      "grad_norm": 17668.158898193855,
      "learning_rate": 1.9344901521521498e-07,
      "logits": -1.6079561710357666,
      "logps": -86.36868286132812,
      "loss": 65.4503,
      "objective": 62.571048736572266,
      "ranking_idealized": 0.612500011920929,
      "ranking_idealized_expo": 0.5208333134651184,
      "ranking_simple": 0.5833333134651184,
      "regularize": 0.14387734234333038,
      "step": 650,
      "wo_beta": 15.527718544006348
    },
    {
      "epoch": 1.8422295701464337,
      "eval_dpo_loss": 0.6790141463279724,
      "eval_logits": -1.504582166671753,
      "eval_logps": -91.89739990234375,
      "eval_loss": 180.12484741210938,
      "eval_objective": 176.55528259277344,
      "eval_ranking_idealized": 0.6024844646453857,
      "eval_ranking_idealized_expo": 0.5232919454574585,
      "eval_ranking_simple": 0.5284678936004639,
      "eval_regularize": 0.4002520442008972,
      "eval_runtime": 372.2243,
      "eval_samples_per_second": 15.555,
      "eval_steps_per_second": 1.298,
      "eval_wo_beta": 16.537281036376953,
      "step": 650
    },
    {
      "dpo_loss": 0.5599731206893921,
      "epoch": 1.8564005668398678,
      "grad_norm": 18359.19761584605,
      "learning_rate": 1.8943039936830344e-07,
      "logits": -1.4786803722381592,
      "logps": -83.48601531982422,
      "loss": 69.3454,
      "objective": 74.32158660888672,
      "ranking_idealized": 0.6000000238418579,
      "ranking_idealized_expo": 0.48750001192092896,
      "ranking_simple": 0.5583333373069763,
      "regularize": 0.1768590807914734,
      "step": 655,
      "wo_beta": 15.510722160339355
    },
    {
      "dpo_loss": 0.5625216960906982,
      "epoch": 1.8705715635333018,
      "grad_norm": 16101.25212298875,
      "learning_rate": 1.854283426454209e-07,
      "logits": -1.545279622077942,
      "logps": -86.23809051513672,
      "loss": 65.0768,
      "objective": 58.69490432739258,
      "ranking_idealized": 0.6041666865348816,
      "ranking_idealized_expo": 0.5333333611488342,
      "ranking_simple": 0.574999988079071,
      "regularize": 0.14343543350696564,
      "step": 660,
      "wo_beta": 17.122419357299805
    },
    {
      "dpo_loss": 0.5581023097038269,
      "epoch": 1.8847425602267358,
      "grad_norm": 16788.328924690355,
      "learning_rate": 1.8144393916891508e-07,
      "logits": -1.526328206062317,
      "logps": -85.24169921875,
      "loss": 63.7541,
      "objective": 67.25133514404297,
      "ranking_idealized": 0.6333333253860474,
      "ranking_idealized_expo": 0.550000011920929,
      "ranking_simple": 0.6208333373069763,
      "regularize": 0.14928290247917175,
      "step": 665,
      "wo_beta": 15.62762451171875
    },
    {
      "dpo_loss": 0.553265392780304,
      "epoch": 1.89891355692017,
      "grad_norm": 18214.847289885216,
      "learning_rate": 1.7747827823491252e-07,
      "logits": -1.4548065662384033,
      "logps": -83.87257385253906,
      "loss": 57.5593,
      "objective": 56.684471130371094,
      "ranking_idealized": 0.5666666626930237,
      "ranking_idealized_expo": 0.4791666567325592,
      "ranking_simple": 0.5333333611488342,
      "regularize": 0.14734616875648499,
      "step": 670,
      "wo_beta": 16.462730407714844
    },
    {
      "dpo_loss": 0.5662250518798828,
      "epoch": 1.9130845536136043,
      "grad_norm": 16267.920642791076,
      "learning_rate": 1.7353244401551565e-07,
      "logits": -1.478503704071045,
      "logps": -84.67176818847656,
      "loss": 60.0148,
      "objective": 62.5386962890625,
      "ranking_idealized": 0.6166666746139526,
      "ranking_idealized_expo": 0.5333333611488342,
      "ranking_simple": 0.574999988079071,
      "regularize": 0.14488257467746735,
      "step": 675,
      "wo_beta": 15.750804901123047
    },
    {
      "dpo_loss": 0.5515182018280029,
      "epoch": 1.9272555503070383,
      "grad_norm": 17294.380974269086,
      "learning_rate": 1.6960751526240118e-07,
      "logits": -1.5540010929107666,
      "logps": -86.8657455444336,
      "loss": 60.2369,
      "objective": 55.505897521972656,
      "ranking_idealized": 0.6166666746139526,
      "ranking_idealized_expo": 0.5,
      "ranking_simple": 0.6041666865348816,
      "regularize": 0.13418342173099518,
      "step": 680,
      "wo_beta": 15.37994384765625
    },
    {
      "dpo_loss": 0.5534944534301758,
      "epoch": 1.9414265470004723,
      "grad_norm": 17684.725981514333,
      "learning_rate": 1.6570456501189994e-07,
      "logits": -1.4706988334655762,
      "logps": -84.12257385253906,
      "loss": 60.967,
      "objective": 55.13488006591797,
      "ranking_idealized": 0.5291666388511658,
      "ranking_idealized_expo": 0.4791666567325592,
      "ranking_simple": 0.5291666388511658,
      "regularize": 0.13964900374412537,
      "step": 685,
      "wo_beta": 15.740229606628418
    },
    {
      "dpo_loss": 0.5616536736488342,
      "epoch": 1.9555975436939064,
      "grad_norm": 17181.452666650763,
      "learning_rate": 1.618246602916397e-07,
      "logits": -1.547702670097351,
      "logps": -84.80847930908203,
      "loss": 56.3999,
      "objective": 58.78403854370117,
      "ranking_idealized": 0.6041666865348816,
      "ranking_idealized_expo": 0.5583333373069763,
      "ranking_simple": 0.5916666388511658,
      "regularize": 0.13416926562786102,
      "step": 690,
      "wo_beta": 16.004222869873047
    },
    {
      "dpo_loss": 0.5623855590820312,
      "epoch": 1.9697685403873406,
      "grad_norm": 15698.603949750828,
      "learning_rate": 1.579688618288305e-07,
      "logits": -1.4090545177459717,
      "logps": -85.53604888916016,
      "loss": 57.3939,
      "objective": 54.246089935302734,
      "ranking_idealized": 0.5666666626930237,
      "ranking_idealized_expo": 0.4833333194255829,
      "ranking_simple": 0.550000011920929,
      "regularize": 0.12334556132555008,
      "step": 695,
      "wo_beta": 16.506813049316406
    },
    {
      "dpo_loss": 0.5531891584396362,
      "epoch": 1.9839395370807746,
      "grad_norm": 17232.305142451998,
      "learning_rate": 1.541382237602721e-07,
      "logits": -1.3992184400558472,
      "logps": -86.07640075683594,
      "loss": 62.3615,
      "objective": 63.35702896118164,
      "ranking_idealized": 0.6041666865348816,
      "ranking_idealized_expo": 0.5208333134651184,
      "ranking_simple": 0.5708333253860474,
      "regularize": 0.14704561233520508,
      "step": 700,
      "wo_beta": 15.364715576171875
    },
    {
      "epoch": 1.9839395370807746,
      "eval_dpo_loss": 0.678434431552887,
      "eval_logits": -1.4984484910964966,
      "eval_logps": -91.58750915527344,
      "eval_loss": 179.3856658935547,
      "eval_objective": 176.00213623046875,
      "eval_ranking_idealized": 0.6024844646453857,
      "eval_ranking_idealized_expo": 0.5232919454574585,
      "eval_ranking_simple": 0.5300207138061523,
      "eval_regularize": 0.3992062211036682,
      "eval_runtime": 368.882,
      "eval_samples_per_second": 15.696,
      "eval_steps_per_second": 1.309,
      "eval_wo_beta": 16.5863037109375,
      "step": 700
    },
    {
      "dpo_loss": 0.5491320490837097,
      "epoch": 1.9981105337742089,
      "grad_norm": 18325.752866553015,
      "learning_rate": 1.5033379334416375e-07,
      "logits": -1.3390460014343262,
      "logps": -84.89611053466797,
      "loss": 62.0311,
      "objective": 66.05867767333984,
      "ranking_idealized": 0.5916666388511658,
      "ranking_idealized_expo": 0.5,
      "ranking_simple": 0.5583333373069763,
      "regularize": 0.15552859008312225,
      "step": 705,
      "wo_beta": 17.203296661376953
    },
    {
      "dpo_loss": 0.5555277466773987,
      "epoch": 2.012281530467643,
      "grad_norm": 16395.524810634437,
      "learning_rate": 1.465566106737942e-07,
      "logits": -1.4830571413040161,
      "logps": -84.74015045166016,
      "loss": 56.8191,
      "objective": 61.21573257446289,
      "ranking_idealized": 0.6083333492279053,
      "ranking_idealized_expo": 0.5208333134651184,
      "ranking_simple": 0.6041666865348816,
      "regularize": 0.1427960991859436,
      "step": 710,
      "wo_beta": 15.91169261932373
    },
    {
      "dpo_loss": 0.5421245098114014,
      "epoch": 2.026452527161077,
      "grad_norm": 17457.779859055492,
      "learning_rate": 1.428077083931907e-07,
      "logits": -1.5156207084655762,
      "logps": -84.15880584716797,
      "loss": 51.1684,
      "objective": 52.4562873840332,
      "ranking_idealized": 0.6791666746139526,
      "ranking_idealized_expo": 0.5708333253860474,
      "ranking_simple": 0.6333333253860474,
      "regularize": 0.12656886875629425,
      "step": 715,
      "wo_beta": 15.599425315856934
    },
    {
      "dpo_loss": 0.5437305569648743,
      "epoch": 2.040623523854511,
      "grad_norm": 15758.315666253699,
      "learning_rate": 1.3908811141480406e-07,
      "logits": -1.4699770212173462,
      "logps": -84.7515869140625,
      "loss": 45.1211,
      "objective": 38.25383758544922,
      "ranking_idealized": 0.6291666626930237,
      "ranking_idealized_expo": 0.5249999761581421,
      "ranking_simple": 0.5958333611488342,
      "regularize": 0.10041950643062592,
      "step": 720,
      "wo_beta": 16.25193214416504
    },
    {
      "dpo_loss": 0.545520544052124,
      "epoch": 2.0547945205479454,
      "grad_norm": 16201.394561682804,
      "learning_rate": 1.353988366393083e-07,
      "logits": -1.5007617473602295,
      "logps": -86.95757293701172,
      "loss": 53.076,
      "objective": 52.1579704284668,
      "ranking_idealized": 0.5833333134651184,
      "ranking_idealized_expo": 0.4958333373069763,
      "ranking_simple": 0.5791666507720947,
      "regularize": 0.12382372468709946,
      "step": 725,
      "wo_beta": 14.748201370239258
    },
    {
      "dpo_loss": 0.5444363951683044,
      "epoch": 2.0689655172413794,
      "grad_norm": 18420.501731961805,
      "learning_rate": 1.3174089267758982e-07,
      "logits": -1.57591712474823,
      "logps": -84.48290252685547,
      "loss": 52.1829,
      "objective": 51.44971466064453,
      "ranking_idealized": 0.5791666507720947,
      "ranking_idealized_expo": 0.49166667461395264,
      "ranking_simple": 0.5791666507720947,
      "regularize": 0.12323012948036194,
      "step": 730,
      "wo_beta": 15.643149375915527
    },
    {
      "dpo_loss": 0.5511536002159119,
      "epoch": 2.0831365139348135,
      "grad_norm": 15885.917813358059,
      "learning_rate": 1.2811527957500343e-07,
      "logits": -1.499257206916809,
      "logps": -84.50511169433594,
      "loss": 44.0743,
      "objective": 49.55210876464844,
      "ranking_idealized": 0.5625,
      "ranking_idealized_expo": 0.47083333134651184,
      "ranking_simple": 0.5333333611488342,
      "regularize": 0.13047467172145844,
      "step": 735,
      "wo_beta": 17.085899353027344
    },
    {
      "dpo_loss": 0.5526978969573975,
      "epoch": 2.0973075106282475,
      "grad_norm": 16073.187372319342,
      "learning_rate": 1.245229885379699e-07,
      "logits": -1.5387953519821167,
      "logps": -84.2315444946289,
      "loss": 48.6885,
      "objective": 53.333866119384766,
      "ranking_idealized": 0.6166666746139526,
      "ranking_idealized_expo": 0.5249999761581421,
      "ranking_simple": 0.5958333611488342,
      "regularize": 0.12818403542041779,
      "step": 740,
      "wo_beta": 15.79825496673584
    },
    {
      "dpo_loss": 0.5457006692886353,
      "epoch": 2.1114785073216815,
      "grad_norm": 18439.02982973444,
      "learning_rate": 1.209650016629899e-07,
      "logits": -1.4960881471633911,
      "logps": -84.55073547363281,
      "loss": 49.7231,
      "objective": 46.95569610595703,
      "ranking_idealized": 0.6333333253860474,
      "ranking_idealized_expo": 0.5583333373069763,
      "ranking_simple": 0.6291666626930237,
      "regularize": 0.11624018102884293,
      "step": 745,
      "wo_beta": 15.793415069580078
    },
    {
      "dpo_loss": 0.5496144890785217,
      "epoch": 2.1256495040151155,
      "grad_norm": 16557.264742662323,
      "learning_rate": 1.1744229166814886e-07,
      "logits": -1.511896014213562,
      "logps": -83.8011245727539,
      "loss": 48.9708,
      "objective": 45.016258239746094,
      "ranking_idealized": 0.574999988079071,
      "ranking_idealized_expo": 0.4833333194255829,
      "ranking_simple": 0.550000011920929,
      "regularize": 0.11198277026414871,
      "step": 750,
      "wo_beta": 15.621644020080566
    },
    {
      "epoch": 2.1256495040151155,
      "eval_dpo_loss": 0.6794183254241943,
      "eval_logits": -1.491926908493042,
      "eval_logps": -92.19331359863281,
      "eval_loss": 179.8103485107422,
      "eval_objective": 176.70277404785156,
      "eval_ranking_idealized": 0.6024844646453857,
      "eval_ranking_idealized_expo": 0.5232919454574585,
      "eval_ranking_simple": 0.5274327397346497,
      "eval_regularize": 0.4011208415031433,
      "eval_runtime": 369.337,
      "eval_samples_per_second": 15.677,
      "eval_steps_per_second": 1.308,
      "eval_wo_beta": 16.588436126708984,
      "step": 750
    },
    {
      "dpo_loss": 0.5448576807975769,
      "epoch": 2.13982050070855,
      "grad_norm": 17193.17890323298,
      "learning_rate": 1.1395582162718523e-07,
      "logits": -1.4843658208847046,
      "logps": -86.97160339355469,
      "loss": 52.8805,
      "objective": 47.13671112060547,
      "ranking_idealized": 0.5916666388511658,
      "ranking_idealized_expo": 0.5166666507720947,
      "ranking_simple": 0.5833333134651184,
      "regularize": 0.11303378641605377,
      "step": 755,
      "wo_beta": 16.85382080078125
    },
    {
      "dpo_loss": 0.5693633556365967,
      "epoch": 2.153991497401984,
      "grad_norm": 16501.093616184942,
      "learning_rate": 1.10506544706196e-07,
      "logits": -1.4409741163253784,
      "logps": -83.32089233398438,
      "loss": 48.6534,
      "objective": 42.576026916503906,
      "ranking_idealized": 0.5958333611488342,
      "ranking_idealized_expo": 0.5208333134651184,
      "ranking_simple": 0.5791666507720947,
      "regularize": 0.10865426063537598,
      "step": 760,
      "wo_beta": 14.651175498962402
    },
    {
      "dpo_loss": 0.550613522529602,
      "epoch": 2.168162494095418,
      "grad_norm": 17100.049348037002,
      "learning_rate": 1.0709540390305061e-07,
      "logits": -1.4873898029327393,
      "logps": -84.55794525146484,
      "loss": 48.0396,
      "objective": 42.508872985839844,
      "ranking_idealized": 0.6208333373069763,
      "ranking_idealized_expo": 0.5375000238418579,
      "ranking_simple": 0.5958333611488342,
      "regularize": 0.10587514191865921,
      "step": 765,
      "wo_beta": 16.372541427612305
    },
    {
      "dpo_loss": 0.5513295531272888,
      "epoch": 2.182333490788852,
      "grad_norm": 16090.026254449373,
      "learning_rate": 1.0372333178958462e-07,
      "logits": -1.5015202760696411,
      "logps": -84.9146499633789,
      "loss": 48.0411,
      "objective": 50.020198822021484,
      "ranking_idealized": 0.5916666388511658,
      "ranking_idealized_expo": 0.49166667461395264,
      "ranking_simple": 0.5708333253860474,
      "regularize": 0.12123490124940872,
      "step": 770,
      "wo_beta": 14.4235200881958
    },
    {
      "dpo_loss": 0.5526517629623413,
      "epoch": 2.196504487482286,
      "grad_norm": 16092.050129989755,
      "learning_rate": 1.0039125025664391e-07,
      "logits": -1.4631909132003784,
      "logps": -86.0343017578125,
      "loss": 50.4495,
      "objective": 48.98077392578125,
      "ranking_idealized": 0.6291666626930237,
      "ranking_idealized_expo": 0.5791666507720947,
      "ranking_simple": 0.6166666746139526,
      "regularize": 0.12355396151542664,
      "step": 775,
      "wo_beta": 17.455713272094727
    },
    {
      "dpo_loss": 0.5543543100357056,
      "epoch": 2.21067548417572,
      "grad_norm": 17672.341083343217,
      "learning_rate": 9.710007026204894e-08,
      "logits": -1.4037829637527466,
      "logps": -86.01419067382812,
      "loss": 48.6758,
      "objective": 49.57035446166992,
      "ranking_idealized": 0.512499988079071,
      "ranking_idealized_expo": 0.44583332538604736,
      "ranking_simple": 0.48750001192092896,
      "regularize": 0.1090199202299118,
      "step": 780,
      "wo_beta": 15.31540298461914
    },
    {
      "dpo_loss": 0.5374571681022644,
      "epoch": 2.2248464808691546,
      "grad_norm": 17104.605257535815,
      "learning_rate": 9.385069158154805e-08,
      "logits": -1.4150718450546265,
      "logps": -85.85627746582031,
      "loss": 47.7928,
      "objective": 41.365787506103516,
      "ranking_idealized": 0.5833333134651184,
      "ranking_idealized_expo": 0.4791666567325592,
      "ranking_simple": 0.5541666746139526,
      "regularize": 0.10406169295310974,
      "step": 785,
      "wo_beta": 15.260098457336426
    },
    {
      "dpo_loss": 0.5317620635032654,
      "epoch": 2.2390174775625886,
      "grad_norm": 16507.88081841276,
      "learning_rate": 9.064400256282755e-08,
      "logits": -1.399611234664917,
      "logps": -86.44268035888672,
      "loss": 52.3325,
      "objective": 45.95266342163086,
      "ranking_idealized": 0.637499988079071,
      "ranking_idealized_expo": 0.5625,
      "ranking_simple": 0.637499988079071,
      "regularize": 0.1156383752822876,
      "step": 790,
      "wo_beta": 15.514897346496582
    },
    {
      "dpo_loss": 0.5416150689125061,
      "epoch": 2.2531884742560226,
      "grad_norm": 14862.627085415834,
      "learning_rate": 8.748087988264668e-08,
      "logits": -1.4897602796554565,
      "logps": -87.291259765625,
      "loss": 50.1301,
      "objective": 47.57838439941406,
      "ranking_idealized": 0.6000000238418579,
      "ranking_idealized_expo": 0.5041666626930237,
      "ranking_simple": 0.5874999761581421,
      "regularize": 0.12325119227170944,
      "step": 795,
      "wo_beta": 14.180596351623535
    },
    {
      "dpo_loss": 0.5349844694137573,
      "epoch": 2.2673594709494567,
      "grad_norm": 16155.361379646765,
      "learning_rate": 8.436218830716258e-08,
      "logits": -1.5085468292236328,
      "logps": -86.47720336914062,
      "loss": 51.9463,
      "objective": 60.657657623291016,
      "ranking_idealized": 0.5916666388511658,
      "ranking_idealized_expo": 0.5041666626930237,
      "ranking_simple": 0.5916666388511658,
      "regularize": 0.13889312744140625,
      "step": 800,
      "wo_beta": 14.652113914489746
    },
    {
      "epoch": 2.2673594709494567,
      "eval_dpo_loss": 0.6781792044639587,
      "eval_logits": -1.4993284940719604,
      "eval_logps": -92.00647735595703,
      "eval_loss": 179.21780395507812,
      "eval_objective": 175.7035675048828,
      "eval_ranking_idealized": 0.6024844646453857,
      "eval_ranking_idealized_expo": 0.5232919454574585,
      "eval_ranking_simple": 0.5289855003356934,
      "eval_regularize": 0.39864060282707214,
      "eval_runtime": 370.6492,
      "eval_samples_per_second": 15.621,
      "eval_steps_per_second": 1.303,
      "eval_wo_beta": 16.56892204284668,
      "step": 800
    },
    {
      "dpo_loss": 0.5452725291252136,
      "epoch": 2.2815304676428907,
      "grad_norm": 16570.72383235664,
      "learning_rate": 8.1288780455512e-08,
      "logits": -1.5190993547439575,
      "logps": -84.90750122070312,
      "loss": 45.9105,
      "objective": 44.02467346191406,
      "ranking_idealized": 0.5833333134651184,
      "ranking_idealized_expo": 0.5208333134651184,
      "ranking_simple": 0.5666666626930237,
      "regularize": 0.10332147032022476,
      "step": 805,
      "wo_beta": 16.12928581237793
    },
    {
      "dpo_loss": 0.5472940802574158,
      "epoch": 2.295701464336325,
      "grad_norm": 17306.744795453895,
      "learning_rate": 7.826149656671385e-08,
      "logits": -1.6159324645996094,
      "logps": -84.2494888305664,
      "loss": 48.3513,
      "objective": 51.42966079711914,
      "ranking_idealized": 0.6458333134651184,
      "ranking_idealized_expo": 0.5458333492279053,
      "ranking_simple": 0.637499988079071,
      "regularize": 0.1283574104309082,
      "step": 810,
      "wo_beta": 14.84490966796875
    },
    {
      "dpo_loss": 0.5408957004547119,
      "epoch": 2.309872461029759,
      "grad_norm": 17714.10311614495,
      "learning_rate": 7.528116426995604e-08,
      "logits": -1.5414897203445435,
      "logps": -86.03276824951172,
      "loss": 44.4155,
      "objective": 42.15602493286133,
      "ranking_idealized": 0.5791666507720947,
      "ranking_idealized_expo": 0.47083333134651184,
      "ranking_simple": 0.5625,
      "regularize": 0.09716067463159561,
      "step": 815,
      "wo_beta": 15.411224365234375
    },
    {
      "dpo_loss": 0.5434551239013672,
      "epoch": 2.324043457723193,
      "grad_norm": 16657.377741221597,
      "learning_rate": 7.234859835833021e-08,
      "logits": -1.4976943731307983,
      "logps": -85.22306060791016,
      "loss": 44.4464,
      "objective": 42.74842834472656,
      "ranking_idealized": 0.5874999761581421,
      "ranking_idealized_expo": 0.5249999761581421,
      "ranking_simple": 0.574999988079071,
      "regularize": 0.1011769101023674,
      "step": 820,
      "wo_beta": 15.093592643737793
    },
    {
      "dpo_loss": 0.5428169965744019,
      "epoch": 2.3382144544166272,
      "grad_norm": 16447.115430947913,
      "learning_rate": 6.94646005660749e-08,
      "logits": -1.5226491689682007,
      "logps": -84.4798812866211,
      "loss": 45.0833,
      "objective": 45.554691314697266,
      "ranking_idealized": 0.625,
      "ranking_idealized_expo": 0.550000011920929,
      "ranking_simple": 0.612500011920929,
      "regularize": 0.11467791348695755,
      "step": 825,
      "wo_beta": 16.547521591186523
    },
    {
      "dpo_loss": 0.5347627997398376,
      "epoch": 2.3523854511100613,
      "grad_norm": 16335.000287824352,
      "learning_rate": 6.662995934939006e-08,
      "logits": -1.5204423666000366,
      "logps": -86.52505493164062,
      "loss": 49.1483,
      "objective": 49.5906867980957,
      "ranking_idealized": 0.6416666507720947,
      "ranking_idealized_expo": 0.5416666865348816,
      "ranking_simple": 0.6166666746139526,
      "regularize": 0.11676025390625,
      "step": 830,
      "wo_beta": 16.500276565551758
    },
    {
      "dpo_loss": 0.5479462742805481,
      "epoch": 2.3665564478034957,
      "grad_norm": 16959.398846741216,
      "learning_rate": 6.384544967088063e-08,
      "logits": -1.462269902229309,
      "logps": -86.40924835205078,
      "loss": 48.4639,
      "objective": 45.90879821777344,
      "ranking_idealized": 0.6166666746139526,
      "ranking_idealized_expo": 0.5083333253860474,
      "ranking_simple": 0.5958333611488342,
      "regularize": 0.11642123758792877,
      "step": 835,
      "wo_beta": 15.002260208129883
    },
    {
      "dpo_loss": 0.5614480376243591,
      "epoch": 2.3807274444969297,
      "grad_norm": 15851.156516804678,
      "learning_rate": 6.111183278768955e-08,
      "logits": -1.4380650520324707,
      "logps": -86.09837341308594,
      "loss": 40.9965,
      "objective": 37.455379486083984,
      "ranking_idealized": 0.5249999761581421,
      "ranking_idealized_expo": 0.4749999940395355,
      "ranking_simple": 0.5,
      "regularize": 0.0973404198884964,
      "step": 840,
      "wo_beta": 15.861971855163574
    },
    {
      "dpo_loss": 0.5543637871742249,
      "epoch": 2.3948984411903638,
      "grad_norm": 19702.171994535533,
      "learning_rate": 5.842985604337769e-08,
      "logits": -1.4723432064056396,
      "logps": -87.03990173339844,
      "loss": 47.1256,
      "objective": 46.43952941894531,
      "ranking_idealized": 0.5958333611488342,
      "ranking_idealized_expo": 0.5416666865348816,
      "ranking_simple": 0.5833333134651184,
      "regularize": 0.11250331997871399,
      "step": 845,
      "wo_beta": 16.375986099243164
    },
    {
      "dpo_loss": 0.5473025441169739,
      "epoch": 2.409069437883798,
      "grad_norm": 16397.021305258124,
      "learning_rate": 5.5800252663607636e-08,
      "logits": -1.509826898574829,
      "logps": -85.1669692993164,
      "loss": 44.3463,
      "objective": 44.0050048828125,
      "ranking_idealized": 0.5874999761581421,
      "ranking_idealized_expo": 0.49166667461395264,
      "ranking_simple": 0.5583333373069763,
      "regularize": 0.10994389653205872,
      "step": 850,
      "wo_beta": 15.741286277770996
    },
    {
      "epoch": 2.409069437883798,
      "eval_dpo_loss": 0.6782696843147278,
      "eval_logits": -1.4917659759521484,
      "eval_logps": -92.23719787597656,
      "eval_loss": 179.17352294921875,
      "eval_objective": 175.77769470214844,
      "eval_ranking_idealized": 0.6024844646453857,
      "eval_ranking_idealized_expo": 0.5232919454574585,
      "eval_ranking_simple": 0.5284678936004639,
      "eval_regularize": 0.39879217743873596,
      "eval_runtime": 368.8355,
      "eval_samples_per_second": 15.698,
      "eval_steps_per_second": 1.31,
      "eval_wo_beta": 16.56818962097168,
      "step": 850
    },
    {
      "dpo_loss": 0.5434221625328064,
      "epoch": 2.423240434577232,
      "grad_norm": 14761.032913238774,
      "learning_rate": 5.3223741555686873e-08,
      "logits": -1.537110447883606,
      "logps": -84.45201110839844,
      "loss": 43.1862,
      "objective": 41.27051544189453,
      "ranking_idealized": 0.5916666388511658,
      "ranking_idealized_expo": 0.5291666388511658,
      "ranking_simple": 0.5833333134651184,
      "regularize": 0.10142151266336441,
      "step": 855,
      "wo_beta": 15.314756393432617
    },
    {
      "dpo_loss": 0.5529573559761047,
      "epoch": 2.4374114312706663,
      "grad_norm": 16145.709635641595,
      "learning_rate": 5.070102711202606e-08,
      "logits": -1.4745042324066162,
      "logps": -85.98120880126953,
      "loss": 42.9052,
      "objective": 38.47713088989258,
      "ranking_idealized": 0.5874999761581421,
      "ranking_idealized_expo": 0.5249999761581421,
      "ranking_simple": 0.574999988079071,
      "regularize": 0.09888458997011185,
      "step": 860,
      "wo_beta": 16.20383644104004
    },
    {
      "dpo_loss": 0.551279604434967,
      "epoch": 2.4515824279641003,
      "grad_norm": 18278.1439042312,
      "learning_rate": 4.8232799017564967e-08,
      "logits": -1.4951705932617188,
      "logps": -85.17224884033203,
      "loss": 44.7949,
      "objective": 45.912452697753906,
      "ranking_idealized": 0.5791666507720947,
      "ranking_idealized_expo": 0.4958333373069763,
      "ranking_simple": 0.574999988079071,
      "regularize": 0.11339430510997772,
      "step": 865,
      "wo_beta": 16.40469741821289
    },
    {
      "dpo_loss": 0.5488017201423645,
      "epoch": 2.4657534246575343,
      "grad_norm": 15873.838165857398,
      "learning_rate": 4.5819732061219475e-08,
      "logits": -1.5395283699035645,
      "logps": -85.87442779541016,
      "loss": 45.7505,
      "objective": 46.77009963989258,
      "ranking_idealized": 0.6000000238418579,
      "ranking_idealized_expo": 0.5208333134651184,
      "ranking_simple": 0.5833333134651184,
      "regularize": 0.11627896875143051,
      "step": 870,
      "wo_beta": 15.758785247802734
    },
    {
      "dpo_loss": 0.5411531329154968,
      "epoch": 2.4799244213509684,
      "grad_norm": 16103.251582257983,
      "learning_rate": 4.346248595140112e-08,
      "logits": -1.4675084352493286,
      "logps": -85.36338806152344,
      "loss": 46.2974,
      "objective": 47.864070892333984,
      "ranking_idealized": 0.6166666746139526,
      "ranking_idealized_expo": 0.5708333253860474,
      "ranking_simple": 0.6000000238418579,
      "regularize": 0.11755504459142685,
      "step": 875,
      "wo_beta": 17.365110397338867
    },
    {
      "dpo_loss": 0.5485495924949646,
      "epoch": 2.4940954180444024,
      "grad_norm": 15694.189473425999,
      "learning_rate": 4.116170513565942e-08,
      "logits": -1.3954468965530396,
      "logps": -85.28124237060547,
      "loss": 41.7812,
      "objective": 37.13809585571289,
      "ranking_idealized": 0.5625,
      "ranking_idealized_expo": 0.5,
      "ranking_simple": 0.5541666746139526,
      "regularize": 0.0931786298751831,
      "step": 880,
      "wo_beta": 17.850427627563477
    },
    {
      "dpo_loss": 0.5530834794044495,
      "epoch": 2.5082664147378364,
      "grad_norm": 17596.947495882203,
      "learning_rate": 3.8918018624496286e-08,
      "logits": -1.562106728553772,
      "logps": -84.21708679199219,
      "loss": 45.0999,
      "objective": 50.288021087646484,
      "ranking_idealized": 0.5916666388511658,
      "ranking_idealized_expo": 0.5249999761581421,
      "ranking_simple": 0.5874999761581421,
      "regularize": 0.11681138724088669,
      "step": 885,
      "wo_beta": 17.1990909576416
    },
    {
      "dpo_loss": 0.5438919067382812,
      "epoch": 2.5224374114312704,
      "grad_norm": 17590.89973277564,
      "learning_rate": 3.673203981940068e-08,
      "logits": -1.4610990285873413,
      "logps": -83.57906341552734,
      "loss": 45.9865,
      "objective": 43.9241828918457,
      "ranking_idealized": 0.5874999761581421,
      "ranking_idealized_expo": 0.5,
      "ranking_simple": 0.5708333253860474,
      "regularize": 0.11596141010522842,
      "step": 890,
      "wo_beta": 14.2067232131958
    },
    {
      "dpo_loss": 0.5475446581840515,
      "epoch": 2.536608408124705,
      "grad_norm": 16852.292477633127,
      "learning_rate": 3.46043663451511e-08,
      "logits": -1.456311583518982,
      "logps": -85.9644775390625,
      "loss": 40.1267,
      "objective": 40.91215133666992,
      "ranking_idealized": 0.5458333492279053,
      "ranking_idealized_expo": 0.49166667461395264,
      "ranking_simple": 0.5375000238418579,
      "regularize": 0.0973983108997345,
      "step": 895,
      "wo_beta": 16.769834518432617
    },
    {
      "dpo_loss": 0.5658264756202698,
      "epoch": 2.550779404818139,
      "grad_norm": 16973.16020360833,
      "learning_rate": 3.2535579886430715e-08,
      "logits": -1.4089369773864746,
      "logps": -84.34557342529297,
      "loss": 44.3015,
      "objective": 48.17121124267578,
      "ranking_idealized": 0.6166666746139526,
      "ranking_idealized_expo": 0.5249999761581421,
      "ranking_simple": 0.5874999761581421,
      "regularize": 0.10806681215763092,
      "step": 900,
      "wo_beta": 15.534666061401367
    },
    {
      "epoch": 2.550779404818139,
      "eval_dpo_loss": 0.678382933139801,
      "eval_logits": -1.4982877969741821,
      "eval_logps": -92.18975830078125,
      "eval_loss": 179.15904235839844,
      "eval_objective": 175.8240203857422,
      "eval_ranking_idealized": 0.6024844646453857,
      "eval_ranking_idealized_expo": 0.5232919454574585,
      "eval_ranking_simple": 0.5279502868652344,
      "eval_regularize": 0.3989744186401367,
      "eval_runtime": 368.6241,
      "eval_samples_per_second": 15.707,
      "eval_steps_per_second": 1.31,
      "eval_wo_beta": 16.590484619140625,
      "step": 900
    },
    {
      "dpo_loss": 0.5509156584739685,
      "epoch": 2.564950401511573,
      "grad_norm": 14234.779776824053,
      "learning_rate": 3.052624602880063e-08,
      "logits": -1.5034754276275635,
      "logps": -84.19306945800781,
      "loss": 40.4357,
      "objective": 38.76914978027344,
      "ranking_idealized": 0.5708333253860474,
      "ranking_idealized_expo": 0.5,
      "ranking_simple": 0.5458333492279053,
      "regularize": 0.09935871511697769,
      "step": 905,
      "wo_beta": 15.227039337158203
    },
    {
      "dpo_loss": 0.5412671566009521,
      "epoch": 2.579121398205007,
      "grad_norm": 16823.804354079846,
      "learning_rate": 2.8576914104074423e-08,
      "logits": -1.4797313213348389,
      "logps": -87.3152084350586,
      "loss": 41.7421,
      "objective": 45.547401428222656,
      "ranking_idealized": 0.612500011920929,
      "ranking_idealized_expo": 0.4958333373069763,
      "ranking_simple": 0.5874999761581421,
      "regularize": 0.10913080722093582,
      "step": 910,
      "wo_beta": 16.365530014038086
    },
    {
      "dpo_loss": 0.5502139329910278,
      "epoch": 2.593292394898441,
      "grad_norm": 15853.539663724245,
      "learning_rate": 2.668811704013646e-08,
      "logits": -1.5921828746795654,
      "logps": -86.05388641357422,
      "loss": 42.1486,
      "objective": 39.99686050415039,
      "ranking_idealized": 0.6291666626930237,
      "ranking_idealized_expo": 0.512499988079071,
      "ranking_simple": 0.6041666865348816,
      "regularize": 0.103940449655056,
      "step": 915,
      "wo_beta": 15.442140579223633
    },
    {
      "dpo_loss": 0.5391423106193542,
      "epoch": 2.6074633915918755,
      "grad_norm": 16570.329855367927,
      "learning_rate": 2.486037121524448e-08,
      "logits": -1.4353820085525513,
      "logps": -85.45365905761719,
      "loss": 46.2712,
      "objective": 45.818546295166016,
      "ranking_idealized": 0.6041666865348816,
      "ranking_idealized_expo": 0.5249999761581421,
      "ranking_simple": 0.5916666388511658,
      "regularize": 0.10537134110927582,
      "step": 920,
      "wo_beta": 15.817992210388184
    },
    {
      "dpo_loss": 0.5430201888084412,
      "epoch": 2.6216343882853095,
      "grad_norm": 16860.801218719196,
      "learning_rate": 2.3094176316856978e-08,
      "logits": -1.4627550840377808,
      "logps": -85.5040512084961,
      "loss": 41.8789,
      "objective": 39.82048797607422,
      "ranking_idealized": 0.574999988079071,
      "ranking_idealized_expo": 0.4749999940395355,
      "ranking_simple": 0.5458333492279053,
      "regularize": 0.10451699048280716,
      "step": 925,
      "wo_beta": 16.46214485168457
    },
    {
      "dpo_loss": 0.5547136068344116,
      "epoch": 2.6358053849787435,
      "grad_norm": 18548.498992469253,
      "learning_rate": 2.1390015205023898e-08,
      "logits": -1.4610332250595093,
      "logps": -85.3515625,
      "loss": 44.5914,
      "objective": 45.85152053833008,
      "ranking_idealized": 0.5416666865348816,
      "ranking_idealized_expo": 0.44583332538604736,
      "ranking_simple": 0.5166666507720947,
      "regularize": 0.11503276228904724,
      "step": 930,
      "wo_beta": 16.94695281982422
    },
    {
      "dpo_loss": 0.5648698210716248,
      "epoch": 2.6499763816721775,
      "grad_norm": 16823.626472777796,
      "learning_rate": 1.974835378037723e-08,
      "logits": -1.4719030857086182,
      "logps": -84.22066497802734,
      "loss": 42.8827,
      "objective": 47.928733825683594,
      "ranking_idealized": 0.6499999761581421,
      "ranking_idealized_expo": 0.574999988079071,
      "ranking_simple": 0.6208333373069763,
      "regularize": 0.11680851131677628,
      "step": 935,
      "wo_beta": 16.337308883666992
    },
    {
      "dpo_loss": 0.5417830944061279,
      "epoch": 2.6641473783656116,
      "grad_norm": 19648.95461433399,
      "learning_rate": 1.816964085675865e-08,
      "logits": -1.50851309299469,
      "logps": -87.09917449951172,
      "loss": 44.1259,
      "objective": 46.947654724121094,
      "ranking_idealized": 0.6000000238418579,
      "ranking_idealized_expo": 0.5333333611488342,
      "ranking_simple": 0.5874999761581421,
      "regularize": 0.112226702272892,
      "step": 940,
      "wo_beta": 17.42079734802246
    },
    {
      "dpo_loss": 0.5409041047096252,
      "epoch": 2.678318375059046,
      "grad_norm": 17233.04175831224,
      "learning_rate": 1.6654308038518056e-08,
      "logits": -1.544434905052185,
      "logps": -85.57273864746094,
      "loss": 41.238,
      "objective": 48.42988967895508,
      "ranking_idealized": 0.5666666626930237,
      "ranking_idealized_expo": 0.48750001192092896,
      "ranking_simple": 0.5583333373069763,
      "regularize": 0.11696865409612656,
      "step": 945,
      "wo_beta": 15.977194786071777
    },
    {
      "dpo_loss": 0.554226815700531,
      "epoch": 2.69248937175248,
      "grad_norm": 17994.23017373139,
      "learning_rate": 1.520276960251751e-08,
      "logits": -1.4273337125778198,
      "logps": -85.7279281616211,
      "loss": 43.4164,
      "objective": 41.65528106689453,
      "ranking_idealized": 0.637499988079071,
      "ranking_idealized_expo": 0.5375000238418579,
      "ranking_simple": 0.6041666865348816,
      "regularize": 0.1018596738576889,
      "step": 950,
      "wo_beta": 15.610276222229004
    },
    {
      "epoch": 2.69248937175248,
      "eval_dpo_loss": 0.6785179376602173,
      "eval_logits": -1.4967025518417358,
      "eval_logps": -92.20464324951172,
      "eval_loss": 179.2801055908203,
      "eval_objective": 176.04083251953125,
      "eval_ranking_idealized": 0.6024844646453857,
      "eval_ranking_idealized_expo": 0.5232919454574585,
      "eval_ranking_simple": 0.5274327397346497,
      "eval_regularize": 0.39928528666496277,
      "eval_runtime": 370.2035,
      "eval_samples_per_second": 15.64,
      "eval_steps_per_second": 1.305,
      "eval_wo_beta": 16.589099884033203,
      "step": 950
    },
    {
      "dpo_loss": 0.5416913628578186,
      "epoch": 2.706660368445914,
      "grad_norm": 16913.4446377394,
      "learning_rate": 1.3815422384871878e-08,
      "logits": -1.5223019123077393,
      "logps": -85.46512603759766,
      "loss": 41.9154,
      "objective": 39.07767105102539,
      "ranking_idealized": 0.5874999761581421,
      "ranking_idealized_expo": 0.4749999940395355,
      "ranking_simple": 0.5583333373069763,
      "regularize": 0.10648568719625473,
      "step": 955,
      "wo_beta": 14.056703567504883
    },
    {
      "dpo_loss": 0.5538465976715088,
      "epoch": 2.720831365139348,
      "grad_norm": 16463.281607975867,
      "learning_rate": 1.2492645672457836e-08,
      "logits": -1.4985733032226562,
      "logps": -85.17173767089844,
      "loss": 42.6366,
      "objective": 43.174072265625,
      "ranking_idealized": 0.5791666507720947,
      "ranking_idealized_expo": 0.49166667461395264,
      "ranking_simple": 0.5708333253860474,
      "regularize": 0.10360194742679596,
      "step": 960,
      "wo_beta": 16.368473052978516
    },
    {
      "dpo_loss": 0.537990391254425,
      "epoch": 2.735002361832782,
      "grad_norm": 17961.612959662347,
      "learning_rate": 1.1234801099220786e-08,
      "logits": -1.509239912033081,
      "logps": -85.44644165039062,
      "loss": 39.0678,
      "objective": 39.20804977416992,
      "ranking_idealized": 0.6416666507720947,
      "ranking_idealized_expo": 0.5666666626930237,
      "ranking_simple": 0.6291666626930237,
      "regularize": 0.10068784654140472,
      "step": 965,
      "wo_beta": 15.433809280395508
    },
    {
      "dpo_loss": 0.5542954802513123,
      "epoch": 2.7491733585262166,
      "grad_norm": 15088.900869572226,
      "learning_rate": 1.004223254730749e-08,
      "logits": -1.519822597503662,
      "logps": -86.41224670410156,
      "loss": 44.4091,
      "objective": 43.453468322753906,
      "ranking_idealized": 0.6208333373069763,
      "ranking_idealized_expo": 0.5333333611488342,
      "ranking_simple": 0.5833333134651184,
      "regularize": 0.11093102395534515,
      "step": 970,
      "wo_beta": 15.656749725341797
    },
    {
      "dpo_loss": 0.5459226965904236,
      "epoch": 2.7633443552196506,
      "grad_norm": 17863.309850864207,
      "learning_rate": 8.915266053052373e-09,
      "logits": -1.4553431272506714,
      "logps": -84.71710968017578,
      "loss": 44.5817,
      "objective": 54.85538101196289,
      "ranking_idealized": 0.6333333253860474,
      "ranking_idealized_expo": 0.5333333611488342,
      "ranking_simple": 0.6083333492279053,
      "regularize": 0.13891781866550446,
      "step": 975,
      "wo_beta": 16.4742374420166
    },
    {
      "dpo_loss": 0.5517151355743408,
      "epoch": 2.7775153519130846,
      "grad_norm": 17532.917025188563,
      "learning_rate": 7.85420971784223e-09,
      "logits": -1.5741106271743774,
      "logps": -84.43816375732422,
      "loss": 49.1041,
      "objective": 50.19886016845703,
      "ranking_idealized": 0.5916666388511658,
      "ranking_idealized_expo": 0.5,
      "ranking_simple": 0.550000011920929,
      "regularize": 0.12199045717716217,
      "step": 980,
      "wo_beta": 14.642070770263672
    },
    {
      "dpo_loss": 0.5505563020706177,
      "epoch": 2.7916863486065187,
      "grad_norm": 15695.994208260572,
      "learning_rate": 6.859353623884567e-09,
      "logits": -1.4095691442489624,
      "logps": -82.93616485595703,
      "loss": 43.0667,
      "objective": 50.173011779785156,
      "ranking_idealized": 0.5791666507720947,
      "ranking_idealized_expo": 0.48750001192092896,
      "ranking_simple": 0.5666666626930237,
      "regularize": 0.1220245286822319,
      "step": 985,
      "wo_beta": 14.67066764831543
    },
    {
      "dpo_loss": 0.5449987649917603,
      "epoch": 2.8058573452999527,
      "grad_norm": 15965.60549784291,
      "learning_rate": 5.930969754901843e-09,
      "logits": -1.484297275543213,
      "logps": -86.44190979003906,
      "loss": 44.9483,
      "objective": 41.374794006347656,
      "ranking_idealized": 0.5666666626930237,
      "ranking_idealized_expo": 0.4749999940395355,
      "ranking_simple": 0.5625,
      "regularize": 0.10469052940607071,
      "step": 990,
      "wo_beta": 16.099658966064453
    },
    {
      "dpo_loss": 0.5509870648384094,
      "epoch": 2.820028341993387,
      "grad_norm": 15636.732404031713,
      "learning_rate": 5.069311921774039e-09,
      "logits": -1.531805396080017,
      "logps": -84.8103256225586,
      "loss": 43.9212,
      "objective": 41.92687225341797,
      "ranking_idealized": 0.6208333373069763,
      "ranking_idealized_expo": 0.5458333492279053,
      "ranking_simple": 0.6083333492279053,
      "regularize": 0.11357491463422775,
      "step": 995,
      "wo_beta": 16.878124237060547
    },
    {
      "dpo_loss": 0.544861376285553,
      "epoch": 2.8341993386868207,
      "grad_norm": 15284.06322793647,
      "learning_rate": 4.274615693149075e-09,
      "logits": -1.4928451776504517,
      "logps": -83.65907287597656,
      "loss": 43.6009,
      "objective": 46.88129425048828,
      "ranking_idealized": 0.6166666746139526,
      "ranking_idealized_expo": 0.5083333253860474,
      "ranking_simple": 0.6000000238418579,
      "regularize": 0.11148179322481155,
      "step": 1000,
      "wo_beta": 15.039312362670898
    },
    {
      "epoch": 2.8341993386868207,
      "eval_dpo_loss": 0.6785008311271667,
      "eval_logits": -1.4978208541870117,
      "eval_logps": -92.27050018310547,
      "eval_loss": 179.2790985107422,
      "eval_objective": 175.9962921142578,
      "eval_ranking_idealized": 0.6024844646453857,
      "eval_ranking_idealized_expo": 0.5232919454574585,
      "eval_ranking_simple": 0.5279502868652344,
      "eval_regularize": 0.39921271800994873,
      "eval_runtime": 373.1717,
      "eval_samples_per_second": 15.516,
      "eval_steps_per_second": 1.294,
      "eval_wo_beta": 16.587968826293945,
      "step": 1000
    },
    {
      "dpo_loss": 0.5515304803848267,
      "epoch": 2.848370335380255,
      "grad_norm": 16385.652650772114,
      "learning_rate": 3.547098331040915e-09,
      "logits": -1.482871413230896,
      "logps": -84.16107940673828,
      "loss": 41.5394,
      "objective": 43.54771041870117,
      "ranking_idealized": 0.6000000238418579,
      "ranking_idealized_expo": 0.512499988079071,
      "ranking_simple": 0.5833333134651184,
      "regularize": 0.11037115752696991,
      "step": 1005,
      "wo_beta": 16.216983795166016
    },
    {
      "dpo_loss": 0.5438867211341858,
      "epoch": 2.862541332073689,
      "grad_norm": 16774.77560364275,
      "learning_rate": 2.886958731432132e-09,
      "logits": -1.542305827140808,
      "logps": -84.64833068847656,
      "loss": 43.5186,
      "objective": 44.43704605102539,
      "ranking_idealized": 0.612500011920929,
      "ranking_idealized_expo": 0.5249999761581421,
      "ranking_simple": 0.5958333611488342,
      "regularize": 0.10451284050941467,
      "step": 1010,
      "wo_beta": 15.962403297424316
    },
    {
      "dpo_loss": 0.5501060485839844,
      "epoch": 2.8767123287671232,
      "grad_norm": 17522.630643855955,
      "learning_rate": 2.294377369897793e-09,
      "logits": -1.4613019227981567,
      "logps": -83.6242446899414,
      "loss": 42.9096,
      "objective": 43.5313720703125,
      "ranking_idealized": 0.6541666388511658,
      "ranking_idealized_expo": 0.550000011920929,
      "ranking_simple": 0.6208333373069763,
      "regularize": 0.1068674772977829,
      "step": 1015,
      "wo_beta": 15.568625450134277
    },
    {
      "dpo_loss": 0.549178957939148,
      "epoch": 2.8908833254605573,
      "grad_norm": 16355.818495658159,
      "learning_rate": 1.769516252265235e-09,
      "logits": -1.4256434440612793,
      "logps": -85.48424530029297,
      "loss": 41.2726,
      "objective": 41.4518928527832,
      "ranking_idealized": 0.5833333134651184,
      "ranking_idealized_expo": 0.5166666507720947,
      "ranking_simple": 0.5708333253860474,
      "regularize": 0.0968986377120018,
      "step": 1020,
      "wo_beta": 18.603649139404297
    },
    {
      "dpo_loss": 0.5602646470069885,
      "epoch": 2.9050543221539913,
      "grad_norm": 15838.312706923856,
      "learning_rate": 1.3125188703233814e-09,
      "logits": -1.5212275981903076,
      "logps": -85.22294616699219,
      "loss": 43.2553,
      "objective": 43.40156555175781,
      "ranking_idealized": 0.6208333373069763,
      "ranking_idealized_expo": 0.5083333253860474,
      "ranking_simple": 0.6000000238418579,
      "regularize": 0.1083984375,
      "step": 1025,
      "wo_beta": 15.021879196166992
    },
    {
      "dpo_loss": 0.5516722798347473,
      "epoch": 2.9192253188474258,
      "grad_norm": 16291.39704850031,
      "learning_rate": 9.235101625932884e-10,
      "logits": -1.5682528018951416,
      "logps": -85.36617279052734,
      "loss": 43.7786,
      "objective": 39.36371612548828,
      "ranking_idealized": 0.5416666865348816,
      "ranking_idealized_expo": 0.4791666567325592,
      "ranking_simple": 0.5375000238418579,
      "regularize": 0.0955829992890358,
      "step": 1030,
      "wo_beta": 16.441747665405273
    },
    {
      "dpo_loss": 0.5399314761161804,
      "epoch": 2.9333963155408598,
      "grad_norm": 16748.657990232314,
      "learning_rate": 6.025964801714411e-10,
      "logits": -1.5260014533996582,
      "logps": -86.03439331054688,
      "loss": 41.3554,
      "objective": 38.64834976196289,
      "ranking_idealized": 0.625,
      "ranking_idealized_expo": 0.5,
      "ranking_simple": 0.5916666388511658,
      "regularize": 0.0944010317325592,
      "step": 1035,
      "wo_beta": 14.993240356445312
    },
    {
      "dpo_loss": 0.5549695491790771,
      "epoch": 2.947567312234294,
      "grad_norm": 15029.578861688902,
      "learning_rate": 3.498655576543441e-10,
      "logits": -1.5111292600631714,
      "logps": -85.76802062988281,
      "loss": 38.8328,
      "objective": 38.22753143310547,
      "ranking_idealized": 0.5708333253860474,
      "ranking_idealized_expo": 0.4833333194255829,
      "ranking_simple": 0.550000011920929,
      "regularize": 0.095832958817482,
      "step": 1040,
      "wo_beta": 16.434364318847656
    },
    {
      "dpo_loss": 0.5555641651153564,
      "epoch": 2.961738308927728,
      "grad_norm": 17992.01977336534,
      "learning_rate": 1.6538648915270793e-10,
      "logits": -1.481310248374939,
      "logps": -87.40520477294922,
      "loss": 38.5959,
      "objective": 39.465171813964844,
      "ranking_idealized": 0.6208333373069763,
      "ranking_idealized_expo": 0.5666666626930237,
      "ranking_simple": 0.6166666746139526,
      "regularize": 0.10020165145397186,
      "step": 1045,
      "wo_beta": 18.04179573059082
    },
    {
      "dpo_loss": 0.5507573485374451,
      "epoch": 2.975909305621162,
      "grad_norm": 17353.770623615765,
      "learning_rate": 4.920970940180957e-11,
      "logits": -1.522560715675354,
      "logps": -83.40792083740234,
      "loss": 47.7054,
      "objective": 48.42831802368164,
      "ranking_idealized": 0.6541666388511658,
      "ranking_idealized_expo": 0.5541666746139526,
      "ranking_simple": 0.6166666746139526,
      "regularize": 0.121465764939785,
      "step": 1050,
      "wo_beta": 15.164950370788574
    },
    {
      "epoch": 2.975909305621162,
      "eval_dpo_loss": 0.6784854531288147,
      "eval_logits": -1.4974991083145142,
      "eval_logps": -92.26132202148438,
      "eval_loss": 179.26217651367188,
      "eval_objective": 175.9752197265625,
      "eval_ranking_idealized": 0.6024844646453857,
      "eval_ranking_idealized_expo": 0.5232919454574585,
      "eval_ranking_simple": 0.5279502868652344,
      "eval_regularize": 0.39915931224823,
      "eval_runtime": 370.7869,
      "eval_samples_per_second": 15.615,
      "eval_steps_per_second": 1.303,
      "eval_wo_beta": 16.585628509521484,
      "step": 1050
    },
    {
      "dpo_loss": 0.5567707419395447,
      "epoch": 2.9900803023145963,
      "grad_norm": 17458.927279367348,
      "learning_rate": 1.3669799732163311e-12,
      "logits": -1.4982311725616455,
      "logps": -84.29004669189453,
      "loss": 47.3347,
      "objective": 49.85600662231445,
      "ranking_idealized": 0.6083333492279053,
      "ranking_idealized_expo": 0.550000011920929,
      "ranking_simple": 0.6000000238418579,
      "regularize": 0.12724058330059052,
      "step": 1055,
      "wo_beta": 15.180956840515137
    },
    {
      "epoch": 2.992914501653283,
      "step": 1056,
      "total_flos": 0.0,
      "train_loss": 87.93023242011215,
      "train_runtime": 38512.4809,
      "train_samples_per_second": 3.957,
      "train_steps_per_second": 0.027
    }
  ],
  "logging_steps": 5,
  "max_steps": 1056,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 3,
  "save_steps": 500,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": false,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 4,
  "trial_name": null,
  "trial_params": null
}