| { | |
| "best_metric": null, | |
| "best_model_checkpoint": null, | |
| "epoch": 4.988190836088805, | |
| "eval_steps": 50, | |
| "global_step": 880, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "dpo_loss": 0.6931471824645996, | |
| "epoch": 0.005668398677373642, | |
| "grad_norm": 13.413595204669061, | |
| "learning_rate": 5.681818181818182e-08, | |
| "logits": -1.3147305250167847, | |
| "logps": -88.0877456665039, | |
| "loss": 0.4113, | |
| "objective": 0.41588976979255676, | |
| "ranking_idealized": 0.6875, | |
| "ranking_idealized_expo": 0.5208333134651184, | |
| "ranking_simple": 0.5208333134651184, | |
| "regularize": 0.41588976979255676, | |
| "step": 1 | |
| }, | |
| { | |
| "dpo_loss": 0.6931002736091614, | |
| "epoch": 0.02834199338686821, | |
| "grad_norm": 13.427332141277578, | |
| "learning_rate": 2.840909090909091e-07, | |
| "logits": -1.3680692911148071, | |
| "logps": -84.26158905029297, | |
| "loss": 0.4129, | |
| "objective": 0.3754810094833374, | |
| "ranking_idealized": 0.6510416865348816, | |
| "ranking_idealized_expo": 0.5572916865348816, | |
| "ranking_simple": 0.546875, | |
| "regularize": 0.3754810094833374, | |
| "step": 5 | |
| }, | |
| { | |
| "dpo_loss": 0.6900920867919922, | |
| "epoch": 0.05668398677373642, | |
| "grad_norm": 12.67240246603534, | |
| "learning_rate": 5.681818181818182e-07, | |
| "logits": -1.4469478130340576, | |
| "logps": -82.44185638427734, | |
| "loss": 0.4149, | |
| "objective": 0.43780091404914856, | |
| "ranking_idealized": 0.675000011920929, | |
| "ranking_idealized_expo": 0.512499988079071, | |
| "ranking_simple": 0.5166666507720947, | |
| "regularize": 0.43780091404914856, | |
| "step": 10 | |
| }, | |
| { | |
| "dpo_loss": 0.6891883015632629, | |
| "epoch": 0.08502598016060463, | |
| "grad_norm": 12.900443318480342, | |
| "learning_rate": 8.522727272727273e-07, | |
| "logits": -1.4273536205291748, | |
| "logps": -81.69231414794922, | |
| "loss": 0.419, | |
| "objective": 0.40471941232681274, | |
| "ranking_idealized": 0.6541666388511658, | |
| "ranking_idealized_expo": 0.5041666626930237, | |
| "ranking_simple": 0.512499988079071, | |
| "regularize": 0.40471941232681274, | |
| "step": 15 | |
| }, | |
| { | |
| "dpo_loss": 0.6918003559112549, | |
| "epoch": 0.11336797354747284, | |
| "grad_norm": 14.313913436693964, | |
| "learning_rate": 1.1363636363636364e-06, | |
| "logits": -1.437472939491272, | |
| "logps": -82.81884765625, | |
| "loss": 0.4037, | |
| "objective": 0.403365820646286, | |
| "ranking_idealized": 0.6791666746139526, | |
| "ranking_idealized_expo": 0.5375000238418579, | |
| "ranking_simple": 0.5333333611488342, | |
| "regularize": 0.403365820646286, | |
| "step": 20 | |
| }, | |
| { | |
| "dpo_loss": 0.6768646836280823, | |
| "epoch": 0.14170996693434104, | |
| "grad_norm": 13.58368371280586, | |
| "learning_rate": 1.4204545454545458e-06, | |
| "logits": -1.5096564292907715, | |
| "logps": -82.65319061279297, | |
| "loss": 0.3931, | |
| "objective": 0.409546822309494, | |
| "ranking_idealized": 0.675000011920929, | |
| "ranking_idealized_expo": 0.4958333373069763, | |
| "ranking_simple": 0.5041666626930237, | |
| "regularize": 0.409546822309494, | |
| "step": 25 | |
| }, | |
| { | |
| "dpo_loss": 0.6711666584014893, | |
| "epoch": 0.17005196032120926, | |
| "grad_norm": 13.105526207742551, | |
| "learning_rate": 1.7045454545454546e-06, | |
| "logits": -1.5050361156463623, | |
| "logps": -83.46080780029297, | |
| "loss": 0.3841, | |
| "objective": 0.38422220945358276, | |
| "ranking_idealized": 0.6875, | |
| "ranking_idealized_expo": 0.5416666865348816, | |
| "ranking_simple": 0.5666666626930237, | |
| "regularize": 0.38422220945358276, | |
| "step": 30 | |
| }, | |
| { | |
| "dpo_loss": 0.6604105234146118, | |
| "epoch": 0.19839395370807747, | |
| "grad_norm": 14.531223489181945, | |
| "learning_rate": 1.9886363636363638e-06, | |
| "logits": -1.5017287731170654, | |
| "logps": -84.02853393554688, | |
| "loss": 0.3722, | |
| "objective": 0.35262614488601685, | |
| "ranking_idealized": 0.6583333611488342, | |
| "ranking_idealized_expo": 0.5041666626930237, | |
| "ranking_simple": 0.5333333611488342, | |
| "regularize": 0.35262614488601685, | |
| "step": 35 | |
| }, | |
| { | |
| "dpo_loss": 0.6500855684280396, | |
| "epoch": 0.22673594709494568, | |
| "grad_norm": 14.371511230761167, | |
| "learning_rate": 2.2727272727272728e-06, | |
| "logits": -1.5274395942687988, | |
| "logps": -84.69414520263672, | |
| "loss": 0.379, | |
| "objective": 0.3910427689552307, | |
| "ranking_idealized": 0.6625000238418579, | |
| "ranking_idealized_expo": 0.5249999761581421, | |
| "ranking_simple": 0.5583333373069763, | |
| "regularize": 0.3910427689552307, | |
| "step": 40 | |
| }, | |
| { | |
| "dpo_loss": 0.639613151550293, | |
| "epoch": 0.25507794048181387, | |
| "grad_norm": 17.129866291690814, | |
| "learning_rate": 2.556818181818182e-06, | |
| "logits": -1.6906952857971191, | |
| "logps": -86.7696304321289, | |
| "loss": 0.3764, | |
| "objective": 0.4166857898235321, | |
| "ranking_idealized": 0.7208333611488342, | |
| "ranking_idealized_expo": 0.5375000238418579, | |
| "ranking_simple": 0.5458333492279053, | |
| "regularize": 0.4166857898235321, | |
| "step": 45 | |
| }, | |
| { | |
| "dpo_loss": 0.637481153011322, | |
| "epoch": 0.2834199338686821, | |
| "grad_norm": 12.595130895032831, | |
| "learning_rate": 2.8409090909090916e-06, | |
| "logits": -1.604835033416748, | |
| "logps": -87.7275161743164, | |
| "loss": 0.3566, | |
| "objective": 0.3444138169288635, | |
| "ranking_idealized": 0.637499988079071, | |
| "ranking_idealized_expo": 0.42500001192092896, | |
| "ranking_simple": 0.5458333492279053, | |
| "regularize": 0.3444138169288635, | |
| "step": 50 | |
| }, | |
| { | |
| "epoch": 0.2834199338686821, | |
| "eval_dpo_loss": 0.6897606253623962, | |
| "eval_logits": -1.6203083992004395, | |
| "eval_logps": -96.4831314086914, | |
| "eval_loss": 0.413291871547699, | |
| "eval_objective": 0.4237224757671356, | |
| "eval_ranking_idealized": 0.6570248007774353, | |
| "eval_ranking_idealized_expo": 0.5113636255264282, | |
| "eval_ranking_simple": 0.5165289044380188, | |
| "eval_regularize": 0.4237224757671356, | |
| "eval_runtime": 259.6612, | |
| "eval_samples_per_second": 22.298, | |
| "eval_steps_per_second": 0.932, | |
| "step": 50 | |
| }, | |
| { | |
| "dpo_loss": 0.6459915041923523, | |
| "epoch": 0.3117619272555503, | |
| "grad_norm": 12.272365561333357, | |
| "learning_rate": 3.125e-06, | |
| "logits": -1.6675435304641724, | |
| "logps": -91.76095581054688, | |
| "loss": 0.3508, | |
| "objective": 0.359805166721344, | |
| "ranking_idealized": 0.7124999761581421, | |
| "ranking_idealized_expo": 0.5416666865348816, | |
| "ranking_simple": 0.5625, | |
| "regularize": 0.359805166721344, | |
| "step": 55 | |
| }, | |
| { | |
| "dpo_loss": 0.6238431334495544, | |
| "epoch": 0.3401039206424185, | |
| "grad_norm": 13.401317383285344, | |
| "learning_rate": 3.409090909090909e-06, | |
| "logits": -1.6435742378234863, | |
| "logps": -88.34529113769531, | |
| "loss": 0.3357, | |
| "objective": 0.3332988917827606, | |
| "ranking_idealized": 0.7041666507720947, | |
| "ranking_idealized_expo": 0.5166666507720947, | |
| "ranking_simple": 0.5791666507720947, | |
| "regularize": 0.3332988917827606, | |
| "step": 60 | |
| }, | |
| { | |
| "dpo_loss": 0.6304014325141907, | |
| "epoch": 0.3684459140292867, | |
| "grad_norm": 12.713715258382189, | |
| "learning_rate": 3.6931818181818186e-06, | |
| "logits": -1.5630245208740234, | |
| "logps": -86.62226867675781, | |
| "loss": 0.3325, | |
| "objective": 0.3380378782749176, | |
| "ranking_idealized": 0.6291666626930237, | |
| "ranking_idealized_expo": 0.5083333253860474, | |
| "ranking_simple": 0.5458333492279053, | |
| "regularize": 0.3380378782749176, | |
| "step": 65 | |
| }, | |
| { | |
| "dpo_loss": 0.627398669719696, | |
| "epoch": 0.39678790741615494, | |
| "grad_norm": 12.34517477378158, | |
| "learning_rate": 3.9772727272727275e-06, | |
| "logits": -1.519757628440857, | |
| "logps": -88.53465270996094, | |
| "loss": 0.3261, | |
| "objective": 0.3733421266078949, | |
| "ranking_idealized": 0.6916666626930237, | |
| "ranking_idealized_expo": 0.5166666507720947, | |
| "ranking_simple": 0.5833333134651184, | |
| "regularize": 0.3733421266078949, | |
| "step": 70 | |
| }, | |
| { | |
| "dpo_loss": 0.618600070476532, | |
| "epoch": 0.42512990080302315, | |
| "grad_norm": 13.394680047264217, | |
| "learning_rate": 4.2613636363636365e-06, | |
| "logits": -1.444022297859192, | |
| "logps": -86.64772033691406, | |
| "loss": 0.3211, | |
| "objective": 0.3317987322807312, | |
| "ranking_idealized": 0.6791666746139526, | |
| "ranking_idealized_expo": 0.5458333492279053, | |
| "ranking_simple": 0.612500011920929, | |
| "regularize": 0.3317987322807312, | |
| "step": 75 | |
| }, | |
| { | |
| "dpo_loss": 0.6121571660041809, | |
| "epoch": 0.45347189418989137, | |
| "grad_norm": 12.637476273331325, | |
| "learning_rate": 4.5454545454545455e-06, | |
| "logits": -1.5524269342422485, | |
| "logps": -82.45478057861328, | |
| "loss": 0.3164, | |
| "objective": 0.33297327160835266, | |
| "ranking_idealized": 0.6666666865348816, | |
| "ranking_idealized_expo": 0.5166666507720947, | |
| "ranking_simple": 0.5833333134651184, | |
| "regularize": 0.33297327160835266, | |
| "step": 80 | |
| }, | |
| { | |
| "dpo_loss": 0.5934053659439087, | |
| "epoch": 0.4818138875767596, | |
| "grad_norm": 11.681739450816789, | |
| "learning_rate": 4.829545454545455e-06, | |
| "logits": -1.5023764371871948, | |
| "logps": -83.81694030761719, | |
| "loss": 0.3085, | |
| "objective": 0.2888755202293396, | |
| "ranking_idealized": 0.6541666388511658, | |
| "ranking_idealized_expo": 0.5249999761581421, | |
| "ranking_simple": 0.6166666746139526, | |
| "regularize": 0.2888755202293396, | |
| "step": 85 | |
| }, | |
| { | |
| "dpo_loss": 0.6050879955291748, | |
| "epoch": 0.5101558809636277, | |
| "grad_norm": 11.586500005637587, | |
| "learning_rate": 4.999921328558333e-06, | |
| "logits": -1.246580958366394, | |
| "logps": -88.33258819580078, | |
| "loss": 0.3085, | |
| "objective": 0.3082594871520996, | |
| "ranking_idealized": 0.6291666626930237, | |
| "ranking_idealized_expo": 0.4791666567325592, | |
| "ranking_simple": 0.5458333492279053, | |
| "regularize": 0.3082594871520996, | |
| "step": 90 | |
| }, | |
| { | |
| "dpo_loss": 0.5785138010978699, | |
| "epoch": 0.538497874350496, | |
| "grad_norm": 11.1585869550919, | |
| "learning_rate": 4.999036331701828e-06, | |
| "logits": -1.2436394691467285, | |
| "logps": -85.56443786621094, | |
| "loss": 0.3059, | |
| "objective": 0.293775349855423, | |
| "ranking_idealized": 0.6833333373069763, | |
| "ranking_idealized_expo": 0.5458333492279053, | |
| "ranking_simple": 0.6208333373069763, | |
| "regularize": 0.293775349855423, | |
| "step": 95 | |
| }, | |
| { | |
| "dpo_loss": 0.5692493319511414, | |
| "epoch": 0.5668398677373642, | |
| "grad_norm": 10.57766394132754, | |
| "learning_rate": 4.997168347957521e-06, | |
| "logits": -1.3053488731384277, | |
| "logps": -83.31777954101562, | |
| "loss": 0.3027, | |
| "objective": 0.31831109523773193, | |
| "ranking_idealized": 0.6708333492279053, | |
| "ranking_idealized_expo": 0.5416666865348816, | |
| "ranking_simple": 0.6083333492279053, | |
| "regularize": 0.31831109523773193, | |
| "step": 100 | |
| }, | |
| { | |
| "epoch": 0.5668398677373642, | |
| "eval_dpo_loss": 0.686188817024231, | |
| "eval_logits": -1.3063371181488037, | |
| "eval_logps": -88.41004180908203, | |
| "eval_loss": 0.41417956352233887, | |
| "eval_objective": 0.41506800055503845, | |
| "eval_ranking_idealized": 0.6570248007774353, | |
| "eval_ranking_idealized_expo": 0.5113636255264282, | |
| "eval_ranking_simple": 0.5216942429542542, | |
| "eval_regularize": 0.41506800055503845, | |
| "eval_runtime": 259.1694, | |
| "eval_samples_per_second": 22.341, | |
| "eval_steps_per_second": 0.934, | |
| "step": 100 | |
| }, | |
| { | |
| "dpo_loss": 0.5781983733177185, | |
| "epoch": 0.5951818611242324, | |
| "grad_norm": 10.454612496016592, | |
| "learning_rate": 4.994318112090048e-06, | |
| "logits": -1.1954314708709717, | |
| "logps": -83.58454132080078, | |
| "loss": 0.3006, | |
| "objective": 0.31248193979263306, | |
| "ranking_idealized": 0.7083333134651184, | |
| "ranking_idealized_expo": 0.5375000238418579, | |
| "ranking_simple": 0.637499988079071, | |
| "regularize": 0.31248193979263306, | |
| "step": 105 | |
| }, | |
| { | |
| "dpo_loss": 0.5739426016807556, | |
| "epoch": 0.6235238545111006, | |
| "grad_norm": 10.705962678396897, | |
| "learning_rate": 4.990486745229364e-06, | |
| "logits": -1.4220352172851562, | |
| "logps": -82.55160522460938, | |
| "loss": 0.3033, | |
| "objective": 0.31312334537506104, | |
| "ranking_idealized": 0.6541666388511658, | |
| "ranking_idealized_expo": 0.4583333432674408, | |
| "ranking_simple": 0.6000000238418579, | |
| "regularize": 0.31312334537506104, | |
| "step": 110 | |
| }, | |
| { | |
| "dpo_loss": 0.5884331464767456, | |
| "epoch": 0.6518658478979689, | |
| "grad_norm": 10.243480321598604, | |
| "learning_rate": 4.985675754429744e-06, | |
| "logits": -1.5664387941360474, | |
| "logps": -80.2437515258789, | |
| "loss": 0.2914, | |
| "objective": 0.29273518919944763, | |
| "ranking_idealized": 0.6583333611488342, | |
| "ranking_idealized_expo": 0.4833333194255829, | |
| "ranking_simple": 0.5708333253860474, | |
| "regularize": 0.29273518919944763, | |
| "step": 115 | |
| }, | |
| { | |
| "dpo_loss": 0.5871846079826355, | |
| "epoch": 0.680207841284837, | |
| "grad_norm": 9.272918607237454, | |
| "learning_rate": 4.9798870320769884e-06, | |
| "logits": -1.5533816814422607, | |
| "logps": -77.25326538085938, | |
| "loss": 0.2962, | |
| "objective": 0.27850577235221863, | |
| "ranking_idealized": 0.6583333611488342, | |
| "ranking_idealized_expo": 0.5333333611488342, | |
| "ranking_simple": 0.6166666746139526, | |
| "regularize": 0.27850577235221863, | |
| "step": 120 | |
| }, | |
| { | |
| "dpo_loss": 0.5849189162254333, | |
| "epoch": 0.7085498346717053, | |
| "grad_norm": 9.087139481090837, | |
| "learning_rate": 4.973122855144066e-06, | |
| "logits": -1.4378304481506348, | |
| "logps": -76.45697784423828, | |
| "loss": 0.286, | |
| "objective": 0.27717408537864685, | |
| "ranking_idealized": 0.699999988079071, | |
| "ranking_idealized_expo": 0.550000011920929, | |
| "ranking_simple": 0.6541666388511658, | |
| "regularize": 0.27717408537864685, | |
| "step": 125 | |
| }, | |
| { | |
| "dpo_loss": 0.5918333530426025, | |
| "epoch": 0.7368918280585735, | |
| "grad_norm": 9.798401176649499, | |
| "learning_rate": 4.965385884295467e-06, | |
| "logits": -1.5077089071273804, | |
| "logps": -75.78624725341797, | |
| "loss": 0.2914, | |
| "objective": 0.29082292318344116, | |
| "ranking_idealized": 0.637499988079071, | |
| "ranking_idealized_expo": 0.4541666805744171, | |
| "ranking_simple": 0.5625, | |
| "regularize": 0.29082292318344116, | |
| "step": 130 | |
| }, | |
| { | |
| "dpo_loss": 0.5844586491584778, | |
| "epoch": 0.7652338214454416, | |
| "grad_norm": 9.388728292193363, | |
| "learning_rate": 4.956679162840646e-06, | |
| "logits": -1.4935728311538696, | |
| "logps": -77.08039093017578, | |
| "loss": 0.2751, | |
| "objective": 0.27092453837394714, | |
| "ranking_idealized": 0.6416666507720947, | |
| "ranking_idealized_expo": 0.5333333611488342, | |
| "ranking_simple": 0.5708333253860474, | |
| "regularize": 0.27092453837394714, | |
| "step": 135 | |
| }, | |
| { | |
| "dpo_loss": 0.5776726007461548, | |
| "epoch": 0.7935758148323099, | |
| "grad_norm": 9.368061718815978, | |
| "learning_rate": 4.947006115536947e-06, | |
| "logits": -1.344637393951416, | |
| "logps": -79.50406646728516, | |
| "loss": 0.278, | |
| "objective": 0.272522896528244, | |
| "ranking_idealized": 0.6291666626930237, | |
| "ranking_idealized_expo": 0.5083333253860474, | |
| "ranking_simple": 0.5708333253860474, | |
| "regularize": 0.272522896528244, | |
| "step": 140 | |
| }, | |
| { | |
| "dpo_loss": 0.583095908164978, | |
| "epoch": 0.821917808219178, | |
| "grad_norm": 8.887316448589162, | |
| "learning_rate": 4.9363705472424825e-06, | |
| "logits": -1.260974645614624, | |
| "logps": -80.34443664550781, | |
| "loss": 0.2702, | |
| "objective": 0.27244552969932556, | |
| "ranking_idealized": 0.6458333134651184, | |
| "ranking_idealized_expo": 0.4833333194255829, | |
| "ranking_simple": 0.5791666507720947, | |
| "regularize": 0.27244552969932556, | |
| "step": 145 | |
| }, | |
| { | |
| "dpo_loss": 0.5877144932746887, | |
| "epoch": 0.8502598016060463, | |
| "grad_norm": 8.678793910307522, | |
| "learning_rate": 4.924776641419513e-06, | |
| "logits": -1.168263554573059, | |
| "logps": -79.24138641357422, | |
| "loss": 0.2706, | |
| "objective": 0.26840564608573914, | |
| "ranking_idealized": 0.6916666626930237, | |
| "ranking_idealized_expo": 0.5416666865348816, | |
| "ranking_simple": 0.6625000238418579, | |
| "regularize": 0.26840564608573914, | |
| "step": 150 | |
| }, | |
| { | |
| "epoch": 0.8502598016060463, | |
| "eval_dpo_loss": 0.6856931447982788, | |
| "eval_logits": -1.1981431245803833, | |
| "eval_logps": -87.367431640625, | |
| "eval_loss": 0.42615318298339844, | |
| "eval_objective": 0.42774394154548645, | |
| "eval_ranking_idealized": 0.6570248007774353, | |
| "eval_ranking_idealized_expo": 0.5113636255264282, | |
| "eval_ranking_simple": 0.5278925895690918, | |
| "eval_regularize": 0.42774394154548645, | |
| "eval_runtime": 258.5616, | |
| "eval_samples_per_second": 22.393, | |
| "eval_steps_per_second": 0.936, | |
| "step": 150 | |
| }, | |
| { | |
| "dpo_loss": 0.5757229924201965, | |
| "epoch": 0.8786017949929145, | |
| "grad_norm": 9.10049250548801, | |
| "learning_rate": 4.9122289584888926e-06, | |
| "logits": -1.1876070499420166, | |
| "logps": -79.34397888183594, | |
| "loss": 0.2634, | |
| "objective": 0.2684144973754883, | |
| "ranking_idealized": 0.6791666746139526, | |
| "ranking_idealized_expo": 0.512499988079071, | |
| "ranking_simple": 0.6416666507720947, | |
| "regularize": 0.2684144973754883, | |
| "step": 155 | |
| }, | |
| { | |
| "dpo_loss": 0.5666177868843079, | |
| "epoch": 0.9069437883797827, | |
| "grad_norm": 8.820116531265606, | |
| "learning_rate": 4.8987324340362445e-06, | |
| "logits": -1.129431962966919, | |
| "logps": -79.80657196044922, | |
| "loss": 0.2548, | |
| "objective": 0.24889370799064636, | |
| "ranking_idealized": 0.6625000238418579, | |
| "ranking_idealized_expo": 0.4958333373069763, | |
| "ranking_simple": 0.6208333373069763, | |
| "regularize": 0.24889370799064636, | |
| "step": 160 | |
| }, | |
| { | |
| "dpo_loss": 0.5674318671226501, | |
| "epoch": 0.9352857817666509, | |
| "grad_norm": 8.768643401715794, | |
| "learning_rate": 4.884292376870567e-06, | |
| "logits": -1.08602774143219, | |
| "logps": -80.07511138916016, | |
| "loss": 0.2569, | |
| "objective": 0.24699881672859192, | |
| "ranking_idealized": 0.6541666388511658, | |
| "ranking_idealized_expo": 0.5249999761581421, | |
| "ranking_simple": 0.6416666507720947, | |
| "regularize": 0.24699881672859192, | |
| "step": 165 | |
| }, | |
| { | |
| "dpo_loss": 0.5818125605583191, | |
| "epoch": 0.9636277751535192, | |
| "grad_norm": 8.727560270319945, | |
| "learning_rate": 4.868914466936038e-06, | |
| "logits": -1.1042813062667847, | |
| "logps": -81.02680206298828, | |
| "loss": 0.2644, | |
| "objective": 0.2629285454750061, | |
| "ranking_idealized": 0.6291666626930237, | |
| "ranking_idealized_expo": 0.512499988079071, | |
| "ranking_simple": 0.6166666746139526, | |
| "regularize": 0.2629285454750061, | |
| "step": 170 | |
| }, | |
| { | |
| "dpo_loss": 0.5690818428993225, | |
| "epoch": 0.9919697685403873, | |
| "grad_norm": 9.360612159540104, | |
| "learning_rate": 4.8526047530778175e-06, | |
| "logits": -1.0876260995864868, | |
| "logps": -81.20608520507812, | |
| "loss": 0.257, | |
| "objective": 0.2705513834953308, | |
| "ranking_idealized": 0.6833333373069763, | |
| "ranking_idealized_expo": 0.574999988079071, | |
| "ranking_simple": 0.675000011920929, | |
| "regularize": 0.2705513834953308, | |
| "step": 175 | |
| }, | |
| { | |
| "dpo_loss": 0.5392836332321167, | |
| "epoch": 1.0203117619272555, | |
| "grad_norm": 8.811963294729349, | |
| "learning_rate": 4.835369650662767e-06, | |
| "logits": -1.2233085632324219, | |
| "logps": -79.34477233886719, | |
| "loss": 0.2467, | |
| "objective": 0.25315728783607483, | |
| "ranking_idealized": 0.6833333373069763, | |
| "ranking_idealized_expo": 0.5416666865348816, | |
| "ranking_simple": 0.6499999761581421, | |
| "regularize": 0.25315728783607483, | |
| "step": 180 | |
| }, | |
| { | |
| "dpo_loss": 0.5429127812385559, | |
| "epoch": 1.0486537553141237, | |
| "grad_norm": 7.990993827783984, | |
| "learning_rate": 4.817215939055984e-06, | |
| "logits": -1.0636595487594604, | |
| "logps": -77.5766830444336, | |
| "loss": 0.2395, | |
| "objective": 0.23463593423366547, | |
| "ranking_idealized": 0.6541666388511658, | |
| "ranking_idealized_expo": 0.4833333194255829, | |
| "ranking_simple": 0.6083333492279053, | |
| "regularize": 0.23463593423366547, | |
| "step": 185 | |
| }, | |
| { | |
| "dpo_loss": 0.5637180209159851, | |
| "epoch": 1.076995748700992, | |
| "grad_norm": 8.256999336661497, | |
| "learning_rate": 4.798150758954164e-06, | |
| "logits": -1.0749539136886597, | |
| "logps": -78.55563354492188, | |
| "loss": 0.2191, | |
| "objective": 0.21444876492023468, | |
| "ranking_idealized": 0.6916666626930237, | |
| "ranking_idealized_expo": 0.5541666746139526, | |
| "ranking_simple": 0.6541666388511658, | |
| "regularize": 0.2144487500190735, | |
| "step": 190 | |
| }, | |
| { | |
| "dpo_loss": 0.5680408477783203, | |
| "epoch": 1.10533774208786, | |
| "grad_norm": 8.413792498503495, | |
| "learning_rate": 4.778181609576832e-06, | |
| "logits": -1.1007658243179321, | |
| "logps": -77.28519439697266, | |
| "loss": 0.2178, | |
| "objective": 0.21195411682128906, | |
| "ranking_idealized": 0.7041666507720947, | |
| "ranking_idealized_expo": 0.5708333253860474, | |
| "ranking_simple": 0.6666666865348816, | |
| "regularize": 0.21195411682128906, | |
| "step": 195 | |
| }, | |
| { | |
| "dpo_loss": 0.5640944838523865, | |
| "epoch": 1.1336797354747283, | |
| "grad_norm": 7.655804437668192, | |
| "learning_rate": 4.757316345716554e-06, | |
| "logits": -1.2176551818847656, | |
| "logps": -76.59423828125, | |
| "loss": 0.2256, | |
| "objective": 0.2188282459974289, | |
| "ranking_idealized": 0.6499999761581421, | |
| "ranking_idealized_expo": 0.5083333253860474, | |
| "ranking_simple": 0.6333333253860474, | |
| "regularize": 0.2188282459974289, | |
| "step": 200 | |
| }, | |
| { | |
| "epoch": 1.1336797354747283, | |
| "eval_dpo_loss": 0.6861998438835144, | |
| "eval_logits": -1.2023168802261353, | |
| "eval_logps": -81.81192016601562, | |
| "eval_loss": 0.43468645215034485, | |
| "eval_objective": 0.434445858001709, | |
| "eval_ranking_idealized": 0.6570248007774353, | |
| "eval_ranking_idealized_expo": 0.5113636255264282, | |
| "eval_ranking_simple": 0.5247933864593506, | |
| "eval_regularize": 0.434445858001709, | |
| "eval_runtime": 259.1196, | |
| "eval_samples_per_second": 22.345, | |
| "eval_steps_per_second": 0.934, | |
| "step": 200 | |
| }, | |
| { | |
| "dpo_loss": 0.5600470304489136, | |
| "epoch": 1.1620217288615966, | |
| "grad_norm": 7.695005611618307, | |
| "learning_rate": 4.735563174649278e-06, | |
| "logits": -1.1184250116348267, | |
| "logps": -77.62281799316406, | |
| "loss": 0.2243, | |
| "objective": 0.22593899071216583, | |
| "ranking_idealized": 0.6458333134651184, | |
| "ranking_idealized_expo": 0.48750001192092896, | |
| "ranking_simple": 0.6000000238418579, | |
| "regularize": 0.22593899071216583, | |
| "step": 205 | |
| }, | |
| { | |
| "dpo_loss": 0.5418952107429504, | |
| "epoch": 1.1903637222484649, | |
| "grad_norm": 7.898188189840939, | |
| "learning_rate": 4.7129306529060415e-06, | |
| "logits": -1.0059646368026733, | |
| "logps": -78.90467834472656, | |
| "loss": 0.2215, | |
| "objective": 0.21866732835769653, | |
| "ranking_idealized": 0.6291666626930237, | |
| "ranking_idealized_expo": 0.5, | |
| "ranking_simple": 0.612500011920929, | |
| "regularize": 0.21866732835769653, | |
| "step": 210 | |
| }, | |
| { | |
| "dpo_loss": 0.5478299856185913, | |
| "epoch": 1.2187057156353331, | |
| "grad_norm": 8.602045809831973, | |
| "learning_rate": 4.68942768290728e-06, | |
| "logits": -0.9527910947799683, | |
| "logps": -78.7351303100586, | |
| "loss": 0.2113, | |
| "objective": 0.21119125187397003, | |
| "ranking_idealized": 0.6666666865348816, | |
| "ranking_idealized_expo": 0.48750001192092896, | |
| "ranking_simple": 0.625, | |
| "regularize": 0.21119123697280884, | |
| "step": 215 | |
| }, | |
| { | |
| "dpo_loss": 0.5662825107574463, | |
| "epoch": 1.2470477090222012, | |
| "grad_norm": 8.077787041174624, | |
| "learning_rate": 4.665063509461098e-06, | |
| "logits": -0.8720958828926086, | |
| "logps": -76.756591796875, | |
| "loss": 0.2138, | |
| "objective": 0.20959888398647308, | |
| "ranking_idealized": 0.7208333611488342, | |
| "ranking_idealized_expo": 0.6041666865348816, | |
| "ranking_simple": 0.6958333253860474, | |
| "regularize": 0.20959888398647308, | |
| "step": 220 | |
| }, | |
| { | |
| "dpo_loss": 0.5461317300796509, | |
| "epoch": 1.2753897024090695, | |
| "grad_norm": 7.874404608360002, | |
| "learning_rate": 4.639847716126855e-06, | |
| "logits": -0.9673039317131042, | |
| "logps": -78.00302124023438, | |
| "loss": 0.2154, | |
| "objective": 0.20724421739578247, | |
| "ranking_idealized": 0.6791666746139526, | |
| "ranking_idealized_expo": 0.5291666388511658, | |
| "ranking_simple": 0.6416666507720947, | |
| "regularize": 0.20724421739578247, | |
| "step": 225 | |
| }, | |
| { | |
| "dpo_loss": 0.5674420595169067, | |
| "epoch": 1.3037316957959377, | |
| "grad_norm": 7.50398022531709, | |
| "learning_rate": 4.613790221445511e-06, | |
| "logits": -0.9144200682640076, | |
| "logps": -78.14189910888672, | |
| "loss": 0.2025, | |
| "objective": 0.21042712032794952, | |
| "ranking_idealized": 0.6541666388511658, | |
| "ranking_idealized_expo": 0.5083333253860474, | |
| "ranking_simple": 0.637499988079071, | |
| "regularize": 0.21042712032794952, | |
| "step": 230 | |
| }, | |
| { | |
| "dpo_loss": 0.5355426669120789, | |
| "epoch": 1.3320736891828058, | |
| "grad_norm": 7.978556347314524, | |
| "learning_rate": 4.586901275038201e-06, | |
| "logits": -1.0436056852340698, | |
| "logps": -76.17823791503906, | |
| "loss": 0.2072, | |
| "objective": 0.19967219233512878, | |
| "ranking_idealized": 0.6875, | |
| "ranking_idealized_expo": 0.48750001192092896, | |
| "ranking_simple": 0.6291666626930237, | |
| "regularize": 0.19967219233512878, | |
| "step": 235 | |
| }, | |
| { | |
| "dpo_loss": 0.5373556613922119, | |
| "epoch": 1.360415682569674, | |
| "grad_norm": 7.6373820167513635, | |
| "learning_rate": 4.559191453574582e-06, | |
| "logits": -0.9692198038101196, | |
| "logps": -78.42900085449219, | |
| "loss": 0.1972, | |
| "objective": 0.19282637536525726, | |
| "ranking_idealized": 0.637499988079071, | |
| "ranking_idealized_expo": 0.4791666567325592, | |
| "ranking_simple": 0.6458333134651184, | |
| "regularize": 0.19282637536525726, | |
| "step": 240 | |
| }, | |
| { | |
| "dpo_loss": 0.5545187592506409, | |
| "epoch": 1.3887576759565423, | |
| "grad_norm": 7.35404104460057, | |
| "learning_rate": 4.530671656612544e-06, | |
| "logits": -0.992374062538147, | |
| "logps": -76.8333969116211, | |
| "loss": 0.1965, | |
| "objective": 0.19396263360977173, | |
| "ranking_idealized": 0.6833333373069763, | |
| "ranking_idealized_expo": 0.5208333134651184, | |
| "ranking_simple": 0.625, | |
| "regularize": 0.19396263360977173, | |
| "step": 245 | |
| }, | |
| { | |
| "dpo_loss": 0.536364734172821, | |
| "epoch": 1.4170996693434104, | |
| "grad_norm": 7.62812382511743, | |
| "learning_rate": 4.501353102310901e-06, | |
| "logits": -0.9773390293121338, | |
| "logps": -76.92967224121094, | |
| "loss": 0.2005, | |
| "objective": 0.2195345014333725, | |
| "ranking_idealized": 0.6625000238418579, | |
| "ranking_idealized_expo": 0.4749999940395355, | |
| "ranking_simple": 0.625, | |
| "regularize": 0.2195345014333725, | |
| "step": 250 | |
| }, | |
| { | |
| "epoch": 1.4170996693434104, | |
| "eval_dpo_loss": 0.6815471053123474, | |
| "eval_logits": -1.0615853071212769, | |
| "eval_logps": -81.82124328613281, | |
| "eval_loss": 0.42915773391723633, | |
| "eval_objective": 0.4288579821586609, | |
| "eval_ranking_idealized": 0.6570248007774353, | |
| "eval_ranking_idealized_expo": 0.5113636255264282, | |
| "eval_ranking_simple": 0.5227272510528564, | |
| "eval_regularize": 0.4288579821586609, | |
| "eval_runtime": 259.2538, | |
| "eval_samples_per_second": 22.333, | |
| "eval_steps_per_second": 0.933, | |
| "step": 250 | |
| }, | |
| { | |
| "dpo_loss": 0.5536395907402039, | |
| "epoch": 1.4454416627302786, | |
| "grad_norm": 7.608994157416838, | |
| "learning_rate": 4.4712473230167775e-06, | |
| "logits": -0.9322084784507751, | |
| "logps": -77.44235229492188, | |
| "loss": 0.1968, | |
| "objective": 0.1830952763557434, | |
| "ranking_idealized": 0.675000011920929, | |
| "ranking_idealized_expo": 0.5458333492279053, | |
| "ranking_simple": 0.6499999761581421, | |
| "regularize": 0.1830952763557434, | |
| "step": 255 | |
| }, | |
| { | |
| "dpo_loss": 0.5560018420219421, | |
| "epoch": 1.473783656117147, | |
| "grad_norm": 7.9576485937673995, | |
| "learning_rate": 4.440366160729393e-06, | |
| "logits": -0.9907886385917664, | |
| "logps": -77.4918212890625, | |
| "loss": 0.197, | |
| "objective": 0.20965854823589325, | |
| "ranking_idealized": 0.6499999761581421, | |
| "ranking_idealized_expo": 0.48750001192092896, | |
| "ranking_simple": 0.6333333253860474, | |
| "regularize": 0.20965854823589325, | |
| "step": 260 | |
| }, | |
| { | |
| "dpo_loss": 0.5228397250175476, | |
| "epoch": 1.5021256495040152, | |
| "grad_norm": 7.710193409626713, | |
| "learning_rate": 4.4087217624420595e-06, | |
| "logits": -0.9860392808914185, | |
| "logps": -76.1131362915039, | |
| "loss": 0.1993, | |
| "objective": 0.18088673055171967, | |
| "ranking_idealized": 0.6333333253860474, | |
| "ranking_idealized_expo": 0.5041666626930237, | |
| "ranking_simple": 0.6333333253860474, | |
| "regularize": 0.18088671565055847, | |
| "step": 265 | |
| }, | |
| { | |
| "dpo_loss": 0.5495325922966003, | |
| "epoch": 1.5304676428908834, | |
| "grad_norm": 7.266856200970056, | |
| "learning_rate": 4.376326575364206e-06, | |
| "logits": -0.9504061341285706, | |
| "logps": -77.44710540771484, | |
| "loss": 0.1911, | |
| "objective": 0.1995639055967331, | |
| "ranking_idealized": 0.6791666746139526, | |
| "ranking_idealized_expo": 0.512499988079071, | |
| "ranking_simple": 0.637499988079071, | |
| "regularize": 0.1995639055967331, | |
| "step": 270 | |
| }, | |
| { | |
| "dpo_loss": 0.5499274730682373, | |
| "epoch": 1.5588096362777515, | |
| "grad_norm": 7.524550189383748, | |
| "learning_rate": 4.34319334202531e-06, | |
| "logits": -0.959584653377533, | |
| "logps": -77.12781524658203, | |
| "loss": 0.1924, | |
| "objective": 0.20593222975730896, | |
| "ranking_idealized": 0.6916666626930237, | |
| "ranking_idealized_expo": 0.5249999761581421, | |
| "ranking_simple": 0.6833333373069763, | |
| "regularize": 0.20593222975730896, | |
| "step": 275 | |
| }, | |
| { | |
| "dpo_loss": 0.5305771827697754, | |
| "epoch": 1.5871516296646198, | |
| "grad_norm": 7.773566185560746, | |
| "learning_rate": 4.309335095262675e-06, | |
| "logits": -0.855711042881012, | |
| "logps": -74.90408325195312, | |
| "loss": 0.1869, | |
| "objective": 0.18780963122844696, | |
| "ranking_idealized": 0.675000011920929, | |
| "ranking_idealized_expo": 0.5041666626930237, | |
| "ranking_simple": 0.6416666507720947, | |
| "regularize": 0.18780963122844696, | |
| "step": 280 | |
| }, | |
| { | |
| "dpo_loss": 0.5325539112091064, | |
| "epoch": 1.615493623051488, | |
| "grad_norm": 7.526664818437735, | |
| "learning_rate": 4.274765153095008e-06, | |
| "logits": -0.8781672120094299, | |
| "logps": -76.42524719238281, | |
| "loss": 0.1948, | |
| "objective": 0.18717069923877716, | |
| "ranking_idealized": 0.675000011920929, | |
| "ranking_idealized_expo": 0.5166666507720947, | |
| "ranking_simple": 0.6583333611488342, | |
| "regularize": 0.18717069923877716, | |
| "step": 285 | |
| }, | |
| { | |
| "dpo_loss": 0.5575224757194519, | |
| "epoch": 1.643835616438356, | |
| "grad_norm": 6.835237556478757, | |
| "learning_rate": 4.239497113483819e-06, | |
| "logits": -0.8322954773902893, | |
| "logps": -74.6546859741211, | |
| "loss": 0.1789, | |
| "objective": 0.182403564453125, | |
| "ranking_idealized": 0.699999988079071, | |
| "ranking_idealized_expo": 0.5625, | |
| "ranking_simple": 0.6625000238418579, | |
| "regularize": 0.182403564453125, | |
| "step": 290 | |
| }, | |
| { | |
| "dpo_loss": 0.5432038903236389, | |
| "epoch": 1.6721776098252243, | |
| "grad_norm": 6.961571934316881, | |
| "learning_rate": 4.203544848984729e-06, | |
| "logits": -0.8109145760536194, | |
| "logps": -72.75186920166016, | |
| "loss": 0.1859, | |
| "objective": 0.18214626610279083, | |
| "ranking_idealized": 0.6583333611488342, | |
| "ranking_idealized_expo": 0.4833333194255829, | |
| "ranking_simple": 0.6416666507720947, | |
| "regularize": 0.18214626610279083, | |
| "step": 295 | |
| }, | |
| { | |
| "dpo_loss": 0.5352488160133362, | |
| "epoch": 1.7005196032120926, | |
| "grad_norm": 7.403474665890531, | |
| "learning_rate": 4.16692250129073e-06, | |
| "logits": -0.9637666344642639, | |
| "logps": -74.78925323486328, | |
| "loss": 0.187, | |
| "objective": 0.20280833542346954, | |
| "ranking_idealized": 0.7083333134651184, | |
| "ranking_idealized_expo": 0.574999988079071, | |
| "ranking_simple": 0.6958333253860474, | |
| "regularize": 0.20280833542346954, | |
| "step": 300 | |
| }, | |
| { | |
| "epoch": 1.7005196032120926, | |
| "eval_dpo_loss": 0.6845287680625916, | |
| "eval_logits": -1.0398027896881104, | |
| "eval_logps": -80.0077133178711, | |
| "eval_loss": 0.43694496154785156, | |
| "eval_objective": 0.43617644906044006, | |
| "eval_ranking_idealized": 0.6570248007774353, | |
| "eval_ranking_idealized_expo": 0.5113636255264282, | |
| "eval_ranking_simple": 0.5258264541625977, | |
| "eval_regularize": 0.43617644906044006, | |
| "eval_runtime": 259.8028, | |
| "eval_samples_per_second": 22.286, | |
| "eval_steps_per_second": 0.931, | |
| "step": 300 | |
| }, | |
| { | |
| "dpo_loss": 0.5487022399902344, | |
| "epoch": 1.7288615965989607, | |
| "grad_norm": 7.543811016492746, | |
| "learning_rate": 4.129644475669617e-06, | |
| "logits": -0.9759048223495483, | |
| "logps": -74.88541412353516, | |
| "loss": 0.1848, | |
| "objective": 0.18824300169944763, | |
| "ranking_idealized": 0.6291666626930237, | |
| "ranking_idealized_expo": 0.5083333253860474, | |
| "ranking_simple": 0.625, | |
| "regularize": 0.18824300169944763, | |
| "step": 305 | |
| }, | |
| { | |
| "dpo_loss": 0.5396175980567932, | |
| "epoch": 1.7572035899858292, | |
| "grad_norm": 7.45667126547559, | |
| "learning_rate": 4.091725435297721e-06, | |
| "logits": -1.014137625694275, | |
| "logps": -71.69686126708984, | |
| "loss": 0.187, | |
| "objective": 0.18091975152492523, | |
| "ranking_idealized": 0.6583333611488342, | |
| "ranking_idealized_expo": 0.5333333611488342, | |
| "ranking_simple": 0.6499999761581421, | |
| "regularize": 0.18091975152492523, | |
| "step": 310 | |
| }, | |
| { | |
| "dpo_loss": 0.5542029142379761, | |
| "epoch": 1.7855455833726972, | |
| "grad_norm": 7.022791945824948, | |
| "learning_rate": 4.053180295492203e-06, | |
| "logits": -0.9052151441574097, | |
| "logps": -72.4874038696289, | |
| "loss": 0.1773, | |
| "objective": 0.17407093942165375, | |
| "ranking_idealized": 0.6541666388511658, | |
| "ranking_idealized_expo": 0.512499988079071, | |
| "ranking_simple": 0.612500011920929, | |
| "regularize": 0.17407093942165375, | |
| "step": 315 | |
| }, | |
| { | |
| "dpo_loss": 0.5506651401519775, | |
| "epoch": 1.8138875767595655, | |
| "grad_norm": 7.473980140791967, | |
| "learning_rate": 4.014024217844167e-06, | |
| "logits": -0.9277843832969666, | |
| "logps": -75.52190399169922, | |
| "loss": 0.1782, | |
| "objective": 0.19924014806747437, | |
| "ranking_idealized": 0.6625000238418579, | |
| "ranking_idealized_expo": 0.5041666626930237, | |
| "ranking_simple": 0.637499988079071, | |
| "regularize": 0.19924013316631317, | |
| "step": 320 | |
| }, | |
| { | |
| "dpo_loss": 0.5471735000610352, | |
| "epoch": 1.8422295701464337, | |
| "grad_norm": 7.500911105092317, | |
| "learning_rate": 3.974272604254906e-06, | |
| "logits": -0.9084165096282959, | |
| "logps": -77.74280548095703, | |
| "loss": 0.1791, | |
| "objective": 0.18182264268398285, | |
| "ranking_idealized": 0.7166666388511658, | |
| "ranking_idealized_expo": 0.5208333134651184, | |
| "ranking_simple": 0.6875, | |
| "regularize": 0.18182264268398285, | |
| "step": 325 | |
| }, | |
| { | |
| "dpo_loss": 0.5376237034797668, | |
| "epoch": 1.8705715635333018, | |
| "grad_norm": 6.956332723343249, | |
| "learning_rate": 3.933941090877615e-06, | |
| "logits": -0.6826075911521912, | |
| "logps": -74.37185668945312, | |
| "loss": 0.1748, | |
| "objective": 0.15574544668197632, | |
| "ranking_idealized": 0.6499999761581421, | |
| "ranking_idealized_expo": 0.4749999940395355, | |
| "ranking_simple": 0.6499999761581421, | |
| "regularize": 0.15574544668197632, | |
| "step": 330 | |
| }, | |
| { | |
| "dpo_loss": 0.5471087694168091, | |
| "epoch": 1.89891355692017, | |
| "grad_norm": 7.004792858354732, | |
| "learning_rate": 3.893045541966975e-06, | |
| "logits": -0.8625032901763916, | |
| "logps": -73.06546020507812, | |
| "loss": 0.1666, | |
| "objective": 0.18181832134723663, | |
| "ranking_idealized": 0.6499999761581421, | |
| "ranking_idealized_expo": 0.4958333373069763, | |
| "ranking_simple": 0.6291666626930237, | |
| "regularize": 0.18181832134723663, | |
| "step": 335 | |
| }, | |
| { | |
| "dpo_loss": 0.5468536615371704, | |
| "epoch": 1.9272555503070383, | |
| "grad_norm": 7.263082545218723, | |
| "learning_rate": 3.8516020436389945e-06, | |
| "logits": -0.8647869825363159, | |
| "logps": -74.24337005615234, | |
| "loss": 0.1658, | |
| "objective": 0.16866040229797363, | |
| "ranking_idealized": 0.7291666865348816, | |
| "ranking_idealized_expo": 0.5458333492279053, | |
| "ranking_simple": 0.7166666388511658, | |
| "regularize": 0.16866040229797363, | |
| "step": 340 | |
| }, | |
| { | |
| "dpo_loss": 0.5612522959709167, | |
| "epoch": 1.9555975436939064, | |
| "grad_norm": 7.17931193885588, | |
| "learning_rate": 3.8096268975436045e-06, | |
| "logits": -0.9274277091026306, | |
| "logps": -72.64501953125, | |
| "loss": 0.1631, | |
| "objective": 0.1834598332643509, | |
| "ranking_idealized": 0.6458333134651184, | |
| "ranking_idealized_expo": 0.5333333611488342, | |
| "ranking_simple": 0.6041666865348816, | |
| "regularize": 0.1834598332643509, | |
| "step": 345 | |
| }, | |
| { | |
| "dpo_loss": 0.5486599802970886, | |
| "epoch": 1.9839395370807746, | |
| "grad_norm": 7.4574942962802675, | |
| "learning_rate": 3.767136614452458e-06, | |
| "logits": -0.8831450343132019, | |
| "logps": -75.68425750732422, | |
| "loss": 0.1664, | |
| "objective": 0.15772633254528046, | |
| "ranking_idealized": 0.6333333253860474, | |
| "ranking_idealized_expo": 0.49166667461395264, | |
| "ranking_simple": 0.6166666746139526, | |
| "regularize": 0.15772633254528046, | |
| "step": 350 | |
| }, | |
| { | |
| "epoch": 1.9839395370807746, | |
| "eval_dpo_loss": 0.6841984987258911, | |
| "eval_logits": -0.9982038140296936, | |
| "eval_logps": -79.63081359863281, | |
| "eval_loss": 0.4382030665874481, | |
| "eval_objective": 0.4358615577220917, | |
| "eval_ranking_idealized": 0.6570248007774353, | |
| "eval_ranking_idealized_expo": 0.5113636255264282, | |
| "eval_ranking_simple": 0.5289255976676941, | |
| "eval_regularize": 0.4358615577220917, | |
| "eval_runtime": 258.5329, | |
| "eval_samples_per_second": 22.396, | |
| "eval_steps_per_second": 0.936, | |
| "step": 350 | |
| }, | |
| { | |
| "dpo_loss": 0.5294139981269836, | |
| "epoch": 2.012281530467643, | |
| "grad_norm": 7.097987961305298, | |
| "learning_rate": 3.724147907764478e-06, | |
| "logits": -0.8353627324104309, | |
| "logps": -75.18488311767578, | |
| "loss": 0.1556, | |
| "objective": 0.14675287902355194, | |
| "ranking_idealized": 0.699999988079071, | |
| "ranking_idealized_expo": 0.5166666507720947, | |
| "ranking_simple": 0.675000011920929, | |
| "regularize": 0.14675287902355194, | |
| "step": 355 | |
| }, | |
| { | |
| "dpo_loss": 0.5278249979019165, | |
| "epoch": 2.040623523854511, | |
| "grad_norm": 7.097858061871919, | |
| "learning_rate": 3.6806776869317074e-06, | |
| "logits": -0.8662230372428894, | |
| "logps": -74.37833404541016, | |
| "loss": 0.1484, | |
| "objective": 0.13966700434684753, | |
| "ranking_idealized": 0.7250000238418579, | |
| "ranking_idealized_expo": 0.5333333611488342, | |
| "ranking_simple": 0.6916666626930237, | |
| "regularize": 0.13966700434684753, | |
| "step": 360 | |
| }, | |
| { | |
| "dpo_loss": 0.5365410447120667, | |
| "epoch": 2.0689655172413794, | |
| "grad_norm": 7.018046135257267, | |
| "learning_rate": 3.6367430508080283e-06, | |
| "logits": -0.9098676443099976, | |
| "logps": -77.11681365966797, | |
| "loss": 0.1521, | |
| "objective": 0.14031733572483063, | |
| "ranking_idealized": 0.6708333492279053, | |
| "ranking_idealized_expo": 0.5041666626930237, | |
| "ranking_simple": 0.6541666388511658, | |
| "regularize": 0.14031733572483063, | |
| "step": 365 | |
| }, | |
| { | |
| "dpo_loss": 0.5375342965126038, | |
| "epoch": 2.0973075106282475, | |
| "grad_norm": 6.853083763524368, | |
| "learning_rate": 3.5923612809233987e-06, | |
| "logits": -0.8166040778160095, | |
| "logps": -74.46318817138672, | |
| "loss": 0.1422, | |
| "objective": 0.14259177446365356, | |
| "ranking_idealized": 0.6541666388511658, | |
| "ranking_idealized_expo": 0.4833333194255829, | |
| "ranking_simple": 0.6499999761581421, | |
| "regularize": 0.14259177446365356, | |
| "step": 370 | |
| }, | |
| { | |
| "dpo_loss": 0.5261290669441223, | |
| "epoch": 2.1256495040151155, | |
| "grad_norm": 6.737212175634408, | |
| "learning_rate": 3.547549834686222e-06, | |
| "logits": -0.7999446392059326, | |
| "logps": -76.43958282470703, | |
| "loss": 0.1396, | |
| "objective": 0.14750143885612488, | |
| "ranking_idealized": 0.7291666865348816, | |
| "ranking_idealized_expo": 0.5958333611488342, | |
| "ranking_simple": 0.7124999761581421, | |
| "regularize": 0.14750143885612488, | |
| "step": 375 | |
| }, | |
| { | |
| "dpo_loss": 0.5414277911186218, | |
| "epoch": 2.153991497401984, | |
| "grad_norm": 7.256488804126431, | |
| "learning_rate": 3.5023263385165346e-06, | |
| "logits": -0.7813270092010498, | |
| "logps": -75.50519561767578, | |
| "loss": 0.1433, | |
| "objective": 0.14621533453464508, | |
| "ranking_idealized": 0.6833333373069763, | |
| "ranking_idealized_expo": 0.5375000238418579, | |
| "ranking_simple": 0.675000011920929, | |
| "regularize": 0.14621533453464508, | |
| "step": 380 | |
| }, | |
| { | |
| "dpo_loss": 0.5473071932792664, | |
| "epoch": 2.182333490788852, | |
| "grad_norm": 7.1971925561639445, | |
| "learning_rate": 3.4567085809127247e-06, | |
| "logits": -0.8520491123199463, | |
| "logps": -77.47991180419922, | |
| "loss": 0.1389, | |
| "objective": 0.13993406295776367, | |
| "ranking_idealized": 0.6458333134651184, | |
| "ranking_idealized_expo": 0.5083333253860474, | |
| "ranking_simple": 0.6583333611488342, | |
| "regularize": 0.13993406295776367, | |
| "step": 385 | |
| }, | |
| { | |
| "dpo_loss": 0.538723349571228, | |
| "epoch": 2.21067548417572, | |
| "grad_norm": 7.027817565044039, | |
| "learning_rate": 3.410714505454486e-06, | |
| "logits": -0.80887770652771, | |
| "logps": -76.32317352294922, | |
| "loss": 0.1308, | |
| "objective": 0.12663429975509644, | |
| "ranking_idealized": 0.6041666865348816, | |
| "ranking_idealized_expo": 0.4833333194255829, | |
| "ranking_simple": 0.6000000238418579, | |
| "regularize": 0.12663428485393524, | |
| "step": 390 | |
| }, | |
| { | |
| "dpo_loss": 0.5224890112876892, | |
| "epoch": 2.2390174775625886, | |
| "grad_norm": 6.949068306199585, | |
| "learning_rate": 3.364362203744777e-06, | |
| "logits": -0.8549118638038635, | |
| "logps": -76.35110473632812, | |
| "loss": 0.1437, | |
| "objective": 0.14411191642284393, | |
| "ranking_idealized": 0.6916666626930237, | |
| "ranking_idealized_expo": 0.5166666507720947, | |
| "ranking_simple": 0.675000011920929, | |
| "regularize": 0.14411191642284393, | |
| "step": 395 | |
| }, | |
| { | |
| "dpo_loss": 0.5168216228485107, | |
| "epoch": 2.2673594709494567, | |
| "grad_norm": 7.191883909269667, | |
| "learning_rate": 3.3176699082935546e-06, | |
| "logits": -0.9377852082252502, | |
| "logps": -77.43128204345703, | |
| "loss": 0.1368, | |
| "objective": 0.13501186668872833, | |
| "ranking_idealized": 0.7583333253860474, | |
| "ranking_idealized_expo": 0.5791666507720947, | |
| "ranking_simple": 0.737500011920929, | |
| "regularize": 0.13501186668872833, | |
| "step": 400 | |
| }, | |
| { | |
| "epoch": 2.2673594709494567, | |
| "eval_dpo_loss": 0.6858804225921631, | |
| "eval_logits": -1.0155284404754639, | |
| "eval_logps": -80.20379638671875, | |
| "eval_loss": 0.4407689571380615, | |
| "eval_objective": 0.43782898783683777, | |
| "eval_ranking_idealized": 0.6570248007774353, | |
| "eval_ranking_idealized_expo": 0.5113636255264282, | |
| "eval_ranking_simple": 0.5320248007774353, | |
| "eval_regularize": 0.43782898783683777, | |
| "eval_runtime": 259.3137, | |
| "eval_samples_per_second": 22.328, | |
| "eval_steps_per_second": 0.933, | |
| "step": 400 | |
| }, | |
| { | |
| "dpo_loss": 0.537171483039856, | |
| "epoch": 2.295701464336325, | |
| "grad_norm": 7.007268345576302, | |
| "learning_rate": 3.2706559853460818e-06, | |
| "logits": -1.0055091381072998, | |
| "logps": -74.74109649658203, | |
| "loss": 0.1383, | |
| "objective": 0.1392410695552826, | |
| "ranking_idealized": 0.737500011920929, | |
| "ranking_idealized_expo": 0.5583333373069763, | |
| "ranking_simple": 0.7083333134651184, | |
| "regularize": 0.1392410695552826, | |
| "step": 405 | |
| }, | |
| { | |
| "dpo_loss": 0.5326074361801147, | |
| "epoch": 2.324043457723193, | |
| "grad_norm": 6.941825657006022, | |
| "learning_rate": 3.2233389276586325e-06, | |
| "logits": -0.8970204591751099, | |
| "logps": -74.2923355102539, | |
| "loss": 0.1297, | |
| "objective": 0.12273009866476059, | |
| "ranking_idealized": 0.6499999761581421, | |
| "ranking_idealized_expo": 0.4833333194255829, | |
| "ranking_simple": 0.6458333134651184, | |
| "regularize": 0.12273009866476059, | |
| "step": 410 | |
| }, | |
| { | |
| "dpo_loss": 0.5270788073539734, | |
| "epoch": 2.3523854511100613, | |
| "grad_norm": 7.366817713585052, | |
| "learning_rate": 3.1757373472244324e-06, | |
| "logits": -0.8971990942955017, | |
| "logps": -74.44580078125, | |
| "loss": 0.1319, | |
| "objective": 0.14558042585849762, | |
| "ranking_idealized": 0.6458333134651184, | |
| "ranking_idealized_expo": 0.5333333611488342, | |
| "ranking_simple": 0.6333333253860474, | |
| "regularize": 0.14558042585849762, | |
| "step": 415 | |
| }, | |
| { | |
| "dpo_loss": 0.5407485961914062, | |
| "epoch": 2.3807274444969297, | |
| "grad_norm": 7.016879855860433, | |
| "learning_rate": 3.127869967952698e-06, | |
| "logits": -0.8165015578269958, | |
| "logps": -75.92127227783203, | |
| "loss": 0.1297, | |
| "objective": 0.13260656595230103, | |
| "ranking_idealized": 0.6458333134651184, | |
| "ranking_idealized_expo": 0.4749999940395355, | |
| "ranking_simple": 0.6333333253860474, | |
| "regularize": 0.13260656595230103, | |
| "step": 420 | |
| }, | |
| { | |
| "dpo_loss": 0.526520848274231, | |
| "epoch": 2.409069437883798, | |
| "grad_norm": 6.605258428992394, | |
| "learning_rate": 3.0797556183036582e-06, | |
| "logits": -0.8272897601127625, | |
| "logps": -74.90123748779297, | |
| "loss": 0.1262, | |
| "objective": 0.12788838148117065, | |
| "ranking_idealized": 0.6625000238418579, | |
| "ranking_idealized_expo": 0.5041666626930237, | |
| "ranking_simple": 0.6499999761581421, | |
| "regularize": 0.12788838148117065, | |
| "step": 425 | |
| }, | |
| { | |
| "dpo_loss": 0.5405741333961487, | |
| "epoch": 2.4374114312706663, | |
| "grad_norm": 6.6924189063588955, | |
| "learning_rate": 3.0314132238824416e-06, | |
| "logits": -0.8260743021965027, | |
| "logps": -75.06442260742188, | |
| "loss": 0.1266, | |
| "objective": 0.11811564862728119, | |
| "ranking_idealized": 0.6708333492279053, | |
| "ranking_idealized_expo": 0.5333333611488342, | |
| "ranking_simple": 0.6416666507720947, | |
| "regularize": 0.11811564117670059, | |
| "step": 430 | |
| }, | |
| { | |
| "dpo_loss": 0.5238969922065735, | |
| "epoch": 2.4657534246575343, | |
| "grad_norm": 6.903355919273098, | |
| "learning_rate": 2.9828617999947647e-06, | |
| "logits": -0.8589097857475281, | |
| "logps": -75.69316101074219, | |
| "loss": 0.1249, | |
| "objective": 0.12512782216072083, | |
| "ranking_idealized": 0.6333333253860474, | |
| "ranking_idealized_expo": 0.5083333253860474, | |
| "ranking_simple": 0.6708333492279053, | |
| "regularize": 0.12512782216072083, | |
| "step": 435 | |
| }, | |
| { | |
| "dpo_loss": 0.5445213317871094, | |
| "epoch": 2.4940954180444024, | |
| "grad_norm": 6.961032298685898, | |
| "learning_rate": 2.9341204441673267e-06, | |
| "logits": -0.7675038576126099, | |
| "logps": -74.55623626708984, | |
| "loss": 0.126, | |
| "objective": 0.12659841775894165, | |
| "ranking_idealized": 0.6333333253860474, | |
| "ranking_idealized_expo": 0.5375000238418579, | |
| "ranking_simple": 0.6208333373069763, | |
| "regularize": 0.12659841775894165, | |
| "step": 440 | |
| }, | |
| { | |
| "dpo_loss": 0.5370126366615295, | |
| "epoch": 2.5224374114312704, | |
| "grad_norm": 6.94281741073779, | |
| "learning_rate": 2.8852083286358647e-06, | |
| "logits": -0.7942711710929871, | |
| "logps": -71.90986633300781, | |
| "loss": 0.1211, | |
| "objective": 0.13307242095470428, | |
| "ranking_idealized": 0.6958333253860474, | |
| "ranking_idealized_expo": 0.5166666507720947, | |
| "ranking_simple": 0.675000011920929, | |
| "regularize": 0.13307242095470428, | |
| "step": 445 | |
| }, | |
| { | |
| "dpo_loss": 0.5622422099113464, | |
| "epoch": 2.550779404818139, | |
| "grad_norm": 7.14501016601336, | |
| "learning_rate": 2.8361446928038298e-06, | |
| "logits": -0.8527530431747437, | |
| "logps": -74.12069702148438, | |
| "loss": 0.122, | |
| "objective": 0.13083526492118835, | |
| "ranking_idealized": 0.6583333611488342, | |
| "ranking_idealized_expo": 0.5, | |
| "ranking_simple": 0.612500011920929, | |
| "regularize": 0.13083526492118835, | |
| "step": 450 | |
| }, | |
| { | |
| "epoch": 2.550779404818139, | |
| "eval_dpo_loss": 0.6862838268280029, | |
| "eval_logits": -0.8946070671081543, | |
| "eval_logps": -78.42880249023438, | |
| "eval_loss": 0.44150272011756897, | |
| "eval_objective": 0.44035181403160095, | |
| "eval_ranking_idealized": 0.6570248007774353, | |
| "eval_ranking_idealized_expo": 0.5113636255264282, | |
| "eval_ranking_simple": 0.5258264541625977, | |
| "eval_regularize": 0.44035181403160095, | |
| "eval_runtime": 258.0077, | |
| "eval_samples_per_second": 22.441, | |
| "eval_steps_per_second": 0.938, | |
| "step": 450 | |
| }, | |
| { | |
| "dpo_loss": 0.5327574610710144, | |
| "epoch": 2.579121398205007, | |
| "grad_norm": 7.160178764520009, | |
| "learning_rate": 2.7869488356746344e-06, | |
| "logits": -0.8465222716331482, | |
| "logps": -74.07278442382812, | |
| "loss": 0.1216, | |
| "objective": 0.1370854526758194, | |
| "ranking_idealized": 0.6666666865348816, | |
| "ranking_idealized_expo": 0.5041666626930237, | |
| "ranking_simple": 0.6625000238418579, | |
| "regularize": 0.1370854526758194, | |
| "step": 455 | |
| }, | |
| { | |
| "dpo_loss": 0.5250583291053772, | |
| "epoch": 2.6074633915918755, | |
| "grad_norm": 7.146508258603712, | |
| "learning_rate": 2.7376401082604563e-06, | |
| "logits": -0.8942106366157532, | |
| "logps": -74.29560852050781, | |
| "loss": 0.1165, | |
| "objective": 0.12135622650384903, | |
| "ranking_idealized": 0.6833333373069763, | |
| "ranking_idealized_expo": 0.512499988079071, | |
| "ranking_simple": 0.6666666865348816, | |
| "regularize": 0.12135622650384903, | |
| "step": 460 | |
| }, | |
| { | |
| "dpo_loss": 0.530348539352417, | |
| "epoch": 2.6358053849787435, | |
| "grad_norm": 7.184783585057386, | |
| "learning_rate": 2.6882379059705953e-06, | |
| "logits": -0.7349064350128174, | |
| "logps": -73.84181213378906, | |
| "loss": 0.1204, | |
| "objective": 0.1292448341846466, | |
| "ranking_idealized": 0.612500011920929, | |
| "ranking_idealized_expo": 0.4958333373069763, | |
| "ranking_simple": 0.6208333373069763, | |
| "regularize": 0.1292448341846466, | |
| "step": 465 | |
| }, | |
| { | |
| "dpo_loss": 0.5289201140403748, | |
| "epoch": 2.6641473783656116, | |
| "grad_norm": 7.1334661921422216, | |
| "learning_rate": 2.6387616609823506e-06, | |
| "logits": -0.7510494589805603, | |
| "logps": -73.42973327636719, | |
| "loss": 0.1121, | |
| "objective": 0.10491514950990677, | |
| "ranking_idealized": 0.6625000238418579, | |
| "ranking_idealized_expo": 0.5583333373069763, | |
| "ranking_simple": 0.6541666388511658, | |
| "regularize": 0.10491514950990677, | |
| "step": 470 | |
| }, | |
| { | |
| "dpo_loss": 0.5439261794090271, | |
| "epoch": 2.69248937175248, | |
| "grad_norm": 7.044559545822627, | |
| "learning_rate": 2.5892308345974517e-06, | |
| "logits": -0.7502660155296326, | |
| "logps": -73.04237365722656, | |
| "loss": 0.1114, | |
| "objective": 0.11930320411920547, | |
| "ranking_idealized": 0.6791666746139526, | |
| "ranking_idealized_expo": 0.4958333373069763, | |
| "ranking_simple": 0.6583333611488342, | |
| "regularize": 0.11930320411920547, | |
| "step": 475 | |
| }, | |
| { | |
| "dpo_loss": 0.528633177280426, | |
| "epoch": 2.720831365139348, | |
| "grad_norm": 7.216679182755541, | |
| "learning_rate": 2.53966490958702e-06, | |
| "logits": -0.832193911075592, | |
| "logps": -73.19383239746094, | |
| "loss": 0.1104, | |
| "objective": 0.10962951928377151, | |
| "ranking_idealized": 0.6833333373069763, | |
| "ranking_idealized_expo": 0.5, | |
| "ranking_simple": 0.6916666626930237, | |
| "regularize": 0.10962951928377151, | |
| "step": 480 | |
| }, | |
| { | |
| "dpo_loss": 0.5266162753105164, | |
| "epoch": 2.7491733585262166, | |
| "grad_norm": 6.942552990674209, | |
| "learning_rate": 2.490083382528097e-06, | |
| "logits": -0.780593752861023, | |
| "logps": -75.74951171875, | |
| "loss": 0.1126, | |
| "objective": 0.10767225921154022, | |
| "ranking_idealized": 0.6708333492279053, | |
| "ranking_idealized_expo": 0.5249999761581421, | |
| "ranking_simple": 0.675000011920929, | |
| "regularize": 0.10767225176095963, | |
| "step": 485 | |
| }, | |
| { | |
| "dpo_loss": 0.5204401016235352, | |
| "epoch": 2.7775153519130846, | |
| "grad_norm": 6.79511973951677, | |
| "learning_rate": 2.440505756134732e-06, | |
| "logits": -0.7421233654022217, | |
| "logps": -74.27189636230469, | |
| "loss": 0.1117, | |
| "objective": 0.11147340387105942, | |
| "ranking_idealized": 0.6833333373069763, | |
| "ranking_idealized_expo": 0.4749999940395355, | |
| "ranking_simple": 0.6833333373069763, | |
| "regularize": 0.11147340387105942, | |
| "step": 490 | |
| }, | |
| { | |
| "dpo_loss": 0.5330770015716553, | |
| "epoch": 2.8058573452999527, | |
| "grad_norm": 7.282976503781254, | |
| "learning_rate": 2.3909515315866606e-06, | |
| "logits": -0.7717820405960083, | |
| "logps": -72.36864471435547, | |
| "loss": 0.1034, | |
| "objective": 0.10382074862718582, | |
| "ranking_idealized": 0.6416666507720947, | |
| "ranking_idealized_expo": 0.4833333194255829, | |
| "ranking_simple": 0.6333333253860474, | |
| "regularize": 0.10382074862718582, | |
| "step": 495 | |
| }, | |
| { | |
| "dpo_loss": 0.5220891833305359, | |
| "epoch": 2.8341993386868207, | |
| "grad_norm": 7.004605377489112, | |
| "learning_rate": 2.341440200858589e-06, | |
| "logits": -0.822429895401001, | |
| "logps": -71.28691864013672, | |
| "loss": 0.1063, | |
| "objective": 0.11509209126234055, | |
| "ranking_idealized": 0.6916666626930237, | |
| "ranking_idealized_expo": 0.5208333134651184, | |
| "ranking_simple": 0.6666666865348816, | |
| "regularize": 0.11509209126234055, | |
| "step": 500 | |
| }, | |
| { | |
| "epoch": 2.8341993386868207, | |
| "eval_dpo_loss": 0.6860550045967102, | |
| "eval_logits": -0.8682713508605957, | |
| "eval_logps": -78.1278076171875, | |
| "eval_loss": 0.44106075167655945, | |
| "eval_objective": 0.43840065598487854, | |
| "eval_ranking_idealized": 0.6570248007774353, | |
| "eval_ranking_idealized_expo": 0.5113636255264282, | |
| "eval_ranking_simple": 0.5299586653709412, | |
| "eval_regularize": 0.43840065598487854, | |
| "eval_runtime": 259.0969, | |
| "eval_samples_per_second": 22.347, | |
| "eval_steps_per_second": 0.934, | |
| "step": 500 | |
| }, | |
| { | |
| "dpo_loss": 0.5273423790931702, | |
| "epoch": 2.862541332073689, | |
| "grad_norm": 6.758921464396605, | |
| "learning_rate": 2.2919912390530945e-06, | |
| "logits": -0.7519776225090027, | |
| "logps": -72.76760864257812, | |
| "loss": 0.1035, | |
| "objective": 0.10023737698793411, | |
| "ranking_idealized": 0.6583333611488342, | |
| "ranking_idealized_expo": 0.5249999761581421, | |
| "ranking_simple": 0.6833333373069763, | |
| "regularize": 0.10023736953735352, | |
| "step": 505 | |
| }, | |
| { | |
| "dpo_loss": 0.5383204817771912, | |
| "epoch": 2.8908833254605573, | |
| "grad_norm": 6.749917335321248, | |
| "learning_rate": 2.242624096740164e-06, | |
| "logits": -0.7136736512184143, | |
| "logps": -73.48322296142578, | |
| "loss": 0.1008, | |
| "objective": 0.10292276740074158, | |
| "ranking_idealized": 0.6958333253860474, | |
| "ranking_idealized_expo": 0.5291666388511658, | |
| "ranking_simple": 0.675000011920929, | |
| "regularize": 0.10292276740074158, | |
| "step": 510 | |
| }, | |
| { | |
| "dpo_loss": 0.5426943302154541, | |
| "epoch": 2.9192253188474258, | |
| "grad_norm": 6.80108683493094, | |
| "learning_rate": 2.193358192306384e-06, | |
| "logits": -0.844546914100647, | |
| "logps": -72.76075744628906, | |
| "loss": 0.1039, | |
| "objective": 0.10740550607442856, | |
| "ranking_idealized": 0.6583333611488342, | |
| "ranking_idealized_expo": 0.4791666567325592, | |
| "ranking_simple": 0.6583333611488342, | |
| "regularize": 0.10740550607442856, | |
| "step": 515 | |
| }, | |
| { | |
| "dpo_loss": 0.5363429188728333, | |
| "epoch": 2.947567312234294, | |
| "grad_norm": 7.22597227685905, | |
| "learning_rate": 2.1442129043167877e-06, | |
| "logits": -0.7738971710205078, | |
| "logps": -74.9779281616211, | |
| "loss": 0.102, | |
| "objective": 0.11110852658748627, | |
| "ranking_idealized": 0.6583333611488342, | |
| "ranking_idealized_expo": 0.512499988079071, | |
| "ranking_simple": 0.6458333134651184, | |
| "regularize": 0.11110852658748627, | |
| "step": 520 | |
| }, | |
| { | |
| "dpo_loss": 0.5326921343803406, | |
| "epoch": 2.975909305621162, | |
| "grad_norm": 7.020529204895981, | |
| "learning_rate": 2.0952075638923656e-06, | |
| "logits": -0.773563027381897, | |
| "logps": -73.9379653930664, | |
| "loss": 0.0967, | |
| "objective": 0.1049317866563797, | |
| "ranking_idealized": 0.6875, | |
| "ranking_idealized_expo": 0.5708333253860474, | |
| "ranking_simple": 0.699999988079071, | |
| "regularize": 0.1049317866563797, | |
| "step": 525 | |
| }, | |
| { | |
| "dpo_loss": 0.5151563286781311, | |
| "epoch": 3.0042512990080303, | |
| "grad_norm": 6.663633146942429, | |
| "learning_rate": 2.046361447106244e-06, | |
| "logits": -0.7806794047355652, | |
| "logps": -73.31244659423828, | |
| "loss": 0.0987, | |
| "objective": 0.10096151381731033, | |
| "ranking_idealized": 0.675000011920929, | |
| "ranking_idealized_expo": 0.512499988079071, | |
| "ranking_simple": 0.6708333492279053, | |
| "regularize": 0.10096149891614914, | |
| "step": 530 | |
| }, | |
| { | |
| "dpo_loss": 0.5298264622688293, | |
| "epoch": 3.0325932923948984, | |
| "grad_norm": 6.838429513652437, | |
| "learning_rate": 1.997693767401503e-06, | |
| "logits": -0.8067893981933594, | |
| "logps": -75.07320404052734, | |
| "loss": 0.0914, | |
| "objective": 0.09071025252342224, | |
| "ranking_idealized": 0.6708333492279053, | |
| "ranking_idealized_expo": 0.5458333492279053, | |
| "ranking_simple": 0.6708333492279053, | |
| "regularize": 0.09071025252342224, | |
| "step": 535 | |
| }, | |
| { | |
| "dpo_loss": 0.5308666825294495, | |
| "epoch": 3.0609352857817664, | |
| "grad_norm": 6.786419151482898, | |
| "learning_rate": 1.9492236680336486e-06, | |
| "logits": -0.8495451807975769, | |
| "logps": -72.48417663574219, | |
| "loss": 0.0869, | |
| "objective": 0.08104575425386429, | |
| "ranking_idealized": 0.6625000238418579, | |
| "ranking_idealized_expo": 0.5375000238418579, | |
| "ranking_simple": 0.6458333134651184, | |
| "regularize": 0.0810457393527031, | |
| "step": 540 | |
| }, | |
| { | |
| "dpo_loss": 0.5390760898590088, | |
| "epoch": 3.089277279168635, | |
| "grad_norm": 6.715619451066239, | |
| "learning_rate": 1.9009702145406728e-06, | |
| "logits": -0.7783963084220886, | |
| "logps": -73.88996887207031, | |
| "loss": 0.0873, | |
| "objective": 0.08420184254646301, | |
| "ranking_idealized": 0.637499988079071, | |
| "ranking_idealized_expo": 0.49166667461395264, | |
| "ranking_simple": 0.6208333373069763, | |
| "regularize": 0.08420184254646301, | |
| "step": 545 | |
| }, | |
| { | |
| "dpo_loss": 0.5265496373176575, | |
| "epoch": 3.117619272555503, | |
| "grad_norm": 6.799228974819442, | |
| "learning_rate": 1.852952387243698e-06, | |
| "logits": -0.6418666243553162, | |
| "logps": -73.9046401977539, | |
| "loss": 0.0878, | |
| "objective": 0.09073540568351746, | |
| "ranking_idealized": 0.699999988079071, | |
| "ranking_idealized_expo": 0.5458333492279053, | |
| "ranking_simple": 0.7041666507720947, | |
| "regularize": 0.09073540568351746, | |
| "step": 550 | |
| }, | |
| { | |
| "epoch": 3.117619272555503, | |
| "eval_dpo_loss": 0.6847736239433289, | |
| "eval_logits": -0.8292139172554016, | |
| "eval_logps": -77.63910675048828, | |
| "eval_loss": 0.44063544273376465, | |
| "eval_objective": 0.43784743547439575, | |
| "eval_ranking_idealized": 0.6570248007774353, | |
| "eval_ranking_idealized_expo": 0.5113636255264282, | |
| "eval_ranking_simple": 0.5330578684806824, | |
| "eval_regularize": 0.43784743547439575, | |
| "eval_runtime": 259.006, | |
| "eval_samples_per_second": 22.355, | |
| "eval_steps_per_second": 0.934, | |
| "step": 550 | |
| }, | |
| { | |
| "dpo_loss": 0.5151117444038391, | |
| "epoch": 3.1459612659423715, | |
| "grad_norm": 7.0486870886771955, | |
| "learning_rate": 1.8051890737811395e-06, | |
| "logits": -0.6810140013694763, | |
| "logps": -74.01383972167969, | |
| "loss": 0.0855, | |
| "objective": 0.08012814819812775, | |
| "ranking_idealized": 0.6833333373069763, | |
| "ranking_idealized_expo": 0.550000011920929, | |
| "ranking_simple": 0.6833333373069763, | |
| "regularize": 0.08012814074754715, | |
| "step": 555 | |
| }, | |
| { | |
| "dpo_loss": 0.5303942561149597, | |
| "epoch": 3.1743032593292395, | |
| "grad_norm": 6.686572075133422, | |
| "learning_rate": 1.7576990616793139e-06, | |
| "logits": -0.7435484528541565, | |
| "logps": -70.75598907470703, | |
| "loss": 0.0855, | |
| "objective": 0.07928713411092758, | |
| "ranking_idealized": 0.6666666865348816, | |
| "ranking_idealized_expo": 0.5041666626930237, | |
| "ranking_simple": 0.6416666507720947, | |
| "regularize": 0.07928712666034698, | |
| "step": 560 | |
| }, | |
| { | |
| "dpo_loss": 0.5257388353347778, | |
| "epoch": 3.2026452527161076, | |
| "grad_norm": 7.035317567433199, | |
| "learning_rate": 1.7105010309624381e-06, | |
| "logits": -0.7857434153556824, | |
| "logps": -71.99687194824219, | |
| "loss": 0.0807, | |
| "objective": 0.07652737945318222, | |
| "ranking_idealized": 0.6916666626930237, | |
| "ranking_idealized_expo": 0.5458333492279053, | |
| "ranking_simple": 0.6833333373069763, | |
| "regularize": 0.07652737945318222, | |
| "step": 565 | |
| }, | |
| { | |
| "dpo_loss": 0.5218066573143005, | |
| "epoch": 3.230987246102976, | |
| "grad_norm": 6.567722467858525, | |
| "learning_rate": 1.6636135468049122e-06, | |
| "logits": -0.7239015698432922, | |
| "logps": -72.04088592529297, | |
| "loss": 0.0814, | |
| "objective": 0.07984793186187744, | |
| "ranking_idealized": 0.6416666507720947, | |
| "ranking_idealized_expo": 0.4791666567325592, | |
| "ranking_simple": 0.6291666626930237, | |
| "regularize": 0.07984793186187744, | |
| "step": 570 | |
| }, | |
| { | |
| "dpo_loss": 0.5289373397827148, | |
| "epoch": 3.259329239489844, | |
| "grad_norm": 6.699684239267555, | |
| "learning_rate": 1.617055052228768e-06, | |
| "logits": -0.7765447497367859, | |
| "logps": -72.74471282958984, | |
| "loss": 0.0823, | |
| "objective": 0.08229862153530121, | |
| "ranking_idealized": 0.7083333134651184, | |
| "ranking_idealized_expo": 0.5041666626930237, | |
| "ranking_simple": 0.6958333253860474, | |
| "regularize": 0.08229862153530121, | |
| "step": 575 | |
| }, | |
| { | |
| "dpo_loss": 0.5433183908462524, | |
| "epoch": 3.287671232876712, | |
| "grad_norm": 6.701807007044451, | |
| "learning_rate": 1.5708438608491816e-06, | |
| "logits": -0.7891409397125244, | |
| "logps": -73.1654052734375, | |
| "loss": 0.0794, | |
| "objective": 0.07175194472074509, | |
| "ranking_idealized": 0.6666666865348816, | |
| "ranking_idealized_expo": 0.5583333373069763, | |
| "ranking_simple": 0.6499999761581421, | |
| "regularize": 0.0717519223690033, | |
| "step": 580 | |
| }, | |
| { | |
| "dpo_loss": 0.5400884747505188, | |
| "epoch": 3.3160132262635806, | |
| "grad_norm": 7.671988032753608, | |
| "learning_rate": 1.524998149670871e-06, | |
| "logits": -0.817208468914032, | |
| "logps": -74.3894271850586, | |
| "loss": 0.0807, | |
| "objective": 0.0763852447271347, | |
| "ranking_idealized": 0.6958333253860474, | |
| "ranking_idealized_expo": 0.5375000238418579, | |
| "ranking_simple": 0.6958333253860474, | |
| "regularize": 0.0763852447271347, | |
| "step": 585 | |
| }, | |
| { | |
| "dpo_loss": 0.539016604423523, | |
| "epoch": 3.3443552196504487, | |
| "grad_norm": 7.045114972527912, | |
| "learning_rate": 1.479535951938243e-06, | |
| "logits": -0.8479073643684387, | |
| "logps": -74.54483795166016, | |
| "loss": 0.0782, | |
| "objective": 0.07480078190565109, | |
| "ranking_idealized": 0.6666666865348816, | |
| "ranking_idealized_expo": 0.4958333373069763, | |
| "ranking_simple": 0.6708333492279053, | |
| "regularize": 0.07480078190565109, | |
| "step": 590 | |
| }, | |
| { | |
| "dpo_loss": 0.5274596810340881, | |
| "epoch": 3.372697213037317, | |
| "grad_norm": 6.813876249894319, | |
| "learning_rate": 1.43447515004208e-06, | |
| "logits": -0.7586421966552734, | |
| "logps": -73.8796615600586, | |
| "loss": 0.0725, | |
| "objective": 0.06939196586608887, | |
| "ranking_idealized": 0.6458333134651184, | |
| "ranking_idealized_expo": 0.5249999761581421, | |
| "ranking_simple": 0.6499999761581421, | |
| "regularize": 0.06939195841550827, | |
| "step": 595 | |
| }, | |
| { | |
| "dpo_loss": 0.507723331451416, | |
| "epoch": 3.4010392064241852, | |
| "grad_norm": 6.843883025807739, | |
| "learning_rate": 1.3898334684855647e-06, | |
| "logits": -0.7954932451248169, | |
| "logps": -73.50051879882812, | |
| "loss": 0.0719, | |
| "objective": 0.0723666176199913, | |
| "ranking_idealized": 0.6916666626930237, | |
| "ranking_idealized_expo": 0.5041666626930237, | |
| "ranking_simple": 0.6875, | |
| "regularize": 0.0723666176199913, | |
| "step": 600 | |
| }, | |
| { | |
| "epoch": 3.4010392064241852, | |
| "eval_dpo_loss": 0.6851052045822144, | |
| "eval_logits": -0.8875375390052795, | |
| "eval_logps": -77.49230194091797, | |
| "eval_loss": 0.439556360244751, | |
| "eval_objective": 0.43727535009384155, | |
| "eval_ranking_idealized": 0.6570248007774353, | |
| "eval_ranking_idealized_expo": 0.5113636255264282, | |
| "eval_ranking_simple": 0.5309917330741882, | |
| "eval_regularize": 0.43727535009384155, | |
| "eval_runtime": 260.0756, | |
| "eval_samples_per_second": 22.263, | |
| "eval_steps_per_second": 0.93, | |
| "step": 600 | |
| }, | |
| { | |
| "dpo_loss": 0.539269745349884, | |
| "epoch": 3.4293811998110533, | |
| "grad_norm": 6.836932990419662, | |
| "learning_rate": 1.3456284669124159e-06, | |
| "logits": -0.8135491013526917, | |
| "logps": -75.25065612792969, | |
| "loss": 0.072, | |
| "objective": 0.06646443903446198, | |
| "ranking_idealized": 0.6583333611488342, | |
| "ranking_idealized_expo": 0.5333333611488342, | |
| "ranking_simple": 0.6708333492279053, | |
| "regularize": 0.06646443903446198, | |
| "step": 605 | |
| }, | |
| { | |
| "dpo_loss": 0.532633364200592, | |
| "epoch": 3.4577231931979218, | |
| "grad_norm": 6.918313902636074, | |
| "learning_rate": 1.301877533199859e-06, | |
| "logits": -0.812556266784668, | |
| "logps": -73.47212219238281, | |
| "loss": 0.0726, | |
| "objective": 0.06124640628695488, | |
| "ranking_idealized": 0.6791666746139526, | |
| "ranking_idealized_expo": 0.5208333134651184, | |
| "ranking_simple": 0.675000011920929, | |
| "regularize": 0.06124639883637428, | |
| "step": 610 | |
| }, | |
| { | |
| "dpo_loss": 0.522627055644989, | |
| "epoch": 3.48606518658479, | |
| "grad_norm": 6.525966688786376, | |
| "learning_rate": 1.2585978766191726e-06, | |
| "logits": -0.8209077715873718, | |
| "logps": -74.44559478759766, | |
| "loss": 0.0694, | |
| "objective": 0.0690179094672203, | |
| "ranking_idealized": 0.6458333134651184, | |
| "ranking_idealized_expo": 0.47083333134651184, | |
| "ranking_simple": 0.6166666746139526, | |
| "regularize": 0.0690179094672203, | |
| "step": 615 | |
| }, | |
| { | |
| "dpo_loss": 0.5149637460708618, | |
| "epoch": 3.514407179971658, | |
| "grad_norm": 6.444084091552882, | |
| "learning_rate": 1.2158065210664848e-06, | |
| "logits": -0.734274685382843, | |
| "logps": -73.22693634033203, | |
| "loss": 0.0739, | |
| "objective": 0.06696704030036926, | |
| "ranking_idealized": 0.637499988079071, | |
| "ranking_idealized_expo": 0.5, | |
| "ranking_simple": 0.6333333253860474, | |
| "regularize": 0.06696703284978867, | |
| "step": 620 | |
| }, | |
| { | |
| "dpo_loss": 0.5303381085395813, | |
| "epoch": 3.5427491733585263, | |
| "grad_norm": 7.253457560302881, | |
| "learning_rate": 1.1735202983664803e-06, | |
| "logits": -0.763002336025238, | |
| "logps": -71.75556182861328, | |
| "loss": 0.0684, | |
| "objective": 0.06924500316381454, | |
| "ranking_idealized": 0.6708333492279053, | |
| "ranking_idealized_expo": 0.5249999761581421, | |
| "ranking_simple": 0.675000011920929, | |
| "regularize": 0.06924500316381454, | |
| "step": 625 | |
| }, | |
| { | |
| "dpo_loss": 0.5344926118850708, | |
| "epoch": 3.5710911667453944, | |
| "grad_norm": 7.160250572063496, | |
| "learning_rate": 1.1317558416516696e-06, | |
| "logits": -0.8248269557952881, | |
| "logps": -72.7044677734375, | |
| "loss": 0.0685, | |
| "objective": 0.06329541653394699, | |
| "ranking_idealized": 0.6333333253860474, | |
| "ranking_idealized_expo": 0.512499988079071, | |
| "ranking_simple": 0.6416666507720947, | |
| "regularize": 0.0632954090833664, | |
| "step": 630 | |
| }, | |
| { | |
| "dpo_loss": 0.5331679582595825, | |
| "epoch": 3.5994331601322624, | |
| "grad_norm": 6.5609855806488575, | |
| "learning_rate": 1.0905295788197993e-06, | |
| "logits": -0.7775312066078186, | |
| "logps": -72.57146453857422, | |
| "loss": 0.0645, | |
| "objective": 0.06953860074281693, | |
| "ranking_idealized": 0.637499988079071, | |
| "ranking_idealized_expo": 0.48750001192092896, | |
| "ranking_simple": 0.6208333373069763, | |
| "regularize": 0.06953860074281693, | |
| "step": 635 | |
| }, | |
| { | |
| "dpo_loss": 0.5081437826156616, | |
| "epoch": 3.627775153519131, | |
| "grad_norm": 6.931309552817511, | |
| "learning_rate": 1.049857726072005e-06, | |
| "logits": -0.7663463354110718, | |
| "logps": -74.27519989013672, | |
| "loss": 0.0641, | |
| "objective": 0.06706919521093369, | |
| "ranking_idealized": 0.6416666507720947, | |
| "ranking_idealized_expo": 0.4625000059604645, | |
| "ranking_simple": 0.6458333134651184, | |
| "regularize": 0.06706918776035309, | |
| "step": 640 | |
| }, | |
| { | |
| "dpo_loss": 0.5230153799057007, | |
| "epoch": 3.656117146905999, | |
| "grad_norm": 6.607222677286154, | |
| "learning_rate": 1.0097562815342215e-06, | |
| "logits": -0.7920152544975281, | |
| "logps": -71.90898132324219, | |
| "loss": 0.0644, | |
| "objective": 0.06532428413629532, | |
| "ranking_idealized": 0.6666666865348816, | |
| "ranking_idealized_expo": 0.5249999761581421, | |
| "ranking_simple": 0.6583333611488342, | |
| "regularize": 0.06532428413629532, | |
| "step": 645 | |
| }, | |
| { | |
| "dpo_loss": 0.5418646335601807, | |
| "epoch": 3.6844591402928675, | |
| "grad_norm": 6.694188599643782, | |
| "learning_rate": 9.702410189643838e-07, | |
| "logits": -0.8044089674949646, | |
| "logps": -72.86306762695312, | |
| "loss": 0.0618, | |
| "objective": 0.05348058044910431, | |
| "ranking_idealized": 0.6458333134651184, | |
| "ranking_idealized_expo": 0.5083333253860474, | |
| "ranking_simple": 0.6499999761581421, | |
| "regularize": 0.05348057672381401, | |
| "step": 650 | |
| }, | |
| { | |
| "epoch": 3.6844591402928675, | |
| "eval_dpo_loss": 0.6855461001396179, | |
| "eval_logits": -0.9103026986122131, | |
| "eval_logps": -77.18383026123047, | |
| "eval_loss": 0.4394647479057312, | |
| "eval_objective": 0.43862253427505493, | |
| "eval_ranking_idealized": 0.6570248007774353, | |
| "eval_ranking_idealized_expo": 0.5113636255264282, | |
| "eval_ranking_simple": 0.5268595218658447, | |
| "eval_regularize": 0.43862253427505493, | |
| "eval_runtime": 259.3939, | |
| "eval_samples_per_second": 22.321, | |
| "eval_steps_per_second": 0.933, | |
| "step": 650 | |
| }, | |
| { | |
| "dpo_loss": 0.5326651930809021, | |
| "epoch": 3.7128011336797355, | |
| "grad_norm": 6.747215671656663, | |
| "learning_rate": 9.313274815478698e-07, | |
| "logits": -0.8151116967201233, | |
| "logps": -72.32289123535156, | |
| "loss": 0.0682, | |
| "objective": 0.06774523854255676, | |
| "ranking_idealized": 0.6583333611488342, | |
| "ranking_idealized_expo": 0.5208333134651184, | |
| "ranking_simple": 0.6666666865348816, | |
| "regularize": 0.06774523109197617, | |
| "step": 655 | |
| }, | |
| { | |
| "dpo_loss": 0.5296677947044373, | |
| "epoch": 3.7411431270666036, | |
| "grad_norm": 6.603209874734441, | |
| "learning_rate": 8.930309757836517e-07, | |
| "logits": -0.8544061779975891, | |
| "logps": -72.76691436767578, | |
| "loss": 0.0605, | |
| "objective": 0.06562329083681107, | |
| "ranking_idealized": 0.6916666626930237, | |
| "ranking_idealized_expo": 0.5541666746139526, | |
| "ranking_simple": 0.6958333253860474, | |
| "regularize": 0.06562329083681107, | |
| "step": 660 | |
| }, | |
| { | |
| "dpo_loss": 0.5254350900650024, | |
| "epoch": 3.769485120453472, | |
| "grad_norm": 6.7420568501361, | |
| "learning_rate": 8.553665654635343e-07, | |
| "logits": -0.7711302042007446, | |
| "logps": -72.90251159667969, | |
| "loss": 0.0589, | |
| "objective": 0.05725221708416939, | |
| "ranking_idealized": 0.6958333253860474, | |
| "ranking_idealized_expo": 0.5166666507720947, | |
| "ranking_simple": 0.6791666746139526, | |
| "regularize": 0.057252202183008194, | |
| "step": 665 | |
| }, | |
| { | |
| "dpo_loss": 0.529458224773407, | |
| "epoch": 3.79782711384034, | |
| "grad_norm": 6.596226604829734, | |
| "learning_rate": 8.183490657468687e-07, | |
| "logits": -0.874411940574646, | |
| "logps": -73.00768280029297, | |
| "loss": 0.0592, | |
| "objective": 0.0637064203619957, | |
| "ranking_idealized": 0.7333333492279053, | |
| "ranking_idealized_expo": 0.5958333611488342, | |
| "ranking_simple": 0.7333333492279053, | |
| "regularize": 0.0637064203619957, | |
| "step": 670 | |
| }, | |
| { | |
| "dpo_loss": 0.5332936644554138, | |
| "epoch": 3.826169107227208, | |
| "grad_norm": 6.620708805759934, | |
| "learning_rate": 7.819930373330669e-07, | |
| "logits": -0.8352341055870056, | |
| "logps": -72.00785827636719, | |
| "loss": 0.0556, | |
| "objective": 0.05346338450908661, | |
| "ranking_idealized": 0.6708333492279053, | |
| "ranking_idealized_expo": 0.5458333492279053, | |
| "ranking_simple": 0.6666666865348816, | |
| "regularize": 0.05346338450908661, | |
| "step": 675 | |
| }, | |
| { | |
| "dpo_loss": 0.5433677434921265, | |
| "epoch": 3.8545111006140766, | |
| "grad_norm": 6.502071174130889, | |
| "learning_rate": 7.463127807341966e-07, | |
| "logits": -0.7840080261230469, | |
| "logps": -72.76580047607422, | |
| "loss": 0.0571, | |
| "objective": 0.05716581270098686, | |
| "ranking_idealized": 0.6541666388511658, | |
| "ranking_idealized_expo": 0.5083333253860474, | |
| "ranking_simple": 0.6416666507720947, | |
| "regularize": 0.057165808975696564, | |
| "step": 680 | |
| }, | |
| { | |
| "dpo_loss": 0.5243024230003357, | |
| "epoch": 3.8828530940009447, | |
| "grad_norm": 6.827039590730102, | |
| "learning_rate": 7.113223306499336e-07, | |
| "logits": -0.8465909361839294, | |
| "logps": -72.07350158691406, | |
| "loss": 0.0572, | |
| "objective": 0.053640857338905334, | |
| "ranking_idealized": 0.6875, | |
| "ranking_idealized_expo": 0.5166666507720947, | |
| "ranking_simple": 0.6791666746139526, | |
| "regularize": 0.05364084988832474, | |
| "step": 685 | |
| }, | |
| { | |
| "dpo_loss": 0.5204752087593079, | |
| "epoch": 3.9111950873878127, | |
| "grad_norm": 6.658562071017675, | |
| "learning_rate": 6.770354504470575e-07, | |
| "logits": -0.8739193677902222, | |
| "logps": -71.3963394165039, | |
| "loss": 0.0561, | |
| "objective": 0.054595671594142914, | |
| "ranking_idealized": 0.7041666507720947, | |
| "ranking_idealized_expo": 0.550000011920929, | |
| "ranking_simple": 0.6958333253860474, | |
| "regularize": 0.05459566414356232, | |
| "step": 690 | |
| }, | |
| { | |
| "dpo_loss": 0.5284795761108398, | |
| "epoch": 3.9395370807746812, | |
| "grad_norm": 6.480093534880764, | |
| "learning_rate": 6.434656267456843e-07, | |
| "logits": -0.8127073049545288, | |
| "logps": -72.72264862060547, | |
| "loss": 0.0567, | |
| "objective": 0.06189308688044548, | |
| "ranking_idealized": 0.6083333492279053, | |
| "ranking_idealized_expo": 0.46666666865348816, | |
| "ranking_simple": 0.6083333492279053, | |
| "regularize": 0.06189308688044548, | |
| "step": 695 | |
| }, | |
| { | |
| "dpo_loss": 0.5350829362869263, | |
| "epoch": 3.9678790741615493, | |
| "grad_norm": 6.7060271282278245, | |
| "learning_rate": 6.106260641143547e-07, | |
| "logits": -0.833633303642273, | |
| "logps": -73.22267150878906, | |
| "loss": 0.0551, | |
| "objective": 0.05491795390844345, | |
| "ranking_idealized": 0.612500011920929, | |
| "ranking_idealized_expo": 0.48750001192092896, | |
| "ranking_simple": 0.6166666746139526, | |
| "regularize": 0.05491795018315315, | |
| "step": 700 | |
| }, | |
| { | |
| "epoch": 3.9678790741615493, | |
| "eval_dpo_loss": 0.6859015226364136, | |
| "eval_logits": -0.9137452244758606, | |
| "eval_logps": -77.72093200683594, | |
| "eval_loss": 0.440186470746994, | |
| "eval_objective": 0.43875548243522644, | |
| "eval_ranking_idealized": 0.6570248007774353, | |
| "eval_ranking_idealized_expo": 0.5113636255264282, | |
| "eval_ranking_simple": 0.5289255976676941, | |
| "eval_regularize": 0.43875548243522644, | |
| "eval_runtime": 259.3718, | |
| "eval_samples_per_second": 22.323, | |
| "eval_steps_per_second": 0.933, | |
| "step": 700 | |
| }, | |
| { | |
| "dpo_loss": 0.5229139924049377, | |
| "epoch": 3.9962210675484178, | |
| "grad_norm": 6.704821856039977, | |
| "learning_rate": 5.785296798760601e-07, | |
| "logits": -0.7782571911811829, | |
| "logps": -72.21749877929688, | |
| "loss": 0.0552, | |
| "objective": 0.048626501113176346, | |
| "ranking_idealized": 0.6666666865348816, | |
| "ranking_idealized_expo": 0.5333333611488342, | |
| "ranking_simple": 0.6666666865348816, | |
| "regularize": 0.04862649738788605, | |
| "step": 705 | |
| }, | |
| { | |
| "dpo_loss": 0.5162150263786316, | |
| "epoch": 4.024563060935286, | |
| "grad_norm": 7.292565223355315, | |
| "learning_rate": 5.471890990272666e-07, | |
| "logits": -0.814688503742218, | |
| "logps": -73.78315734863281, | |
| "loss": 0.0446, | |
| "objective": 0.045689478516578674, | |
| "ranking_idealized": 0.6916666626930237, | |
| "ranking_idealized_expo": 0.4791666567325592, | |
| "ranking_simple": 0.675000011920929, | |
| "regularize": 0.04568947106599808, | |
| "step": 710 | |
| }, | |
| { | |
| "dpo_loss": 0.5273666977882385, | |
| "epoch": 4.052905054322154, | |
| "grad_norm": 6.71901500064993, | |
| "learning_rate": 5.166166492719124e-07, | |
| "logits": -0.7857570648193359, | |
| "logps": -72.7701187133789, | |
| "loss": 0.0422, | |
| "objective": 0.04041726142168045, | |
| "ranking_idealized": 0.6833333373069763, | |
| "ranking_idealized_expo": 0.5416666865348816, | |
| "ranking_simple": 0.675000011920929, | |
| "regularize": 0.04041723534464836, | |
| "step": 715 | |
| }, | |
| { | |
| "dpo_loss": 0.5177367329597473, | |
| "epoch": 4.081247047709022, | |
| "grad_norm": 6.659794774093643, | |
| "learning_rate": 4.868243561723535e-07, | |
| "logits": -0.7260258793830872, | |
| "logps": -75.34712219238281, | |
| "loss": 0.0448, | |
| "objective": 0.041196659207344055, | |
| "ranking_idealized": 0.6583333611488342, | |
| "ranking_idealized_expo": 0.5208333134651184, | |
| "ranking_simple": 0.6583333611488342, | |
| "regularize": 0.04119665548205376, | |
| "step": 720 | |
| }, | |
| { | |
| "dpo_loss": 0.5408446192741394, | |
| "epoch": 4.109589041095891, | |
| "grad_norm": 6.565784729673605, | |
| "learning_rate": 4.57823938419153e-07, | |
| "logits": -0.786972165107727, | |
| "logps": -72.7632064819336, | |
| "loss": 0.0445, | |
| "objective": 0.04578384384512901, | |
| "ranking_idealized": 0.6541666388511658, | |
| "ranking_idealized_expo": 0.5375000238418579, | |
| "ranking_simple": 0.6541666388511658, | |
| "regularize": 0.045783836394548416, | |
| "step": 725 | |
| }, | |
| { | |
| "dpo_loss": 0.5255261659622192, | |
| "epoch": 4.137931034482759, | |
| "grad_norm": 6.9190144367452975, | |
| "learning_rate": 4.2962680322157335e-07, | |
| "logits": -0.8668403625488281, | |
| "logps": -72.80368041992188, | |
| "loss": 0.0469, | |
| "objective": 0.0492975153028965, | |
| "ranking_idealized": 0.675000011920929, | |
| "ranking_idealized_expo": 0.5375000238418579, | |
| "ranking_simple": 0.675000011920929, | |
| "regularize": 0.04929749667644501, | |
| "step": 730 | |
| }, | |
| { | |
| "dpo_loss": 0.5357745885848999, | |
| "epoch": 4.166273027869627, | |
| "grad_norm": 6.713218850119101, | |
| "learning_rate": 4.0224404182059443e-07, | |
| "logits": -0.7645056843757629, | |
| "logps": -74.07119750976562, | |
| "loss": 0.04, | |
| "objective": 0.04291637986898422, | |
| "ranking_idealized": 0.6791666746139526, | |
| "ranking_idealized_expo": 0.5291666388511658, | |
| "ranking_simple": 0.6791666746139526, | |
| "regularize": 0.042916372418403625, | |
| "step": 735 | |
| }, | |
| { | |
| "dpo_loss": 0.5108519196510315, | |
| "epoch": 4.194615021256495, | |
| "grad_norm": 6.802036362247038, | |
| "learning_rate": 3.756864251262143e-07, | |
| "logits": -0.8028141260147095, | |
| "logps": -73.28968048095703, | |
| "loss": 0.0412, | |
| "objective": 0.044031720608472824, | |
| "ranking_idealized": 0.6791666746139526, | |
| "ranking_idealized_expo": 0.5208333134651184, | |
| "ranking_simple": 0.6708333492279053, | |
| "regularize": 0.04403171315789223, | |
| "step": 740 | |
| }, | |
| { | |
| "dpo_loss": 0.5228927731513977, | |
| "epoch": 4.222957014643363, | |
| "grad_norm": 6.653699462048885, | |
| "learning_rate": 3.499643994807486e-07, | |
| "logits": -0.8697967529296875, | |
| "logps": -70.57527160644531, | |
| "loss": 0.0428, | |
| "objective": 0.04283791035413742, | |
| "ranking_idealized": 0.6291666626930237, | |
| "ranking_idealized_expo": 0.48750001192092896, | |
| "ranking_simple": 0.6416666507720947, | |
| "regularize": 0.042837902903556824, | |
| "step": 745 | |
| }, | |
| { | |
| "dpo_loss": 0.5164041519165039, | |
| "epoch": 4.251299008030231, | |
| "grad_norm": 6.911061908397223, | |
| "learning_rate": 3.250880825498026e-07, | |
| "logits": -0.9160488843917847, | |
| "logps": -72.56361389160156, | |
| "loss": 0.0388, | |
| "objective": 0.03974687308073044, | |
| "ranking_idealized": 0.6458333134651184, | |
| "ranking_idealized_expo": 0.5291666388511658, | |
| "ranking_simple": 0.6541666388511658, | |
| "regularize": 0.03974686935544014, | |
| "step": 750 | |
| }, | |
| { | |
| "epoch": 4.251299008030231, | |
| "eval_dpo_loss": 0.685931921005249, | |
| "eval_logits": -0.8975909352302551, | |
| "eval_logps": -77.07003021240234, | |
| "eval_loss": 0.4403546452522278, | |
| "eval_objective": 0.438612699508667, | |
| "eval_ranking_idealized": 0.6570248007774353, | |
| "eval_ranking_idealized_expo": 0.5113636255264282, | |
| "eval_ranking_simple": 0.5309917330741882, | |
| "eval_regularize": 0.438612699508667, | |
| "eval_runtime": 259.3145, | |
| "eval_samples_per_second": 22.328, | |
| "eval_steps_per_second": 0.933, | |
| "step": 750 | |
| }, | |
| { | |
| "dpo_loss": 0.5212615132331848, | |
| "epoch": 4.2796410014171, | |
| "grad_norm": 6.600495039472016, | |
| "learning_rate": 3.0106725934252095e-07, | |
| "logits": -0.8632883429527283, | |
| "logps": -71.9403076171875, | |
| "loss": 0.0401, | |
| "objective": 0.039546623826026917, | |
| "ranking_idealized": 0.6791666746139526, | |
| "ranking_idealized_expo": 0.5333333611488342, | |
| "ranking_simple": 0.6708333492279053, | |
| "regularize": 0.03954662010073662, | |
| "step": 755 | |
| }, | |
| { | |
| "dpo_loss": 0.5251290798187256, | |
| "epoch": 4.307982994803968, | |
| "grad_norm": 6.496157144727595, | |
| "learning_rate": 2.779113783626916e-07, | |
| "logits": -0.8375190496444702, | |
| "logps": -73.24321746826172, | |
| "loss": 0.0397, | |
| "objective": 0.03997815027832985, | |
| "ranking_idealized": 0.7083333134651184, | |
| "ranking_idealized_expo": 0.5666666626930237, | |
| "ranking_simple": 0.6958333253860474, | |
| "regularize": 0.039978139102458954, | |
| "step": 760 | |
| }, | |
| { | |
| "dpo_loss": 0.5071607828140259, | |
| "epoch": 4.336324988190836, | |
| "grad_norm": 6.910390218786043, | |
| "learning_rate": 2.5562954789221164e-07, | |
| "logits": -0.8520547747612, | |
| "logps": -73.03939056396484, | |
| "loss": 0.0405, | |
| "objective": 0.036770131438970566, | |
| "ranking_idealized": 0.7291666865348816, | |
| "ranking_idealized_expo": 0.5291666388511658, | |
| "ranking_simple": 0.7291666865348816, | |
| "regularize": 0.03677012771368027, | |
| "step": 765 | |
| }, | |
| { | |
| "dpo_loss": 0.509007453918457, | |
| "epoch": 4.364666981577704, | |
| "grad_norm": 6.462624726733492, | |
| "learning_rate": 2.3423053240837518e-07, | |
| "logits": -0.7884809970855713, | |
| "logps": -71.66094207763672, | |
| "loss": 0.0396, | |
| "objective": 0.03581638261675835, | |
| "ranking_idealized": 0.6666666865348816, | |
| "ranking_idealized_expo": 0.5083333253860474, | |
| "ranking_simple": 0.6666666865348816, | |
| "regularize": 0.03581637516617775, | |
| "step": 770 | |
| }, | |
| { | |
| "dpo_loss": 0.5204900503158569, | |
| "epoch": 4.393008974964572, | |
| "grad_norm": 6.9115021717677445, | |
| "learning_rate": 2.137227491364016e-07, | |
| "logits": -0.8254011869430542, | |
| "logps": -72.81352233886719, | |
| "loss": 0.0382, | |
| "objective": 0.03518719598650932, | |
| "ranking_idealized": 0.6666666865348816, | |
| "ranking_idealized_expo": 0.512499988079071, | |
| "ranking_simple": 0.6583333611488342, | |
| "regularize": 0.03518717736005783, | |
| "step": 775 | |
| }, | |
| { | |
| "dpo_loss": 0.5285161733627319, | |
| "epoch": 4.42135096835144, | |
| "grad_norm": 6.553973237097607, | |
| "learning_rate": 1.941142647385469e-07, | |
| "logits": -0.843854546546936, | |
| "logps": -71.42961120605469, | |
| "loss": 0.0393, | |
| "objective": 0.03610792011022568, | |
| "ranking_idealized": 0.6541666388511658, | |
| "ranking_idealized_expo": 0.5041666626930237, | |
| "ranking_simple": 0.6541666388511658, | |
| "regularize": 0.03610791638493538, | |
| "step": 780 | |
| }, | |
| { | |
| "dpo_loss": 0.5424137115478516, | |
| "epoch": 4.449692961738309, | |
| "grad_norm": 6.303662663640985, | |
| "learning_rate": 1.7541279214111277e-07, | |
| "logits": -0.8690065741539001, | |
| "logps": -71.59990692138672, | |
| "loss": 0.0409, | |
| "objective": 0.0452335849404335, | |
| "ranking_idealized": 0.6583333611488342, | |
| "ranking_idealized_expo": 0.5291666388511658, | |
| "ranking_simple": 0.6416666507720947, | |
| "regularize": 0.04523357003927231, | |
| "step": 785 | |
| }, | |
| { | |
| "dpo_loss": 0.5163858532905579, | |
| "epoch": 4.478034955125177, | |
| "grad_norm": 6.732149069961477, | |
| "learning_rate": 1.5762568750059604e-07, | |
| "logits": -0.8400804400444031, | |
| "logps": -73.97518920898438, | |
| "loss": 0.0369, | |
| "objective": 0.042594779282808304, | |
| "ranking_idealized": 0.7291666865348816, | |
| "ranking_idealized_expo": 0.5583333373069763, | |
| "ranking_simple": 0.7124999761581421, | |
| "regularize": 0.04259476438164711, | |
| "step": 790 | |
| }, | |
| { | |
| "dpo_loss": 0.523327112197876, | |
| "epoch": 4.506376948512045, | |
| "grad_norm": 6.468243868538749, | |
| "learning_rate": 1.4075994731016895e-07, | |
| "logits": -0.7410406470298767, | |
| "logps": -74.17599487304688, | |
| "loss": 0.037, | |
| "objective": 0.03256476670503616, | |
| "ranking_idealized": 0.6541666388511658, | |
| "ranking_idealized_expo": 0.550000011920929, | |
| "ranking_simple": 0.6499999761581421, | |
| "regularize": 0.03256473317742348, | |
| "step": 795 | |
| }, | |
| { | |
| "dpo_loss": 0.5217226147651672, | |
| "epoch": 4.534718941898913, | |
| "grad_norm": 6.810051199376682, | |
| "learning_rate": 1.2482220564763669e-07, | |
| "logits": -0.7353635430335999, | |
| "logps": -72.93279266357422, | |
| "loss": 0.0382, | |
| "objective": 0.03855961933732033, | |
| "ranking_idealized": 0.6875, | |
| "ranking_idealized_expo": 0.512499988079071, | |
| "ranking_simple": 0.6916666626930237, | |
| "regularize": 0.03855961188673973, | |
| "step": 800 | |
| }, | |
| { | |
| "epoch": 4.534718941898913, | |
| "eval_dpo_loss": 0.6858941912651062, | |
| "eval_logits": -0.8972411155700684, | |
| "eval_logps": -77.24726867675781, | |
| "eval_loss": 0.4401608407497406, | |
| "eval_objective": 0.4384419023990631, | |
| "eval_ranking_idealized": 0.6570248007774353, | |
| "eval_ranking_idealized_expo": 0.5113636255264282, | |
| "eval_ranking_simple": 0.5320248007774353, | |
| "eval_regularize": 0.4384419023990631, | |
| "eval_runtime": 259.7863, | |
| "eval_samples_per_second": 22.288, | |
| "eval_steps_per_second": 0.932, | |
| "step": 800 | |
| }, | |
| { | |
| "dpo_loss": 0.5223442316055298, | |
| "epoch": 4.563060935285781, | |
| "grad_norm": 6.589373819397174, | |
| "learning_rate": 1.0981873156594381e-07, | |
| "logits": -0.8119328022003174, | |
| "logps": -71.91402435302734, | |
| "loss": 0.0371, | |
| "objective": 0.03391700237989426, | |
| "ranking_idealized": 0.6458333134651184, | |
| "ranking_idealized_expo": 0.46666666865348816, | |
| "ranking_simple": 0.6541666388511658, | |
| "regularize": 0.03391699120402336, | |
| "step": 805 | |
| }, | |
| { | |
| "dpo_loss": 0.5284490585327148, | |
| "epoch": 4.59140292867265, | |
| "grad_norm": 6.877467315737168, | |
| "learning_rate": 9.575542662726756e-08, | |
| "logits": -0.8636207580566406, | |
| "logps": -71.29000854492188, | |
| "loss": 0.0362, | |
| "objective": 0.034173477441072464, | |
| "ranking_idealized": 0.5874999761581421, | |
| "ranking_idealized_expo": 0.4625000059604645, | |
| "ranking_simple": 0.6000000238418579, | |
| "regularize": 0.034173473715782166, | |
| "step": 810 | |
| }, | |
| { | |
| "dpo_loss": 0.532176673412323, | |
| "epoch": 4.619744922059518, | |
| "grad_norm": 6.410472181963987, | |
| "learning_rate": 8.26378225816582e-08, | |
| "logits": -0.7376688122749329, | |
| "logps": -72.42508697509766, | |
| "loss": 0.0338, | |
| "objective": 0.03282972797751427, | |
| "ranking_idealized": 0.7250000238418579, | |
| "ranking_idealized_expo": 0.5916666388511658, | |
| "ranking_simple": 0.737500011920929, | |
| "regularize": 0.03282969817519188, | |
| "step": 815 | |
| }, | |
| { | |
| "dpo_loss": 0.5355073809623718, | |
| "epoch": 4.648086915446386, | |
| "grad_norm": 6.840519860011787, | |
| "learning_rate": 7.047107919114588e-08, | |
| "logits": -0.8241658806800842, | |
| "logps": -72.70903778076172, | |
| "loss": 0.0329, | |
| "objective": 0.029066000133752823, | |
| "ranking_idealized": 0.6875, | |
| "ranking_idealized_expo": 0.5625, | |
| "ranking_simple": 0.6875, | |
| "regularize": 0.029065988957881927, | |
| "step": 820 | |
| }, | |
| { | |
| "dpo_loss": 0.5363853573799133, | |
| "epoch": 4.6764289088332545, | |
| "grad_norm": 6.5036481206185615, | |
| "learning_rate": 5.92599822001666e-08, | |
| "logits": -0.7593883872032166, | |
| "logps": -70.98949432373047, | |
| "loss": 0.0362, | |
| "objective": 0.03307075425982475, | |
| "ranking_idealized": 0.6166666746139526, | |
| "ranking_idealized_expo": 0.5, | |
| "ranking_simple": 0.6041666865348816, | |
| "regularize": 0.03307074308395386, | |
| "step": 825 | |
| }, | |
| { | |
| "dpo_loss": 0.5210588574409485, | |
| "epoch": 4.7047709022201225, | |
| "grad_norm": 6.54657496412036, | |
| "learning_rate": 4.9008941453107527e-08, | |
| "logits": -0.882694661617279, | |
| "logps": -72.7275390625, | |
| "loss": 0.0382, | |
| "objective": 0.04189787432551384, | |
| "ranking_idealized": 0.675000011920929, | |
| "ranking_idealized_expo": 0.5166666507720947, | |
| "ranking_simple": 0.6875, | |
| "regularize": 0.04189785197377205, | |
| "step": 830 | |
| }, | |
| { | |
| "dpo_loss": 0.5095342993736267, | |
| "epoch": 4.733112895606991, | |
| "grad_norm": 6.620950665910206, | |
| "learning_rate": 3.972198915970976e-08, | |
| "logits": -0.8053682446479797, | |
| "logps": -73.1785888671875, | |
| "loss": 0.0356, | |
| "objective": 0.03564568608999252, | |
| "ranking_idealized": 0.6499999761581421, | |
| "ranking_idealized_expo": 0.48750001192092896, | |
| "ranking_simple": 0.6416666507720947, | |
| "regularize": 0.03564564883708954, | |
| "step": 835 | |
| }, | |
| { | |
| "dpo_loss": 0.5004899501800537, | |
| "epoch": 4.7614548889938595, | |
| "grad_norm": 6.845348700884689, | |
| "learning_rate": 3.1402778309014284e-08, | |
| "logits": -0.8175690174102783, | |
| "logps": -73.01248168945312, | |
| "loss": 0.0343, | |
| "objective": 0.030798058956861496, | |
| "ranking_idealized": 0.6958333253860474, | |
| "ranking_idealized_expo": 0.5541666746139526, | |
| "ranking_simple": 0.6875, | |
| "regularize": 0.03079797886312008, | |
| "step": 840 | |
| }, | |
| { | |
| "dpo_loss": 0.5335594415664673, | |
| "epoch": 4.7897968823807275, | |
| "grad_norm": 6.869470340595315, | |
| "learning_rate": 2.4054581232470785e-08, | |
| "logits": -0.823119044303894, | |
| "logps": -72.64087677001953, | |
| "loss": 0.0337, | |
| "objective": 0.03280922397971153, | |
| "ranking_idealized": 0.6416666507720947, | |
| "ranking_idealized_expo": 0.4583333432674408, | |
| "ranking_simple": 0.6416666507720947, | |
| "regularize": 0.032809216529130936, | |
| "step": 845 | |
| }, | |
| { | |
| "dpo_loss": 0.5230380892753601, | |
| "epoch": 4.818138875767596, | |
| "grad_norm": 6.728592904487897, | |
| "learning_rate": 1.768028831677926e-08, | |
| "logits": -0.8133633732795715, | |
| "logps": -71.71713256835938, | |
| "loss": 0.032, | |
| "objective": 0.028577150776982307, | |
| "ranking_idealized": 0.6833333373069763, | |
| "ranking_idealized_expo": 0.5291666388511658, | |
| "ranking_simple": 0.6833333373069763, | |
| "regularize": 0.028577139601111412, | |
| "step": 850 | |
| }, | |
| { | |
| "epoch": 4.818138875767596, | |
| "eval_dpo_loss": 0.686019778251648, | |
| "eval_logits": -0.8982982635498047, | |
| "eval_logps": -77.20530700683594, | |
| "eval_loss": 0.440197616815567, | |
| "eval_objective": 0.4385392367839813, | |
| "eval_ranking_idealized": 0.6570248007774353, | |
| "eval_ranking_idealized_expo": 0.5113636255264282, | |
| "eval_ranking_simple": 0.5320248007774353, | |
| "eval_regularize": 0.4385392367839813, | |
| "eval_runtime": 259.2486, | |
| "eval_samples_per_second": 22.334, | |
| "eval_steps_per_second": 0.933, | |
| "step": 850 | |
| }, | |
| { | |
| "dpo_loss": 0.533964216709137, | |
| "epoch": 4.846480869154464, | |
| "grad_norm": 6.460809600085337, | |
| "learning_rate": 1.2282406866966078e-08, | |
| "logits": -0.7880451679229736, | |
| "logps": -72.03130340576172, | |
| "loss": 0.033, | |
| "objective": 0.03257838636636734, | |
| "ranking_idealized": 0.7083333134651184, | |
| "ranking_idealized_expo": 0.5375000238418579, | |
| "ranking_simple": 0.7083333134651184, | |
| "regularize": 0.032578371465206146, | |
| "step": 855 | |
| }, | |
| { | |
| "dpo_loss": 0.5133672952651978, | |
| "epoch": 4.874822862541333, | |
| "grad_norm": 6.904438097045362, | |
| "learning_rate": 7.863060120144316e-09, | |
| "logits": -0.7785756587982178, | |
| "logps": -71.80149841308594, | |
| "loss": 0.034, | |
| "objective": 0.03182807192206383, | |
| "ranking_idealized": 0.7166666388511658, | |
| "ranking_idealized_expo": 0.550000011920929, | |
| "ranking_simple": 0.7124999761581421, | |
| "regularize": 0.031828057020902634, | |
| "step": 860 | |
| }, | |
| { | |
| "dpo_loss": 0.5214442014694214, | |
| "epoch": 4.903164855928201, | |
| "grad_norm": 6.578317256063193, | |
| "learning_rate": 4.423986410346526e-09, | |
| "logits": -0.8345638513565063, | |
| "logps": -70.81376647949219, | |
| "loss": 0.0332, | |
| "objective": 0.037510212510824203, | |
| "ranking_idealized": 0.6625000238418579, | |
| "ranking_idealized_expo": 0.4958333373069763, | |
| "ranking_simple": 0.6583333611488342, | |
| "regularize": 0.03751020506024361, | |
| "step": 865 | |
| }, | |
| { | |
| "dpo_loss": 0.5174622535705566, | |
| "epoch": 4.931506849315069, | |
| "grad_norm": 6.93105315931173, | |
| "learning_rate": 1.9665384847583622e-09, | |
| "logits": -0.8424772024154663, | |
| "logps": -72.85356903076172, | |
| "loss": 0.0344, | |
| "objective": 0.0343938022851944, | |
| "ranking_idealized": 0.6916666626930237, | |
| "ranking_idealized_expo": 0.512499988079071, | |
| "ranking_simple": 0.6875, | |
| "regularize": 0.034393779933452606, | |
| "step": 870 | |
| }, | |
| { | |
| "dpo_loss": 0.524601399898529, | |
| "epoch": 4.959848842701937, | |
| "grad_norm": 6.733993703356916, | |
| "learning_rate": 4.916829716183901e-10, | |
| "logits": -0.790997326374054, | |
| "logps": -72.49830627441406, | |
| "loss": 0.0325, | |
| "objective": 0.03138989955186844, | |
| "ranking_idealized": 0.6833333373069763, | |
| "ranking_idealized_expo": 0.5458333492279053, | |
| "ranking_simple": 0.675000011920929, | |
| "regularize": 0.03138989210128784, | |
| "step": 875 | |
| }, | |
| { | |
| "dpo_loss": 0.5205584764480591, | |
| "epoch": 4.988190836088805, | |
| "grad_norm": 6.678618993937897, | |
| "learning_rate": 0.0, | |
| "logits": -0.8767089247703552, | |
| "logps": -72.85419464111328, | |
| "loss": 0.0328, | |
| "objective": 0.030487608164548874, | |
| "ranking_idealized": 0.6708333492279053, | |
| "ranking_idealized_expo": 0.5291666388511658, | |
| "ranking_simple": 0.6625000238418579, | |
| "regularize": 0.030487585812807083, | |
| "step": 880 | |
| }, | |
| { | |
| "epoch": 4.988190836088805, | |
| "step": 880, | |
| "total_flos": 0.0, | |
| "train_loss": 0.15004684163088147, | |
| "train_runtime": 35354.4588, | |
| "train_samples_per_second": 7.185, | |
| "train_steps_per_second": 0.025 | |
| } | |
| ], | |
| "logging_steps": 5, | |
| "max_steps": 880, | |
| "num_input_tokens_seen": 0, | |
| "num_train_epochs": 5, | |
| "save_steps": 500, | |
| "stateful_callbacks": { | |
| "TrainerControl": { | |
| "args": { | |
| "should_epoch_stop": false, | |
| "should_evaluate": false, | |
| "should_log": false, | |
| "should_save": false, | |
| "should_training_stop": false | |
| }, | |
| "attributes": {} | |
| } | |
| }, | |
| "total_flos": 0.0, | |
| "train_batch_size": 4, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |