| { | |
| "best_metric": null, | |
| "best_model_checkpoint": null, | |
| "epoch": 0.9962264150943396, | |
| "eval_steps": 500, | |
| "global_step": 99, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "epoch": 0.010062893081761006, | |
| "grad_norm": 15.443952917965191, | |
| "learning_rate": 1e-07, | |
| "logps/chosen": -77.71961212158203, | |
| "logps/rejected": -77.03057861328125, | |
| "loss": 1.0499, | |
| "losses/dpo": 0.6931471824645996, | |
| "losses/sft": 0.3425563871860504, | |
| "losses/total": 1.0357035398483276, | |
| "ref_logps/chosen": -77.71961212158203, | |
| "ref_logps/rejected": -77.03057861328125, | |
| "rewards/accuracies": 0.0, | |
| "rewards/chosen": 0.0, | |
| "rewards/margins": 0.0, | |
| "rewards/rejected": 0.0, | |
| "step": 1 | |
| }, | |
| { | |
| "epoch": 0.02012578616352201, | |
| "grad_norm": 14.525541091089869, | |
| "learning_rate": 2e-07, | |
| "logps/chosen": -81.42757415771484, | |
| "logps/rejected": -74.36381530761719, | |
| "loss": 1.0697, | |
| "losses/dpo": 0.6931471824645996, | |
| "losses/sft": 0.34544649720191956, | |
| "losses/total": 1.0385937690734863, | |
| "ref_logps/chosen": -81.42757415771484, | |
| "ref_logps/rejected": -74.36381530761719, | |
| "rewards/accuracies": 0.0, | |
| "rewards/chosen": 0.0, | |
| "rewards/margins": 0.0, | |
| "rewards/rejected": 0.0, | |
| "step": 2 | |
| }, | |
| { | |
| "epoch": 0.03018867924528302, | |
| "grad_norm": 14.972861178344003, | |
| "learning_rate": 3e-07, | |
| "logps/chosen": -77.72991943359375, | |
| "logps/rejected": -78.03253936767578, | |
| "loss": 1.0672, | |
| "losses/dpo": 0.7052053809165955, | |
| "losses/sft": 0.3688685894012451, | |
| "losses/total": 1.0740740299224854, | |
| "ref_logps/chosen": -77.74811553955078, | |
| "ref_logps/rejected": -78.08845520019531, | |
| "rewards/accuracies": 0.546875, | |
| "rewards/chosen": 0.0018183797365054488, | |
| "rewards/margins": -0.003773280419409275, | |
| "rewards/rejected": 0.00559166120365262, | |
| "step": 3 | |
| }, | |
| { | |
| "epoch": 0.04025157232704402, | |
| "grad_norm": 14.651655373074027, | |
| "learning_rate": 4e-07, | |
| "logps/chosen": -73.76988983154297, | |
| "logps/rejected": -74.19403839111328, | |
| "loss": 1.0523, | |
| "losses/dpo": 0.6988011598587036, | |
| "losses/sft": 0.3054599463939667, | |
| "losses/total": 1.0042610168457031, | |
| "ref_logps/chosen": -73.7587890625, | |
| "ref_logps/rejected": -74.23301696777344, | |
| "rewards/accuracies": 0.4609375, | |
| "rewards/chosen": -0.00110996060539037, | |
| "rewards/margins": -0.00500828679651022, | |
| "rewards/rejected": 0.0038983263075351715, | |
| "step": 4 | |
| }, | |
| { | |
| "epoch": 0.050314465408805034, | |
| "grad_norm": 15.110967055469406, | |
| "learning_rate": 5e-07, | |
| "logps/chosen": -77.6102523803711, | |
| "logps/rejected": -72.1615982055664, | |
| "loss": 1.0563, | |
| "losses/dpo": 0.6893635988235474, | |
| "losses/sft": 0.3531861901283264, | |
| "losses/total": 1.042549729347229, | |
| "ref_logps/chosen": -77.61317443847656, | |
| "ref_logps/rejected": -72.10652923583984, | |
| "rewards/accuracies": 0.5390625, | |
| "rewards/chosen": 0.0002926290035247803, | |
| "rewards/margins": 0.005799999460577965, | |
| "rewards/rejected": -0.005507371388375759, | |
| "step": 5 | |
| }, | |
| { | |
| "epoch": 0.06037735849056604, | |
| "grad_norm": 14.98273253406139, | |
| "learning_rate": 4.998603909325637e-07, | |
| "logps/chosen": -82.17599487304688, | |
| "logps/rejected": -75.54823303222656, | |
| "loss": 1.0568, | |
| "losses/dpo": 0.6767082214355469, | |
| "losses/sft": 0.42767006158828735, | |
| "losses/total": 1.104378342628479, | |
| "ref_logps/chosen": -82.23617553710938, | |
| "ref_logps/rejected": -75.60321807861328, | |
| "rewards/accuracies": 0.4921875, | |
| "rewards/chosen": 0.006017628125846386, | |
| "rewards/margins": 0.0005183331668376923, | |
| "rewards/rejected": 0.005499294493347406, | |
| "step": 6 | |
| }, | |
| { | |
| "epoch": 0.07044025157232704, | |
| "grad_norm": 15.812032171234971, | |
| "learning_rate": 4.994417196557883e-07, | |
| "logps/chosen": -80.02710723876953, | |
| "logps/rejected": -74.0862045288086, | |
| "loss": 1.0599, | |
| "losses/dpo": 0.6971860527992249, | |
| "losses/sft": 0.36510100960731506, | |
| "losses/total": 1.0622870922088623, | |
| "ref_logps/chosen": -80.04718017578125, | |
| "ref_logps/rejected": -74.11724090576172, | |
| "rewards/accuracies": 0.5234375, | |
| "rewards/chosen": 0.002006138674914837, | |
| "rewards/margins": -0.0010970293078571558, | |
| "rewards/rejected": 0.0031031689140945673, | |
| "step": 7 | |
| }, | |
| { | |
| "epoch": 0.08050314465408805, | |
| "grad_norm": 16.026806076613063, | |
| "learning_rate": 4.98744453772126e-07, | |
| "logps/chosen": -81.10508728027344, | |
| "logps/rejected": -72.62604522705078, | |
| "loss": 1.0513, | |
| "losses/dpo": 0.6706422567367554, | |
| "losses/sft": 0.32770049571990967, | |
| "losses/total": 0.998342752456665, | |
| "ref_logps/chosen": -81.17386627197266, | |
| "ref_logps/rejected": -72.63655090332031, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": 0.006877122446894646, | |
| "rewards/margins": 0.005827218294143677, | |
| "rewards/rejected": 0.0010499038035050035, | |
| "step": 8 | |
| }, | |
| { | |
| "epoch": 0.09056603773584905, | |
| "grad_norm": 14.880720919169196, | |
| "learning_rate": 4.97769372038695e-07, | |
| "logps/chosen": -82.00043487548828, | |
| "logps/rejected": -77.90165710449219, | |
| "loss": 1.0566, | |
| "losses/dpo": 0.6756574511528015, | |
| "losses/sft": 0.3787515163421631, | |
| "losses/total": 1.0544090270996094, | |
| "ref_logps/chosen": -82.00239562988281, | |
| "ref_logps/rejected": -77.88880157470703, | |
| "rewards/accuracies": 0.5234375, | |
| "rewards/chosen": 0.0001968789438251406, | |
| "rewards/margins": 0.001483186730183661, | |
| "rewards/rejected": -0.0012863080482929945, | |
| "step": 9 | |
| }, | |
| { | |
| "epoch": 0.10062893081761007, | |
| "grad_norm": 15.517516535710937, | |
| "learning_rate": 4.965175634975071e-07, | |
| "logps/chosen": -88.80177307128906, | |
| "logps/rejected": -80.90895080566406, | |
| "loss": 1.0859, | |
| "losses/dpo": 0.7250989675521851, | |
| "losses/sft": 0.3838391602039337, | |
| "losses/total": 1.108938217163086, | |
| "ref_logps/chosen": -88.875732421875, | |
| "ref_logps/rejected": -80.98503112792969, | |
| "rewards/accuracies": 0.5390625, | |
| "rewards/chosen": 0.007396275643259287, | |
| "rewards/margins": -0.00021131080575287342, | |
| "rewards/rejected": 0.007607587147504091, | |
| "step": 10 | |
| }, | |
| { | |
| "epoch": 0.11069182389937107, | |
| "grad_norm": 14.958071998861861, | |
| "learning_rate": 4.949904262591467e-07, | |
| "logps/chosen": -80.21826171875, | |
| "logps/rejected": -73.97025299072266, | |
| "loss": 1.0449, | |
| "losses/dpo": 0.6753466725349426, | |
| "losses/sft": 0.40270546078681946, | |
| "losses/total": 1.0780521631240845, | |
| "ref_logps/chosen": -80.33733367919922, | |
| "ref_logps/rejected": -74.07726287841797, | |
| "rewards/accuracies": 0.5078125, | |
| "rewards/chosen": 0.011908243410289288, | |
| "rewards/margins": 0.0012067598290741444, | |
| "rewards/rejected": 0.01070148404687643, | |
| "step": 11 | |
| }, | |
| { | |
| "epoch": 0.12075471698113208, | |
| "grad_norm": 15.554570066549323, | |
| "learning_rate": 4.931896659412593e-07, | |
| "logps/chosen": -82.6167221069336, | |
| "logps/rejected": -79.10201263427734, | |
| "loss": 1.0498, | |
| "losses/dpo": 0.6855930685997009, | |
| "losses/sft": 0.37060120701789856, | |
| "losses/total": 1.0561943054199219, | |
| "ref_logps/chosen": -82.68681335449219, | |
| "ref_logps/rejected": -79.13915252685547, | |
| "rewards/accuracies": 0.515625, | |
| "rewards/chosen": 0.007009676657617092, | |
| "rewards/margins": 0.0032958725932985544, | |
| "rewards/rejected": 0.003713802434504032, | |
| "step": 12 | |
| }, | |
| { | |
| "epoch": 0.13081761006289308, | |
| "grad_norm": 15.575888738679977, | |
| "learning_rate": 4.911172937635942e-07, | |
| "logps/chosen": -78.73028564453125, | |
| "logps/rejected": -77.23632049560547, | |
| "loss": 1.0541, | |
| "losses/dpo": 0.703624963760376, | |
| "losses/sft": 0.3454435467720032, | |
| "losses/total": 1.049068570137024, | |
| "ref_logps/chosen": -78.94412231445312, | |
| "ref_logps/rejected": -77.30516815185547, | |
| "rewards/accuracies": 0.5234375, | |
| "rewards/chosen": 0.021384168416261673, | |
| "rewards/margins": 0.014499638229608536, | |
| "rewards/rejected": 0.006884530186653137, | |
| "step": 13 | |
| }, | |
| { | |
| "epoch": 0.14088050314465408, | |
| "grad_norm": 16.866802733593584, | |
| "learning_rate": 4.88775624301728e-07, | |
| "logps/chosen": -85.04679870605469, | |
| "logps/rejected": -71.76718139648438, | |
| "loss": 1.0699, | |
| "losses/dpo": 0.7083428502082825, | |
| "losses/sft": 0.38208234310150146, | |
| "losses/total": 1.0904252529144287, | |
| "ref_logps/chosen": -85.30770111083984, | |
| "ref_logps/rejected": -71.88570404052734, | |
| "rewards/accuracies": 0.5859375, | |
| "rewards/chosen": 0.02608950436115265, | |
| "rewards/margins": 0.014237894676625729, | |
| "rewards/rejected": 0.011851610615849495, | |
| "step": 14 | |
| }, | |
| { | |
| "epoch": 0.1509433962264151, | |
| "grad_norm": 15.574664831922826, | |
| "learning_rate": 4.861672729019796e-07, | |
| "logps/chosen": -80.3941650390625, | |
| "logps/rejected": -68.88729858398438, | |
| "loss": 1.0521, | |
| "losses/dpo": 0.6916662454605103, | |
| "losses/sft": 0.37930455803871155, | |
| "losses/total": 1.0709707736968994, | |
| "ref_logps/chosen": -80.68431091308594, | |
| "ref_logps/rejected": -68.9327392578125, | |
| "rewards/accuracies": 0.6328125, | |
| "rewards/chosen": 0.02901477925479412, | |
| "rewards/margins": 0.024471886456012726, | |
| "rewards/rejected": 0.004542896058410406, | |
| "step": 15 | |
| }, | |
| { | |
| "epoch": 0.1610062893081761, | |
| "grad_norm": 14.882165248684807, | |
| "learning_rate": 4.832951527604006e-07, | |
| "logps/chosen": -82.48802185058594, | |
| "logps/rejected": -78.44295501708984, | |
| "loss": 1.0461, | |
| "losses/dpo": 0.6916183233261108, | |
| "losses/sft": 0.36963188648223877, | |
| "losses/total": 1.0612502098083496, | |
| "ref_logps/chosen": -82.8018798828125, | |
| "ref_logps/rejected": -78.61569213867188, | |
| "rewards/accuracies": 0.59375, | |
| "rewards/chosen": 0.0313868448138237, | |
| "rewards/margins": 0.014113493263721466, | |
| "rewards/rejected": 0.017273351550102234, | |
| "step": 16 | |
| }, | |
| { | |
| "epoch": 0.1710691823899371, | |
| "grad_norm": 14.311327622150413, | |
| "learning_rate": 4.801624716691072e-07, | |
| "logps/chosen": -76.1909408569336, | |
| "logps/rejected": -71.96290588378906, | |
| "loss": 1.0383, | |
| "losses/dpo": 0.6969538927078247, | |
| "losses/sft": 0.3102976977825165, | |
| "losses/total": 1.0072516202926636, | |
| "ref_logps/chosen": -76.5750732421875, | |
| "ref_logps/rejected": -72.12234497070312, | |
| "rewards/accuracies": 0.6796875, | |
| "rewards/chosen": 0.038413576781749725, | |
| "rewards/margins": 0.02246994338929653, | |
| "rewards/rejected": 0.015943635255098343, | |
| "step": 17 | |
| }, | |
| { | |
| "epoch": 0.1811320754716981, | |
| "grad_norm": 15.165695980080356, | |
| "learning_rate": 4.767727284335852e-07, | |
| "logps/chosen": -79.1435317993164, | |
| "logps/rejected": -74.56543731689453, | |
| "loss": 1.0555, | |
| "losses/dpo": 0.7003558278083801, | |
| "losses/sft": 0.3900759220123291, | |
| "losses/total": 1.0904316902160645, | |
| "ref_logps/chosen": -79.60275268554688, | |
| "ref_logps/rejected": -74.77250671386719, | |
| "rewards/accuracies": 0.6796875, | |
| "rewards/chosen": 0.04592147842049599, | |
| "rewards/margins": 0.02521485835313797, | |
| "rewards/rejected": 0.020706620067358017, | |
| "step": 18 | |
| }, | |
| { | |
| "epoch": 0.19119496855345913, | |
| "grad_norm": 15.578067919764813, | |
| "learning_rate": 4.7312970896497027e-07, | |
| "logps/chosen": -83.0239486694336, | |
| "logps/rejected": -74.53577423095703, | |
| "loss": 1.0743, | |
| "losses/dpo": 0.700573742389679, | |
| "losses/sft": 0.4360811114311218, | |
| "losses/total": 1.1366548538208008, | |
| "ref_logps/chosen": -83.43629455566406, | |
| "ref_logps/rejected": -74.72197723388672, | |
| "rewards/accuracies": 0.6015625, | |
| "rewards/chosen": 0.04123528674244881, | |
| "rewards/margins": 0.022615134716033936, | |
| "rewards/rejected": 0.01862015575170517, | |
| "step": 19 | |
| }, | |
| { | |
| "epoch": 0.20125786163522014, | |
| "grad_norm": 16.148080108602343, | |
| "learning_rate": 4.6923748205166785e-07, | |
| "logps/chosen": -86.2251205444336, | |
| "logps/rejected": -78.16014099121094, | |
| "loss": 1.0688, | |
| "losses/dpo": 0.6666887998580933, | |
| "losses/sft": 0.40739819407463074, | |
| "losses/total": 1.0740869045257568, | |
| "ref_logps/chosen": -86.82108306884766, | |
| "ref_logps/rejected": -78.41912078857422, | |
| "rewards/accuracies": 0.6875, | |
| "rewards/chosen": 0.05959618091583252, | |
| "rewards/margins": 0.03369804471731186, | |
| "rewards/rejected": 0.02589813806116581, | |
| "step": 20 | |
| }, | |
| { | |
| "epoch": 0.21132075471698114, | |
| "grad_norm": 14.878899557347989, | |
| "learning_rate": 4.6510039481503485e-07, | |
| "logps/chosen": -75.44450378417969, | |
| "logps/rejected": -71.4926528930664, | |
| "loss": 1.0377, | |
| "losses/dpo": 0.7134190201759338, | |
| "losses/sft": 0.38239145278930664, | |
| "losses/total": 1.0958105325698853, | |
| "ref_logps/chosen": -75.94510650634766, | |
| "ref_logps/rejected": -71.6917953491211, | |
| "rewards/accuracies": 0.6796875, | |
| "rewards/chosen": 0.05006038397550583, | |
| "rewards/margins": 0.030146455392241478, | |
| "rewards/rejected": 0.0199139267206192, | |
| "step": 21 | |
| }, | |
| { | |
| "epoch": 0.22138364779874214, | |
| "grad_norm": 15.226230246124961, | |
| "learning_rate": 4.6072306785419924e-07, | |
| "logps/chosen": -80.53596496582031, | |
| "logps/rejected": -80.97723388671875, | |
| "loss": 1.0284, | |
| "losses/dpo": 0.6952553391456604, | |
| "losses/sft": 0.27885523438453674, | |
| "losses/total": 0.9741106033325195, | |
| "ref_logps/chosen": -81.25791931152344, | |
| "ref_logps/rejected": -81.23795318603516, | |
| "rewards/accuracies": 0.6796875, | |
| "rewards/chosen": 0.07219455391168594, | |
| "rewards/margins": 0.046122319996356964, | |
| "rewards/rejected": 0.026072228327393532, | |
| "step": 22 | |
| }, | |
| { | |
| "epoch": 0.23144654088050315, | |
| "grad_norm": 14.590224847017746, | |
| "learning_rate": 4.5611039008544007e-07, | |
| "logps/chosen": -77.15602111816406, | |
| "logps/rejected": -72.4802017211914, | |
| "loss": 1.0256, | |
| "losses/dpo": 0.6954873204231262, | |
| "losses/sft": 0.30716028809547424, | |
| "losses/total": 1.0026476383209229, | |
| "ref_logps/chosen": -77.8787841796875, | |
| "ref_logps/rejected": -72.68162536621094, | |
| "rewards/accuracies": 0.6953125, | |
| "rewards/chosen": 0.07227593660354614, | |
| "rewards/margins": 0.052133046090602875, | |
| "rewards/rejected": 0.020142890512943268, | |
| "step": 23 | |
| }, | |
| { | |
| "epoch": 0.24150943396226415, | |
| "grad_norm": 14.408995030982073, | |
| "learning_rate": 4.5126751328189073e-07, | |
| "logps/chosen": -75.66419982910156, | |
| "logps/rejected": -73.60345458984375, | |
| "loss": 1.025, | |
| "losses/dpo": 0.6652034521102905, | |
| "losses/sft": 0.353601336479187, | |
| "losses/total": 1.0188047885894775, | |
| "ref_logps/chosen": -76.47856903076172, | |
| "ref_logps/rejected": -73.73396301269531, | |
| "rewards/accuracies": 0.7421875, | |
| "rewards/chosen": 0.08143681287765503, | |
| "rewards/margins": 0.06838638335466385, | |
| "rewards/rejected": 0.013050435110926628, | |
| "step": 24 | |
| }, | |
| { | |
| "epoch": 0.25157232704402516, | |
| "grad_norm": 15.305978491237383, | |
| "learning_rate": 4.4619984631966524e-07, | |
| "logps/chosen": -85.6600341796875, | |
| "logps/rejected": -78.23870849609375, | |
| "loss": 1.0417, | |
| "losses/dpo": 0.6948066353797913, | |
| "losses/sft": 0.4345339834690094, | |
| "losses/total": 1.1293405294418335, | |
| "ref_logps/chosen": -86.60327911376953, | |
| "ref_logps/rejected": -78.51371002197266, | |
| "rewards/accuracies": 0.6953125, | |
| "rewards/chosen": 0.09432436525821686, | |
| "rewards/margins": 0.06682416051626205, | |
| "rewards/rejected": 0.027500208467245102, | |
| "step": 25 | |
| }, | |
| { | |
| "epoch": 0.26163522012578616, | |
| "grad_norm": 14.998262009207098, | |
| "learning_rate": 4.4091304913683303e-07, | |
| "logps/chosen": -78.49246215820312, | |
| "logps/rejected": -73.33612060546875, | |
| "loss": 1.0172, | |
| "losses/dpo": 0.6582043766975403, | |
| "losses/sft": 0.36708328127861023, | |
| "losses/total": 1.0252876281738281, | |
| "ref_logps/chosen": -79.48141479492188, | |
| "ref_logps/rejected": -73.58092498779297, | |
| "rewards/accuracies": 0.7578125, | |
| "rewards/chosen": 0.09889513999223709, | |
| "rewards/margins": 0.07441384345293045, | |
| "rewards/rejected": 0.024481287226080894, | |
| "step": 26 | |
| }, | |
| { | |
| "epoch": 0.27169811320754716, | |
| "grad_norm": 15.27620011007559, | |
| "learning_rate": 4.354130264119894e-07, | |
| "logps/chosen": -84.75664520263672, | |
| "logps/rejected": -81.69904327392578, | |
| "loss": 1.0329, | |
| "losses/dpo": 0.6849141716957092, | |
| "losses/sft": 0.39731666445732117, | |
| "losses/total": 1.0822309255599976, | |
| "ref_logps/chosen": -85.81089782714844, | |
| "ref_logps/rejected": -82.00923156738281, | |
| "rewards/accuracies": 0.65625, | |
| "rewards/chosen": 0.10542524605989456, | |
| "rewards/margins": 0.07440595328807831, | |
| "rewards/rejected": 0.031019294634461403, | |
| "step": 27 | |
| }, | |
| { | |
| "epoch": 0.28176100628930817, | |
| "grad_norm": 14.743778070925911, | |
| "learning_rate": 4.2970592096948233e-07, | |
| "logps/chosen": -80.38286590576172, | |
| "logps/rejected": -72.54148864746094, | |
| "loss": 1.0242, | |
| "losses/dpo": 0.646571695804596, | |
| "losses/sft": 0.3782905042171478, | |
| "losses/total": 1.0248621702194214, | |
| "ref_logps/chosen": -81.38656616210938, | |
| "ref_logps/rejected": -72.7380142211914, | |
| "rewards/accuracies": 0.734375, | |
| "rewards/chosen": 0.10037041455507278, | |
| "rewards/margins": 0.08071772754192352, | |
| "rewards/rejected": 0.019652685150504112, | |
| "step": 28 | |
| }, | |
| { | |
| "epoch": 0.2918238993710692, | |
| "grad_norm": 14.816804858263165, | |
| "learning_rate": 4.237981069186606e-07, | |
| "logps/chosen": -80.61254119873047, | |
| "logps/rejected": -68.75757598876953, | |
| "loss": 1.0156, | |
| "losses/dpo": 0.6498047709465027, | |
| "losses/sft": 0.35897865891456604, | |
| "losses/total": 1.0087834596633911, | |
| "ref_logps/chosen": -81.66033935546875, | |
| "ref_logps/rejected": -68.87429809570312, | |
| "rewards/accuracies": 0.703125, | |
| "rewards/chosen": 0.10477961599826813, | |
| "rewards/margins": 0.09310723096132278, | |
| "rewards/rejected": 0.01167239435017109, | |
| "step": 29 | |
| }, | |
| { | |
| "epoch": 0.3018867924528302, | |
| "grad_norm": 14.461566845563905, | |
| "learning_rate": 4.1769618253480586e-07, | |
| "logps/chosen": -82.01522827148438, | |
| "logps/rejected": -75.83625793457031, | |
| "loss": 1.0182, | |
| "losses/dpo": 0.6898115277290344, | |
| "losses/sft": 0.4069589078426361, | |
| "losses/total": 1.0967704057693481, | |
| "ref_logps/chosen": -83.18346405029297, | |
| "ref_logps/rejected": -76.05616760253906, | |
| "rewards/accuracies": 0.6875, | |
| "rewards/chosen": 0.11682344973087311, | |
| "rewards/margins": 0.09483154118061066, | |
| "rewards/rejected": 0.021991904824972153, | |
| "step": 30 | |
| }, | |
| { | |
| "epoch": 0.3119496855345912, | |
| "grad_norm": 15.942497361038132, | |
| "learning_rate": 4.114069628897006e-07, | |
| "logps/chosen": -85.87028503417969, | |
| "logps/rejected": -73.85649108886719, | |
| "loss": 1.0156, | |
| "losses/dpo": 0.6715642213821411, | |
| "losses/sft": 0.41779857873916626, | |
| "losses/total": 1.0893628597259521, | |
| "ref_logps/chosen": -87.21926879882812, | |
| "ref_logps/rejected": -74.12530517578125, | |
| "rewards/accuracies": 0.734375, | |
| "rewards/chosen": 0.13489778339862823, | |
| "rewards/margins": 0.10801610350608826, | |
| "rewards/rejected": 0.02688167802989483, | |
| "step": 31 | |
| }, | |
| { | |
| "epoch": 0.3220125786163522, | |
| "grad_norm": 13.82769050245281, | |
| "learning_rate": 4.049374722400612e-07, | |
| "logps/chosen": -74.70889282226562, | |
| "logps/rejected": -69.65875244140625, | |
| "loss": 1.0103, | |
| "losses/dpo": 0.6761119961738586, | |
| "losses/sft": 0.3233855962753296, | |
| "losses/total": 0.9994975924491882, | |
| "ref_logps/chosen": -75.858154296875, | |
| "ref_logps/rejected": -69.84304809570312, | |
| "rewards/accuracies": 0.71875, | |
| "rewards/chosen": 0.11492575705051422, | |
| "rewards/margins": 0.09649605304002762, | |
| "rewards/rejected": 0.0184297114610672, | |
| "step": 32 | |
| }, | |
| { | |
| "epoch": 0.3320754716981132, | |
| "grad_norm": 14.038904140570896, | |
| "learning_rate": 3.982949361823388e-07, | |
| "logps/chosen": -78.57609558105469, | |
| "logps/rejected": -74.79994201660156, | |
| "loss": 1.0256, | |
| "losses/dpo": 0.6977800130844116, | |
| "losses/sft": 0.3623635470867157, | |
| "losses/total": 1.0601435899734497, | |
| "ref_logps/chosen": -79.52400970458984, | |
| "ref_logps/rejected": -75.06456756591797, | |
| "rewards/accuracies": 0.65625, | |
| "rewards/chosen": 0.09479095041751862, | |
| "rewards/margins": 0.06832793354988098, | |
| "rewards/rejected": 0.026463022455573082, | |
| "step": 33 | |
| }, | |
| { | |
| "epoch": 0.3421383647798742, | |
| "grad_norm": 15.003139146574807, | |
| "learning_rate": 3.9148677358264883e-07, | |
| "logps/chosen": -80.80524444580078, | |
| "logps/rejected": -75.07316589355469, | |
| "loss": 1.0322, | |
| "losses/dpo": 0.6698934435844421, | |
| "losses/sft": 0.3599715232849121, | |
| "losses/total": 1.029865026473999, | |
| "ref_logps/chosen": -82.07255554199219, | |
| "ref_logps/rejected": -75.4892578125, | |
| "rewards/accuracies": 0.6796875, | |
| "rewards/chosen": 0.12673160433769226, | |
| "rewards/margins": 0.08512222021818161, | |
| "rewards/rejected": 0.041609376668930054, | |
| "step": 34 | |
| }, | |
| { | |
| "epoch": 0.3522012578616352, | |
| "grad_norm": 14.255000386833563, | |
| "learning_rate": 3.845205882908432e-07, | |
| "logps/chosen": -73.87222290039062, | |
| "logps/rejected": -72.94102478027344, | |
| "loss": 1.0045, | |
| "losses/dpo": 0.7203497886657715, | |
| "losses/sft": 0.36445504426956177, | |
| "losses/total": 1.0848047733306885, | |
| "ref_logps/chosen": -75.16232299804688, | |
| "ref_logps/rejected": -73.02802276611328, | |
| "rewards/accuracies": 0.71875, | |
| "rewards/chosen": 0.12901026010513306, | |
| "rewards/margins": 0.12031099945306778, | |
| "rewards/rejected": 0.008699269965291023, | |
| "step": 35 | |
| }, | |
| { | |
| "epoch": 0.3622641509433962, | |
| "grad_norm": 12.941056546878112, | |
| "learning_rate": 3.7740416064797934e-07, | |
| "logps/chosen": -70.72003173828125, | |
| "logps/rejected": -68.8385009765625, | |
| "loss": 0.9812, | |
| "losses/dpo": 0.6256467700004578, | |
| "losses/sft": 0.39033231139183044, | |
| "losses/total": 1.0159790515899658, | |
| "ref_logps/chosen": -71.86844635009766, | |
| "ref_logps/rejected": -68.81616973876953, | |
| "rewards/accuracies": 0.71875, | |
| "rewards/chosen": 0.11484173685312271, | |
| "rewards/margins": 0.11707467585802078, | |
| "rewards/rejected": -0.0022329380735754967, | |
| "step": 36 | |
| }, | |
| { | |
| "epoch": 0.3723270440251572, | |
| "grad_norm": 14.226051241378633, | |
| "learning_rate": 3.7014543879667095e-07, | |
| "logps/chosen": -75.77134704589844, | |
| "logps/rejected": -72.75182342529297, | |
| "loss": 0.9871, | |
| "losses/dpo": 0.7132631540298462, | |
| "losses/sft": 0.3754574656486511, | |
| "losses/total": 1.0887205600738525, | |
| "ref_logps/chosen": -77.1048812866211, | |
| "ref_logps/rejected": -72.76545715332031, | |
| "rewards/accuracies": 0.7265625, | |
| "rewards/chosen": 0.133354052901268, | |
| "rewards/margins": 0.13199105858802795, | |
| "rewards/rejected": 0.0013630054891109467, | |
| "step": 37 | |
| }, | |
| { | |
| "epoch": 0.38238993710691827, | |
| "grad_norm": 14.196980254865485, | |
| "learning_rate": 3.6275252980402544e-07, | |
| "logps/chosen": -85.88675689697266, | |
| "logps/rejected": -79.93682861328125, | |
| "loss": 1.0015, | |
| "losses/dpo": 0.6279609799385071, | |
| "losses/sft": 0.34624436497688293, | |
| "losses/total": 0.9742053151130676, | |
| "ref_logps/chosen": -87.35802459716797, | |
| "ref_logps/rejected": -79.97398376464844, | |
| "rewards/accuracies": 0.7421875, | |
| "rewards/chosen": 0.14712581038475037, | |
| "rewards/margins": 0.14341090619564056, | |
| "rewards/rejected": 0.0037149174604564905, | |
| "step": 38 | |
| }, | |
| { | |
| "epoch": 0.39245283018867927, | |
| "grad_norm": 14.456015586003081, | |
| "learning_rate": 3.5523369060708374e-07, | |
| "logps/chosen": -78.47836303710938, | |
| "logps/rejected": -79.48917388916016, | |
| "loss": 1.0232, | |
| "losses/dpo": 0.7399665117263794, | |
| "losses/sft": 0.3791757822036743, | |
| "losses/total": 1.1191422939300537, | |
| "ref_logps/chosen": -79.63422393798828, | |
| "ref_logps/rejected": -79.65493774414062, | |
| "rewards/accuracies": 0.7109375, | |
| "rewards/chosen": 0.11558625847101212, | |
| "rewards/margins": 0.09900960326194763, | |
| "rewards/rejected": 0.016576647758483887, | |
| "step": 39 | |
| }, | |
| { | |
| "epoch": 0.4025157232704403, | |
| "grad_norm": 14.505318885700563, | |
| "learning_rate": 3.475973187908737e-07, | |
| "logps/chosen": -88.03107452392578, | |
| "logps/rejected": -80.5452880859375, | |
| "loss": 1.0003, | |
| "losses/dpo": 0.6669642329216003, | |
| "losses/sft": 0.3931300640106201, | |
| "losses/total": 1.0600943565368652, | |
| "ref_logps/chosen": -89.49748992919922, | |
| "ref_logps/rejected": -80.5037841796875, | |
| "rewards/accuracies": 0.734375, | |
| "rewards/chosen": 0.14664065837860107, | |
| "rewards/margins": 0.15079057216644287, | |
| "rewards/rejected": -0.004149916581809521, | |
| "step": 40 | |
| }, | |
| { | |
| "epoch": 0.4125786163522013, | |
| "grad_norm": 14.149342282091867, | |
| "learning_rate": 3.3985194320937815e-07, | |
| "logps/chosen": -82.236328125, | |
| "logps/rejected": -77.66332244873047, | |
| "loss": 1.0057, | |
| "losses/dpo": 0.7688466310501099, | |
| "losses/sft": 0.3329574167728424, | |
| "losses/total": 1.1018040180206299, | |
| "ref_logps/chosen": -83.41654968261719, | |
| "ref_logps/rejected": -77.58070373535156, | |
| "rewards/accuracies": 0.7421875, | |
| "rewards/chosen": 0.11802264302968979, | |
| "rewards/margins": 0.12628453969955444, | |
| "rewards/rejected": -0.008261887356638908, | |
| "step": 41 | |
| }, | |
| { | |
| "epoch": 0.4226415094339623, | |
| "grad_norm": 13.618988720784087, | |
| "learning_rate": 3.3200621445989225e-07, | |
| "logps/chosen": -80.78431701660156, | |
| "logps/rejected": -75.85997772216797, | |
| "loss": 1.0087, | |
| "losses/dpo": 0.7258040904998779, | |
| "losses/sft": 0.3291688561439514, | |
| "losses/total": 1.0549728870391846, | |
| "ref_logps/chosen": -81.78661346435547, | |
| "ref_logps/rejected": -75.47698974609375, | |
| "rewards/accuracies": 0.703125, | |
| "rewards/chosen": 0.10023011267185211, | |
| "rewards/margins": 0.1385280042886734, | |
| "rewards/rejected": -0.038297899067401886, | |
| "step": 42 | |
| }, | |
| { | |
| "epoch": 0.4327044025157233, | |
| "grad_norm": 14.273197027918624, | |
| "learning_rate": 3.240688952214085e-07, | |
| "logps/chosen": -78.64404296875, | |
| "logps/rejected": -75.18115234375, | |
| "loss": 0.9865, | |
| "losses/dpo": 0.7115089893341064, | |
| "losses/sft": 0.3947640061378479, | |
| "losses/total": 1.1062729358673096, | |
| "ref_logps/chosen": -80.05891418457031, | |
| "ref_logps/rejected": -75.02284240722656, | |
| "rewards/accuracies": 0.71875, | |
| "rewards/chosen": 0.14148636162281036, | |
| "rewards/margins": 0.15731626749038696, | |
| "rewards/rejected": -0.015829918906092644, | |
| "step": 43 | |
| }, | |
| { | |
| "epoch": 0.4427672955974843, | |
| "grad_norm": 13.480343487130659, | |
| "learning_rate": 3.160488504678215e-07, | |
| "logps/chosen": -74.60444641113281, | |
| "logps/rejected": -73.48818969726562, | |
| "loss": 0.9864, | |
| "losses/dpo": 0.66825270652771, | |
| "losses/sft": 0.3295142650604248, | |
| "losses/total": 0.99776691198349, | |
| "ref_logps/chosen": -75.63268280029297, | |
| "ref_logps/rejected": -73.1849365234375, | |
| "rewards/accuracies": 0.65625, | |
| "rewards/chosen": 0.10282407701015472, | |
| "rewards/margins": 0.133149191737175, | |
| "rewards/rejected": -0.030325107276439667, | |
| "step": 44 | |
| }, | |
| { | |
| "epoch": 0.4528301886792453, | |
| "grad_norm": 14.0102985883088, | |
| "learning_rate": 3.0795503756688205e-07, | |
| "logps/chosen": -86.0777587890625, | |
| "logps/rejected": -75.60399627685547, | |
| "loss": 0.9862, | |
| "losses/dpo": 0.658064067363739, | |
| "losses/sft": 0.36845076084136963, | |
| "losses/total": 1.0265148878097534, | |
| "ref_logps/chosen": -87.51325988769531, | |
| "ref_logps/rejected": -74.97911071777344, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": 0.1435496211051941, | |
| "rewards/margins": 0.20603831112384796, | |
| "rewards/rejected": -0.06248871609568596, | |
| "step": 45 | |
| }, | |
| { | |
| "epoch": 0.4628930817610063, | |
| "grad_norm": 13.904517806164792, | |
| "learning_rate": 2.99796496275959e-07, | |
| "logps/chosen": -85.28681945800781, | |
| "logps/rejected": -79.6563491821289, | |
| "loss": 0.9989, | |
| "losses/dpo": 0.6056103706359863, | |
| "losses/sft": 0.34353241324424744, | |
| "losses/total": 0.9491427540779114, | |
| "ref_logps/chosen": -86.63987731933594, | |
| "ref_logps/rejected": -79.10623931884766, | |
| "rewards/accuracies": 0.71875, | |
| "rewards/chosen": 0.13530582189559937, | |
| "rewards/margins": 0.19031640887260437, | |
| "rewards/rejected": -0.0550105907022953, | |
| "step": 46 | |
| }, | |
| { | |
| "epoch": 0.4729559748427673, | |
| "grad_norm": 13.560110623241957, | |
| "learning_rate": 2.915823386457825e-07, | |
| "logps/chosen": -77.61348724365234, | |
| "logps/rejected": -75.27241516113281, | |
| "loss": 0.9656, | |
| "losses/dpo": 0.6847131252288818, | |
| "losses/sft": 0.40810227394104004, | |
| "losses/total": 1.0928153991699219, | |
| "ref_logps/chosen": -78.91578674316406, | |
| "ref_logps/rejected": -74.3929443359375, | |
| "rewards/accuracies": 0.7421875, | |
| "rewards/chosen": 0.13022957742214203, | |
| "rewards/margins": 0.2181762158870697, | |
| "rewards/rejected": -0.08794663846492767, | |
| "step": 47 | |
| }, | |
| { | |
| "epoch": 0.4830188679245283, | |
| "grad_norm": 13.954359522887207, | |
| "learning_rate": 2.8332173884344473e-07, | |
| "logps/chosen": -87.6903305053711, | |
| "logps/rejected": -76.82035064697266, | |
| "loss": 0.9552, | |
| "losses/dpo": 0.5738142728805542, | |
| "losses/sft": 0.3640093207359314, | |
| "losses/total": 0.9378236532211304, | |
| "ref_logps/chosen": -89.48294830322266, | |
| "ref_logps/rejected": -75.78794860839844, | |
| "rewards/accuracies": 0.828125, | |
| "rewards/chosen": 0.1792612373828888, | |
| "rewards/margins": 0.28250131011009216, | |
| "rewards/rejected": -0.10324011743068695, | |
| "step": 48 | |
| }, | |
| { | |
| "epoch": 0.4930817610062893, | |
| "grad_norm": 14.369413009124699, | |
| "learning_rate": 2.7502392290602463e-07, | |
| "logps/chosen": -86.66905212402344, | |
| "logps/rejected": -76.85792541503906, | |
| "loss": 0.9481, | |
| "losses/dpo": 0.6439822316169739, | |
| "losses/sft": 0.3433207869529724, | |
| "losses/total": 0.9873029589653015, | |
| "ref_logps/chosen": -88.44965362548828, | |
| "ref_logps/rejected": -75.8476333618164, | |
| "rewards/accuracies": 0.796875, | |
| "rewards/chosen": 0.17805951833724976, | |
| "rewards/margins": 0.27908915281295776, | |
| "rewards/rejected": -0.10102960467338562, | |
| "step": 49 | |
| }, | |
| { | |
| "epoch": 0.5031446540880503, | |
| "grad_norm": 13.398792594698223, | |
| "learning_rate": 2.6669815843628043e-07, | |
| "logps/chosen": -80.97222900390625, | |
| "logps/rejected": -71.44869232177734, | |
| "loss": 0.9467, | |
| "losses/dpo": 0.7079877257347107, | |
| "losses/sft": 0.31868329644203186, | |
| "losses/total": 1.0266709327697754, | |
| "ref_logps/chosen": -82.57719421386719, | |
| "ref_logps/rejected": -70.47540283203125, | |
| "rewards/accuracies": 0.7578125, | |
| "rewards/chosen": 0.16049541532993317, | |
| "rewards/margins": 0.25782501697540283, | |
| "rewards/rejected": -0.09732960164546967, | |
| "step": 50 | |
| }, | |
| { | |
| "epoch": 0.5132075471698113, | |
| "grad_norm": 14.007896777723824, | |
| "learning_rate": 2.583537442519186e-07, | |
| "logps/chosen": -79.37796783447266, | |
| "logps/rejected": -78.45370483398438, | |
| "loss": 0.9709, | |
| "losses/dpo": 0.6879909038543701, | |
| "losses/sft": 0.4556385278701782, | |
| "losses/total": 1.1436294317245483, | |
| "ref_logps/chosen": -80.48783874511719, | |
| "ref_logps/rejected": -77.22457122802734, | |
| "rewards/accuracies": 0.734375, | |
| "rewards/chosen": 0.11098764091730118, | |
| "rewards/margins": 0.2339007407426834, | |
| "rewards/rejected": -0.12291311472654343, | |
| "step": 51 | |
| }, | |
| { | |
| "epoch": 0.5232704402515723, | |
| "grad_norm": 13.571929188656334, | |
| "learning_rate": 2.5e-07, | |
| "logps/chosen": -76.12169647216797, | |
| "logps/rejected": -75.76917266845703, | |
| "loss": 0.9754, | |
| "losses/dpo": 0.551976203918457, | |
| "losses/sft": 0.34610363841056824, | |
| "losses/total": 0.8980798125267029, | |
| "ref_logps/chosen": -77.15998840332031, | |
| "ref_logps/rejected": -75.1527328491211, | |
| "rewards/accuracies": 0.6953125, | |
| "rewards/chosen": 0.10382916778326035, | |
| "rewards/margins": 0.1654740273952484, | |
| "rewards/rejected": -0.06164485588669777, | |
| "step": 52 | |
| }, | |
| { | |
| "epoch": 0.5333333333333333, | |
| "grad_norm": 13.78983992632312, | |
| "learning_rate": 2.416462557480814e-07, | |
| "logps/chosen": -78.70403289794922, | |
| "logps/rejected": -74.04467010498047, | |
| "loss": 0.9668, | |
| "losses/dpo": 0.7154827117919922, | |
| "losses/sft": 0.355939120054245, | |
| "losses/total": 1.0714218616485596, | |
| "ref_logps/chosen": -79.78579711914062, | |
| "ref_logps/rejected": -72.72827911376953, | |
| "rewards/accuracies": 0.6953125, | |
| "rewards/chosen": 0.10817612707614899, | |
| "rewards/margins": 0.23981451988220215, | |
| "rewards/rejected": -0.13163839280605316, | |
| "step": 53 | |
| }, | |
| { | |
| "epoch": 0.5433962264150943, | |
| "grad_norm": 13.17030909486059, | |
| "learning_rate": 2.333018415637196e-07, | |
| "logps/chosen": -78.02973937988281, | |
| "logps/rejected": -76.5998764038086, | |
| "loss": 0.9702, | |
| "losses/dpo": 0.6811453104019165, | |
| "losses/sft": 0.3610793948173523, | |
| "losses/total": 1.0422247648239136, | |
| "ref_logps/chosen": -79.0947265625, | |
| "ref_logps/rejected": -75.32565307617188, | |
| "rewards/accuracies": 0.6953125, | |
| "rewards/chosen": 0.10649853199720383, | |
| "rewards/margins": 0.23392075300216675, | |
| "rewards/rejected": -0.12742222845554352, | |
| "step": 54 | |
| }, | |
| { | |
| "epoch": 0.5534591194968553, | |
| "grad_norm": 13.825294877409265, | |
| "learning_rate": 2.249760770939754e-07, | |
| "logps/chosen": -82.60687255859375, | |
| "logps/rejected": -81.40107727050781, | |
| "loss": 0.9518, | |
| "losses/dpo": 0.8651823401451111, | |
| "losses/sft": 0.33423036336898804, | |
| "losses/total": 1.1994127035140991, | |
| "ref_logps/chosen": -84.10595703125, | |
| "ref_logps/rejected": -80.23603057861328, | |
| "rewards/accuracies": 0.7890625, | |
| "rewards/chosen": 0.1499093621969223, | |
| "rewards/margins": 0.2664135694503784, | |
| "rewards/rejected": -0.11650420725345612, | |
| "step": 55 | |
| }, | |
| { | |
| "epoch": 0.5635220125786163, | |
| "grad_norm": 13.434417359184703, | |
| "learning_rate": 2.1667826115655535e-07, | |
| "logps/chosen": -77.57618713378906, | |
| "logps/rejected": -72.0264892578125, | |
| "loss": 0.9118, | |
| "losses/dpo": 0.6483964323997498, | |
| "losses/sft": 0.4290625751018524, | |
| "losses/total": 1.0774589776992798, | |
| "ref_logps/chosen": -79.1749496459961, | |
| "ref_logps/rejected": -70.45938110351562, | |
| "rewards/accuracies": 0.796875, | |
| "rewards/chosen": 0.15987561643123627, | |
| "rewards/margins": 0.3165854811668396, | |
| "rewards/rejected": -0.15670984983444214, | |
| "step": 56 | |
| }, | |
| { | |
| "epoch": 0.5735849056603773, | |
| "grad_norm": 13.682845711236109, | |
| "learning_rate": 2.0841766135421747e-07, | |
| "logps/chosen": -77.24420928955078, | |
| "logps/rejected": -74.8659439086914, | |
| "loss": 0.9385, | |
| "losses/dpo": 0.5598583221435547, | |
| "losses/sft": 0.32705366611480713, | |
| "losses/total": 0.8869119882583618, | |
| "ref_logps/chosen": -78.34547424316406, | |
| "ref_logps/rejected": -73.31063842773438, | |
| "rewards/accuracies": 0.7578125, | |
| "rewards/chosen": 0.11012652516365051, | |
| "rewards/margins": 0.26565757393836975, | |
| "rewards/rejected": -0.15553104877471924, | |
| "step": 57 | |
| }, | |
| { | |
| "epoch": 0.5836477987421383, | |
| "grad_norm": 14.285195042847882, | |
| "learning_rate": 2.0020350372404102e-07, | |
| "logps/chosen": -84.25393676757812, | |
| "logps/rejected": -81.29403686523438, | |
| "loss": 1.0015, | |
| "losses/dpo": 0.5723427534103394, | |
| "losses/sft": 0.350294291973114, | |
| "losses/total": 0.9226371049880981, | |
| "ref_logps/chosen": -85.54225158691406, | |
| "ref_logps/rejected": -80.76034545898438, | |
| "rewards/accuracies": 0.6875, | |
| "rewards/chosen": 0.1288313865661621, | |
| "rewards/margins": 0.18219929933547974, | |
| "rewards/rejected": -0.05336792394518852, | |
| "step": 58 | |
| }, | |
| { | |
| "epoch": 0.5937106918238994, | |
| "grad_norm": 13.633332779144865, | |
| "learning_rate": 1.920449624331179e-07, | |
| "logps/chosen": -81.46337890625, | |
| "logps/rejected": -78.45436096191406, | |
| "loss": 0.9685, | |
| "losses/dpo": 0.5848886370658875, | |
| "losses/sft": 0.3466652035713196, | |
| "losses/total": 0.931553840637207, | |
| "ref_logps/chosen": -82.74446105957031, | |
| "ref_logps/rejected": -77.16352844238281, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": 0.12810780107975006, | |
| "rewards/margins": 0.2571909427642822, | |
| "rewards/rejected": -0.12908315658569336, | |
| "step": 59 | |
| }, | |
| { | |
| "epoch": 0.6037735849056604, | |
| "grad_norm": 13.112336410349611, | |
| "learning_rate": 1.839511495321785e-07, | |
| "logps/chosen": -77.9227294921875, | |
| "logps/rejected": -78.29096221923828, | |
| "loss": 0.9701, | |
| "losses/dpo": 0.6548681855201721, | |
| "losses/sft": 0.34684374928474426, | |
| "losses/total": 1.0017119646072388, | |
| "ref_logps/chosen": -78.95404052734375, | |
| "ref_logps/rejected": -77.03378295898438, | |
| "rewards/accuracies": 0.6953125, | |
| "rewards/chosen": 0.10313084721565247, | |
| "rewards/margins": 0.22884881496429443, | |
| "rewards/rejected": -0.12571796774864197, | |
| "step": 60 | |
| }, | |
| { | |
| "epoch": 0.6138364779874214, | |
| "grad_norm": 13.600151225844877, | |
| "learning_rate": 1.7593110477859152e-07, | |
| "logps/chosen": -79.69929504394531, | |
| "logps/rejected": -81.41608428955078, | |
| "loss": 0.9724, | |
| "losses/dpo": 0.7238990664482117, | |
| "losses/sft": 0.3255448639392853, | |
| "losses/total": 1.0494439601898193, | |
| "ref_logps/chosen": -80.66063690185547, | |
| "ref_logps/rejected": -80.092041015625, | |
| "rewards/accuracies": 0.6796875, | |
| "rewards/chosen": 0.09613411873579025, | |
| "rewards/margins": 0.22853833436965942, | |
| "rewards/rejected": -0.13240423798561096, | |
| "step": 61 | |
| }, | |
| { | |
| "epoch": 0.6238993710691824, | |
| "grad_norm": 13.73953093508848, | |
| "learning_rate": 1.679937855401077e-07, | |
| "logps/chosen": -77.4028091430664, | |
| "logps/rejected": -76.37467956542969, | |
| "loss": 0.9901, | |
| "losses/dpo": 0.8189263939857483, | |
| "losses/sft": 0.3098299205303192, | |
| "losses/total": 1.1287564039230347, | |
| "ref_logps/chosen": -78.21166229248047, | |
| "ref_logps/rejected": -75.11843872070312, | |
| "rewards/accuracies": 0.6875, | |
| "rewards/chosen": 0.0808858722448349, | |
| "rewards/margins": 0.20650942623615265, | |
| "rewards/rejected": -0.12562356889247894, | |
| "step": 62 | |
| }, | |
| { | |
| "epoch": 0.6339622641509434, | |
| "grad_norm": 13.434557079661225, | |
| "learning_rate": 1.6014805679062183e-07, | |
| "logps/chosen": -82.96167755126953, | |
| "logps/rejected": -74.5992431640625, | |
| "loss": 0.9358, | |
| "losses/dpo": 0.5670045614242554, | |
| "losses/sft": 0.3809337019920349, | |
| "losses/total": 0.9479383230209351, | |
| "ref_logps/chosen": -84.36255645751953, | |
| "ref_logps/rejected": -72.7698974609375, | |
| "rewards/accuracies": 0.7578125, | |
| "rewards/chosen": 0.14008811116218567, | |
| "rewards/margins": 0.32302191853523254, | |
| "rewards/rejected": -0.18293380737304688, | |
| "step": 63 | |
| }, | |
| { | |
| "epoch": 0.6440251572327044, | |
| "grad_norm": 13.593026912115091, | |
| "learning_rate": 1.524026812091263e-07, | |
| "logps/chosen": -79.88929748535156, | |
| "logps/rejected": -81.100830078125, | |
| "loss": 0.9879, | |
| "losses/dpo": 0.667205274105072, | |
| "losses/sft": 0.3584054708480835, | |
| "losses/total": 1.0256106853485107, | |
| "ref_logps/chosen": -80.77157592773438, | |
| "ref_logps/rejected": -79.93651580810547, | |
| "rewards/accuracies": 0.6875, | |
| "rewards/chosen": 0.0882289931178093, | |
| "rewards/margins": 0.20465928316116333, | |
| "rewards/rejected": -0.11643026769161224, | |
| "step": 64 | |
| }, | |
| { | |
| "epoch": 0.6540880503144654, | |
| "grad_norm": 13.167920882870893, | |
| "learning_rate": 1.447663093929163e-07, | |
| "logps/chosen": -77.92182922363281, | |
| "logps/rejected": -75.83228302001953, | |
| "loss": 0.9598, | |
| "losses/dpo": 0.7064578533172607, | |
| "losses/sft": 0.3876456618309021, | |
| "losses/total": 1.094103455543518, | |
| "ref_logps/chosen": -78.7850341796875, | |
| "ref_logps/rejected": -74.05996704101562, | |
| "rewards/accuracies": 0.7265625, | |
| "rewards/chosen": 0.08632123470306396, | |
| "rewards/margins": 0.2635524570941925, | |
| "rewards/rejected": -0.17723123729228973, | |
| "step": 65 | |
| }, | |
| { | |
| "epoch": 0.6641509433962264, | |
| "grad_norm": 13.6875600213724, | |
| "learning_rate": 1.372474701959745e-07, | |
| "logps/chosen": -78.46904754638672, | |
| "logps/rejected": -74.88655090332031, | |
| "loss": 0.9566, | |
| "losses/dpo": 0.8276803493499756, | |
| "losses/sft": 0.3391786813735962, | |
| "losses/total": 1.1668590307235718, | |
| "ref_logps/chosen": -79.26625061035156, | |
| "ref_logps/rejected": -73.1043701171875, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": 0.07972101867198944, | |
| "rewards/margins": 0.25793933868408203, | |
| "rewards/rejected": -0.17821836471557617, | |
| "step": 66 | |
| }, | |
| { | |
| "epoch": 0.6742138364779874, | |
| "grad_norm": 13.714831020339835, | |
| "learning_rate": 1.2985456120332906e-07, | |
| "logps/chosen": -83.13707733154297, | |
| "logps/rejected": -73.50003051757812, | |
| "loss": 0.9304, | |
| "losses/dpo": 0.552372932434082, | |
| "losses/sft": 0.3785245716571808, | |
| "losses/total": 0.9308975338935852, | |
| "ref_logps/chosen": -84.77952575683594, | |
| "ref_logps/rejected": -71.75003814697266, | |
| "rewards/accuracies": 0.8046875, | |
| "rewards/chosen": 0.16424375772476196, | |
| "rewards/margins": 0.3392435610294342, | |
| "rewards/rejected": -0.17499980330467224, | |
| "step": 67 | |
| }, | |
| { | |
| "epoch": 0.6842767295597484, | |
| "grad_norm": 13.196746131149819, | |
| "learning_rate": 1.225958393520206e-07, | |
| "logps/chosen": -78.00275421142578, | |
| "logps/rejected": -71.03691864013672, | |
| "loss": 0.9349, | |
| "losses/dpo": 0.5955305695533752, | |
| "losses/sft": 0.36189359426498413, | |
| "losses/total": 0.9574241638183594, | |
| "ref_logps/chosen": -78.86644744873047, | |
| "ref_logps/rejected": -68.57377624511719, | |
| "rewards/accuracies": 0.78125, | |
| "rewards/chosen": 0.0863691195845604, | |
| "rewards/margins": 0.33268311619758606, | |
| "rewards/rejected": -0.24631400406360626, | |
| "step": 68 | |
| }, | |
| { | |
| "epoch": 0.6943396226415094, | |
| "grad_norm": 13.32726104884298, | |
| "learning_rate": 1.1547941170915685e-07, | |
| "logps/chosen": -85.18400573730469, | |
| "logps/rejected": -74.97825622558594, | |
| "loss": 0.9174, | |
| "losses/dpo": 0.5665487051010132, | |
| "losses/sft": 0.36319419741630554, | |
| "losses/total": 0.9297429323196411, | |
| "ref_logps/chosen": -86.47076416015625, | |
| "ref_logps/rejected": -72.61034393310547, | |
| "rewards/accuracies": 0.765625, | |
| "rewards/chosen": 0.12867560982704163, | |
| "rewards/margins": 0.36546623706817627, | |
| "rewards/rejected": -0.23679062724113464, | |
| "step": 69 | |
| }, | |
| { | |
| "epoch": 0.7044025157232704, | |
| "grad_norm": 14.283665942013558, | |
| "learning_rate": 1.0851322641735117e-07, | |
| "logps/chosen": -83.07630157470703, | |
| "logps/rejected": -81.20217895507812, | |
| "loss": 0.976, | |
| "losses/dpo": 0.7452372312545776, | |
| "losses/sft": 0.337566077709198, | |
| "losses/total": 1.0828032493591309, | |
| "ref_logps/chosen": -84.08202362060547, | |
| "ref_logps/rejected": -79.51718139648438, | |
| "rewards/accuracies": 0.7265625, | |
| "rewards/chosen": 0.10057278722524643, | |
| "rewards/margins": 0.2690734267234802, | |
| "rewards/rejected": -0.1685006469488144, | |
| "step": 70 | |
| }, | |
| { | |
| "epoch": 0.7144654088050314, | |
| "grad_norm": 12.916070035590293, | |
| "learning_rate": 1.0170506381766119e-07, | |
| "logps/chosen": -74.7712631225586, | |
| "logps/rejected": -74.12356567382812, | |
| "loss": 0.9084, | |
| "losses/dpo": 0.5794627666473389, | |
| "losses/sft": 0.40780600905418396, | |
| "losses/total": 0.9872688055038452, | |
| "ref_logps/chosen": -75.7598876953125, | |
| "ref_logps/rejected": -71.68157196044922, | |
| "rewards/accuracies": 0.7421875, | |
| "rewards/chosen": 0.0988626778125763, | |
| "rewards/margins": 0.3430623412132263, | |
| "rewards/rejected": -0.24419964849948883, | |
| "step": 71 | |
| }, | |
| { | |
| "epoch": 0.7245283018867924, | |
| "grad_norm": 13.489560558340342, | |
| "learning_rate": 9.506252775993881e-08, | |
| "logps/chosen": -78.59689331054688, | |
| "logps/rejected": -77.52888488769531, | |
| "loss": 0.937, | |
| "losses/dpo": 0.7861602306365967, | |
| "losses/sft": 0.3728378415107727, | |
| "losses/total": 1.1589981317520142, | |
| "ref_logps/chosen": -79.52552032470703, | |
| "ref_logps/rejected": -75.24673461914062, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": 0.09286268055438995, | |
| "rewards/margins": 0.3210780918598175, | |
| "rewards/rejected": -0.22821539640426636, | |
| "step": 72 | |
| }, | |
| { | |
| "epoch": 0.7345911949685534, | |
| "grad_norm": 13.406140514068719, | |
| "learning_rate": 8.859303711029939e-08, | |
| "logps/chosen": -79.9459457397461, | |
| "logps/rejected": -76.30967712402344, | |
| "loss": 0.9496, | |
| "losses/dpo": 0.5502427220344543, | |
| "losses/sft": 0.36247241497039795, | |
| "losses/total": 0.9127151966094971, | |
| "ref_logps/chosen": -80.76979064941406, | |
| "ref_logps/rejected": -73.900390625, | |
| "rewards/accuracies": 0.78125, | |
| "rewards/chosen": 0.08238394558429718, | |
| "rewards/margins": 0.323311448097229, | |
| "rewards/rejected": -0.24092751741409302, | |
| "step": 73 | |
| }, | |
| { | |
| "epoch": 0.7446540880503144, | |
| "grad_norm": 13.255817188222572, | |
| "learning_rate": 8.230381746519419e-08, | |
| "logps/chosen": -83.04154968261719, | |
| "logps/rejected": -78.92507934570312, | |
| "loss": 0.9526, | |
| "losses/dpo": 0.5468727946281433, | |
| "losses/sft": 0.32324424386024475, | |
| "losses/total": 0.8701170682907104, | |
| "ref_logps/chosen": -84.06158447265625, | |
| "ref_logps/rejected": -76.94873046875, | |
| "rewards/accuracies": 0.7421875, | |
| "rewards/chosen": 0.10200438648462296, | |
| "rewards/margins": 0.2996390461921692, | |
| "rewards/rejected": -0.19763466715812683, | |
| "step": 74 | |
| }, | |
| { | |
| "epoch": 0.7547169811320755, | |
| "grad_norm": 13.448309012154779, | |
| "learning_rate": 7.620189308133943e-08, | |
| "logps/chosen": -82.13368225097656, | |
| "logps/rejected": -80.07732391357422, | |
| "loss": 0.9415, | |
| "losses/dpo": 0.6907780170440674, | |
| "losses/sft": 0.3356712758541107, | |
| "losses/total": 1.0264493227005005, | |
| "ref_logps/chosen": -82.98661041259766, | |
| "ref_logps/rejected": -77.53411865234375, | |
| "rewards/accuracies": 0.765625, | |
| "rewards/chosen": 0.08529379218816757, | |
| "rewards/margins": 0.33961349725723267, | |
| "rewards/rejected": -0.2543197274208069, | |
| "step": 75 | |
| }, | |
| { | |
| "epoch": 0.7647798742138365, | |
| "grad_norm": 13.705859719966018, | |
| "learning_rate": 7.02940790305177e-08, | |
| "logps/chosen": -84.20864868164062, | |
| "logps/rejected": -73.40209197998047, | |
| "loss": 0.9356, | |
| "losses/dpo": 0.5647323131561279, | |
| "losses/sft": 0.3960777521133423, | |
| "losses/total": 0.960810124874115, | |
| "ref_logps/chosen": -84.89796447753906, | |
| "ref_logps/rejected": -70.65544128417969, | |
| "rewards/accuracies": 0.765625, | |
| "rewards/chosen": 0.06893075257539749, | |
| "rewards/margins": 0.3435964584350586, | |
| "rewards/rejected": -0.2746656835079193, | |
| "step": 76 | |
| }, | |
| { | |
| "epoch": 0.7748427672955975, | |
| "grad_norm": 13.738145113740726, | |
| "learning_rate": 6.45869735880106e-08, | |
| "logps/chosen": -78.38431549072266, | |
| "logps/rejected": -76.76514434814453, | |
| "loss": 0.9343, | |
| "losses/dpo": 0.48376235365867615, | |
| "losses/sft": 0.36545315384864807, | |
| "losses/total": 0.8492154479026794, | |
| "ref_logps/chosen": -79.1065673828125, | |
| "ref_logps/rejected": -74.5013427734375, | |
| "rewards/accuracies": 0.7421875, | |
| "rewards/chosen": 0.07222675532102585, | |
| "rewards/margins": 0.29860708117485046, | |
| "rewards/rejected": -0.2263803333044052, | |
| "step": 77 | |
| }, | |
| { | |
| "epoch": 0.7849056603773585, | |
| "grad_norm": 15.243013296605241, | |
| "learning_rate": 5.908695086316701e-08, | |
| "logps/chosen": -86.07107543945312, | |
| "logps/rejected": -84.10150909423828, | |
| "loss": 0.9363, | |
| "losses/dpo": 0.6692842245101929, | |
| "losses/sft": 0.3787580132484436, | |
| "losses/total": 1.0480422973632812, | |
| "ref_logps/chosen": -87.29566955566406, | |
| "ref_logps/rejected": -81.75081634521484, | |
| "rewards/accuracies": 0.7578125, | |
| "rewards/chosen": 0.12246063351631165, | |
| "rewards/margins": 0.35752907395362854, | |
| "rewards/rejected": -0.2350684255361557, | |
| "step": 78 | |
| }, | |
| { | |
| "epoch": 0.7949685534591195, | |
| "grad_norm": 13.426200075061494, | |
| "learning_rate": 5.380015368033475e-08, | |
| "logps/chosen": -80.97967529296875, | |
| "logps/rejected": -78.46061706542969, | |
| "loss": 0.9532, | |
| "losses/dpo": 0.7194112539291382, | |
| "losses/sft": 0.3862980306148529, | |
| "losses/total": 1.1057093143463135, | |
| "ref_logps/chosen": -81.8296127319336, | |
| "ref_logps/rejected": -76.18516540527344, | |
| "rewards/accuracies": 0.6953125, | |
| "rewards/chosen": 0.08499410003423691, | |
| "rewards/margins": 0.31253981590270996, | |
| "rewards/rejected": -0.22754575312137604, | |
| "step": 79 | |
| }, | |
| { | |
| "epoch": 0.8050314465408805, | |
| "grad_norm": 14.18385541473246, | |
| "learning_rate": 4.8732486718109284e-08, | |
| "logps/chosen": -80.56340789794922, | |
| "logps/rejected": -80.10472106933594, | |
| "loss": 0.956, | |
| "losses/dpo": 0.7997117042541504, | |
| "losses/sft": 0.2968393862247467, | |
| "losses/total": 1.0965510606765747, | |
| "ref_logps/chosen": -81.14766693115234, | |
| "ref_logps/rejected": -77.70572662353516, | |
| "rewards/accuracies": 0.7265625, | |
| "rewards/chosen": 0.058426402509212494, | |
| "rewards/margins": 0.2983267903327942, | |
| "rewards/rejected": -0.2399004101753235, | |
| "step": 80 | |
| }, | |
| { | |
| "epoch": 0.8150943396226416, | |
| "grad_norm": 13.451956561854924, | |
| "learning_rate": 4.388960991455998e-08, | |
| "logps/chosen": -83.258544921875, | |
| "logps/rejected": -73.33403015136719, | |
| "loss": 0.9075, | |
| "losses/dpo": 0.6336656808853149, | |
| "losses/sft": 0.3658246695995331, | |
| "losses/total": 0.9994903206825256, | |
| "ref_logps/chosen": -84.42308044433594, | |
| "ref_logps/rejected": -70.38734436035156, | |
| "rewards/accuracies": 0.8203125, | |
| "rewards/chosen": 0.11645462363958359, | |
| "rewards/margins": 0.4111226797103882, | |
| "rewards/rejected": -0.294668048620224, | |
| "step": 81 | |
| }, | |
| { | |
| "epoch": 0.8251572327044026, | |
| "grad_norm": 13.420430376295206, | |
| "learning_rate": 3.927693214580075e-08, | |
| "logps/chosen": -79.82371520996094, | |
| "logps/rejected": -76.38064575195312, | |
| "loss": 0.9636, | |
| "losses/dpo": 0.7534294128417969, | |
| "losses/sft": 0.30548882484436035, | |
| "losses/total": 1.0589182376861572, | |
| "ref_logps/chosen": -80.31490325927734, | |
| "ref_logps/rejected": -74.10685729980469, | |
| "rewards/accuracies": 0.71875, | |
| "rewards/chosen": 0.04911880940198898, | |
| "rewards/margins": 0.27649733424186707, | |
| "rewards/rejected": -0.22737851738929749, | |
| "step": 82 | |
| }, | |
| { | |
| "epoch": 0.8352201257861636, | |
| "grad_norm": 13.178604910224282, | |
| "learning_rate": 3.4899605184965206e-08, | |
| "logps/chosen": -79.35425567626953, | |
| "logps/rejected": -72.01036071777344, | |
| "loss": 0.9413, | |
| "losses/dpo": 0.5128786563873291, | |
| "losses/sft": 0.36703866720199585, | |
| "losses/total": 0.8799173831939697, | |
| "ref_logps/chosen": -80.35247039794922, | |
| "ref_logps/rejected": -69.59880065917969, | |
| "rewards/accuracies": 0.734375, | |
| "rewards/chosen": 0.09982186555862427, | |
| "rewards/margins": 0.3409779965877533, | |
| "rewards/rejected": -0.24115616083145142, | |
| "step": 83 | |
| }, | |
| { | |
| "epoch": 0.8452830188679246, | |
| "grad_norm": 13.358792610889648, | |
| "learning_rate": 3.076251794833212e-08, | |
| "logps/chosen": -82.73087310791016, | |
| "logps/rejected": -78.44337463378906, | |
| "loss": 0.9477, | |
| "losses/dpo": 0.5781779289245605, | |
| "losses/sft": 0.37605956196784973, | |
| "losses/total": 0.9542375206947327, | |
| "ref_logps/chosen": -83.96467590332031, | |
| "ref_logps/rejected": -76.55040740966797, | |
| "rewards/accuracies": 0.7578125, | |
| "rewards/chosen": 0.12338028848171234, | |
| "rewards/margins": 0.31267744302749634, | |
| "rewards/rejected": -0.1892971247434616, | |
| "step": 84 | |
| }, | |
| { | |
| "epoch": 0.8553459119496856, | |
| "grad_norm": 13.690907855317144, | |
| "learning_rate": 2.6870291035029718e-08, | |
| "logps/chosen": -79.540771484375, | |
| "logps/rejected": -78.94165802001953, | |
| "loss": 0.9684, | |
| "losses/dpo": 0.7159697413444519, | |
| "losses/sft": 0.3729468584060669, | |
| "losses/total": 1.0889166593551636, | |
| "ref_logps/chosen": -80.23945617675781, | |
| "ref_logps/rejected": -76.9882583618164, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": 0.06986873596906662, | |
| "rewards/margins": 0.265209436416626, | |
| "rewards/rejected": -0.19534069299697876, | |
| "step": 85 | |
| }, | |
| { | |
| "epoch": 0.8654088050314466, | |
| "grad_norm": 13.105540805914607, | |
| "learning_rate": 2.3227271566414823e-08, | |
| "logps/chosen": -77.78494262695312, | |
| "logps/rejected": -71.80929565429688, | |
| "loss": 0.9314, | |
| "losses/dpo": 0.8130008578300476, | |
| "losses/sft": 0.33129045367240906, | |
| "losses/total": 1.1442912817001343, | |
| "ref_logps/chosen": -78.75529479980469, | |
| "ref_logps/rejected": -69.36322784423828, | |
| "rewards/accuracies": 0.8203125, | |
| "rewards/chosen": 0.09703564643859863, | |
| "rewards/margins": 0.34164199233055115, | |
| "rewards/rejected": -0.24460633099079132, | |
| "step": 86 | |
| }, | |
| { | |
| "epoch": 0.8754716981132076, | |
| "grad_norm": 13.612775733069526, | |
| "learning_rate": 1.9837528330892776e-08, | |
| "logps/chosen": -82.26605224609375, | |
| "logps/rejected": -76.89611053466797, | |
| "loss": 0.9146, | |
| "losses/dpo": 0.6761611700057983, | |
| "losses/sft": 0.31893810629844666, | |
| "losses/total": 0.9950993061065674, | |
| "ref_logps/chosen": -83.41204071044922, | |
| "ref_logps/rejected": -74.252197265625, | |
| "rewards/accuracies": 0.7734375, | |
| "rewards/chosen": 0.11459847539663315, | |
| "rewards/margins": 0.3789900541305542, | |
| "rewards/rejected": -0.26439157128334045, | |
| "step": 87 | |
| }, | |
| { | |
| "epoch": 0.8855345911949686, | |
| "grad_norm": 13.33006314162012, | |
| "learning_rate": 1.6704847239599363e-08, | |
| "logps/chosen": -83.5063247680664, | |
| "logps/rejected": -76.12062072753906, | |
| "loss": 0.9517, | |
| "losses/dpo": 0.5405787229537964, | |
| "losses/sft": 0.33481675386428833, | |
| "losses/total": 0.8753954768180847, | |
| "ref_logps/chosen": -84.05892181396484, | |
| "ref_logps/rejected": -73.51615142822266, | |
| "rewards/accuracies": 0.7421875, | |
| "rewards/chosen": 0.05525832623243332, | |
| "rewards/margins": 0.31570491194725037, | |
| "rewards/rejected": -0.26044660806655884, | |
| "step": 88 | |
| }, | |
| { | |
| "epoch": 0.8955974842767296, | |
| "grad_norm": 13.537155227659039, | |
| "learning_rate": 1.383272709802033e-08, | |
| "logps/chosen": -82.11839294433594, | |
| "logps/rejected": -79.336669921875, | |
| "loss": 0.9444, | |
| "losses/dpo": 0.7212787866592407, | |
| "losses/sft": 0.3310409486293793, | |
| "losses/total": 1.0523197650909424, | |
| "ref_logps/chosen": -83.14412689208984, | |
| "ref_logps/rejected": -77.21094512939453, | |
| "rewards/accuracies": 0.8046875, | |
| "rewards/chosen": 0.10257288068532944, | |
| "rewards/margins": 0.3151445984840393, | |
| "rewards/rejected": -0.21257171034812927, | |
| "step": 89 | |
| }, | |
| { | |
| "epoch": 0.9056603773584906, | |
| "grad_norm": 14.311506779786631, | |
| "learning_rate": 1.1224375698271893e-08, | |
| "logps/chosen": -80.742919921875, | |
| "logps/rejected": -80.19947814941406, | |
| "loss": 0.9524, | |
| "losses/dpo": 0.808186948299408, | |
| "losses/sft": 0.3583277463912964, | |
| "losses/total": 1.1665147542953491, | |
| "ref_logps/chosen": -81.75139617919922, | |
| "ref_logps/rejected": -78.16023254394531, | |
| "rewards/accuracies": 0.71875, | |
| "rewards/chosen": 0.10084706544876099, | |
| "rewards/margins": 0.30477190017700195, | |
| "rewards/rejected": -0.20392483472824097, | |
| "step": 90 | |
| }, | |
| { | |
| "epoch": 0.9157232704402516, | |
| "grad_norm": 13.55537996131118, | |
| "learning_rate": 8.882706236405885e-09, | |
| "logps/chosen": -77.98944854736328, | |
| "logps/rejected": -78.98483276367188, | |
| "loss": 0.9742, | |
| "losses/dpo": 0.8716121315956116, | |
| "losses/sft": 0.3347938060760498, | |
| "losses/total": 1.2064059972763062, | |
| "ref_logps/chosen": -78.49334716796875, | |
| "ref_logps/rejected": -76.73573303222656, | |
| "rewards/accuracies": 0.671875, | |
| "rewards/chosen": 0.05039064586162567, | |
| "rewards/margins": 0.27530133724212646, | |
| "rewards/rejected": -0.2249106764793396, | |
| "step": 91 | |
| }, | |
| { | |
| "epoch": 0.9257861635220126, | |
| "grad_norm": 13.793163914910798, | |
| "learning_rate": 6.810334058740736e-09, | |
| "logps/chosen": -81.20413970947266, | |
| "logps/rejected": -79.94811248779297, | |
| "loss": 0.9541, | |
| "losses/dpo": 0.7405242323875427, | |
| "losses/sft": 0.37262600660324097, | |
| "losses/total": 1.1131502389907837, | |
| "ref_logps/chosen": -82.03514862060547, | |
| "ref_logps/rejected": -77.8819351196289, | |
| "rewards/accuracies": 0.7265625, | |
| "rewards/chosen": 0.08309987932443619, | |
| "rewards/margins": 0.2897171676158905, | |
| "rewards/rejected": -0.20661728084087372, | |
| "step": 92 | |
| }, | |
| { | |
| "epoch": 0.9358490566037736, | |
| "grad_norm": 12.979428116887359, | |
| "learning_rate": 5.009573740853313e-09, | |
| "logps/chosen": -78.45671844482422, | |
| "logps/rejected": -72.1712875366211, | |
| "loss": 0.8749, | |
| "losses/dpo": 0.6323506832122803, | |
| "losses/sft": 0.37680134177207947, | |
| "losses/total": 1.0091519355773926, | |
| "ref_logps/chosen": -79.63896179199219, | |
| "ref_logps/rejected": -68.9825210571289, | |
| "rewards/accuracies": 0.8359375, | |
| "rewards/chosen": 0.118224136531353, | |
| "rewards/margins": 0.43710076808929443, | |
| "rewards/rejected": -0.31887662410736084, | |
| "step": 93 | |
| }, | |
| { | |
| "epoch": 0.9459119496855346, | |
| "grad_norm": 12.91270906410155, | |
| "learning_rate": 3.482436502492858e-09, | |
| "logps/chosen": -78.55203247070312, | |
| "logps/rejected": -76.18038940429688, | |
| "loss": 0.9144, | |
| "losses/dpo": 0.6181066036224365, | |
| "losses/sft": 0.37585684657096863, | |
| "losses/total": 0.9939634799957275, | |
| "ref_logps/chosen": -79.37383270263672, | |
| "ref_logps/rejected": -73.2895278930664, | |
| "rewards/accuracies": 0.7734375, | |
| "rewards/chosen": 0.08218016475439072, | |
| "rewards/margins": 0.3712664246559143, | |
| "rewards/rejected": -0.2890862822532654, | |
| "step": 94 | |
| }, | |
| { | |
| "epoch": 0.9559748427672956, | |
| "grad_norm": 13.290626630295796, | |
| "learning_rate": 2.2306279613049926e-09, | |
| "logps/chosen": -78.57327270507812, | |
| "logps/rejected": -77.99625396728516, | |
| "loss": 0.926, | |
| "losses/dpo": 0.6854672431945801, | |
| "losses/sft": 0.41287970542907715, | |
| "losses/total": 1.0983469486236572, | |
| "ref_logps/chosen": -79.61153411865234, | |
| "ref_logps/rejected": -75.51338958740234, | |
| "rewards/accuracies": 0.7734375, | |
| "rewards/chosen": 0.1038258820772171, | |
| "rewards/margins": 0.3521122932434082, | |
| "rewards/rejected": -0.2482864260673523, | |
| "step": 95 | |
| }, | |
| { | |
| "epoch": 0.9660377358490566, | |
| "grad_norm": 13.431385680233705, | |
| "learning_rate": 1.255546227873966e-09, | |
| "logps/chosen": -79.0116195678711, | |
| "logps/rejected": -79.90180969238281, | |
| "loss": 0.9308, | |
| "losses/dpo": 0.7681677937507629, | |
| "losses/sft": 0.41490912437438965, | |
| "losses/total": 1.1830769777297974, | |
| "ref_logps/chosen": -79.99933624267578, | |
| "ref_logps/rejected": -77.70450592041016, | |
| "rewards/accuracies": 0.78125, | |
| "rewards/chosen": 0.09877175092697144, | |
| "rewards/margins": 0.31850215792655945, | |
| "rewards/rejected": -0.21973037719726562, | |
| "step": 96 | |
| }, | |
| { | |
| "epoch": 0.9761006289308176, | |
| "grad_norm": 14.079932869223676, | |
| "learning_rate": 5.58280344211709e-10, | |
| "logps/chosen": -84.25556945800781, | |
| "logps/rejected": -80.85723876953125, | |
| "loss": 0.9438, | |
| "losses/dpo": 0.6998444199562073, | |
| "losses/sft": 0.4053678512573242, | |
| "losses/total": 1.1052123308181763, | |
| "ref_logps/chosen": -84.91212463378906, | |
| "ref_logps/rejected": -78.10282135009766, | |
| "rewards/accuracies": 0.7109375, | |
| "rewards/chosen": 0.0656551793217659, | |
| "rewards/margins": 0.341097354888916, | |
| "rewards/rejected": -0.2754421830177307, | |
| "step": 97 | |
| }, | |
| { | |
| "epoch": 0.9861635220125786, | |
| "grad_norm": 14.278966959359417, | |
| "learning_rate": 1.3960906743634703e-10, | |
| "logps/chosen": -80.89794158935547, | |
| "logps/rejected": -84.88188171386719, | |
| "loss": 0.9774, | |
| "losses/dpo": 0.7035096883773804, | |
| "losses/sft": 0.31059831380844116, | |
| "losses/total": 1.0141080617904663, | |
| "ref_logps/chosen": -81.65719604492188, | |
| "ref_logps/rejected": -83.00572204589844, | |
| "rewards/accuracies": 0.671875, | |
| "rewards/chosen": 0.07592566311359406, | |
| "rewards/margins": 0.26354214549064636, | |
| "rewards/rejected": -0.1876164972782135, | |
| "step": 98 | |
| }, | |
| { | |
| "epoch": 0.9962264150943396, | |
| "grad_norm": 12.961129324284881, | |
| "learning_rate": 0.0, | |
| "logps/chosen": -77.79227447509766, | |
| "logps/rejected": -73.44792175292969, | |
| "loss": 0.935, | |
| "losses/dpo": 0.673121452331543, | |
| "losses/sft": 0.3511947691440582, | |
| "losses/total": 1.0243163108825684, | |
| "ref_logps/chosen": -78.48436737060547, | |
| "ref_logps/rejected": -70.95289611816406, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": 0.06920892000198364, | |
| "rewards/margins": 0.31871092319488525, | |
| "rewards/rejected": -0.249502032995224, | |
| "step": 99 | |
| }, | |
| { | |
| "epoch": 0.9962264150943396, | |
| "step": 99, | |
| "total_flos": 0.0, | |
| "train_loss": 0.9872819769262063, | |
| "train_runtime": 3540.7882, | |
| "train_samples_per_second": 3.592, | |
| "train_steps_per_second": 0.028 | |
| } | |
| ], | |
| "logging_steps": 1.0, | |
| "max_steps": 99, | |
| "num_input_tokens_seen": 0, | |
| "num_train_epochs": 1, | |
| "save_steps": 1000, | |
| "stateful_callbacks": { | |
| "TrainerControl": { | |
| "args": { | |
| "should_epoch_stop": false, | |
| "should_evaluate": false, | |
| "should_log": false, | |
| "should_save": true, | |
| "should_training_stop": true | |
| }, | |
| "attributes": {} | |
| } | |
| }, | |
| "total_flos": 0.0, | |
| "train_batch_size": 1, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |