| { | |
| "best_metric": null, | |
| "best_model_checkpoint": null, | |
| "epoch": 1.986206896551724, | |
| "eval_steps": 16, | |
| "global_step": 114, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "epoch": 0.017422867513611617, | |
| "grad_norm": 0.39681947231292725, | |
| "kl": 0.0, | |
| "learning_rate": 2.5e-06, | |
| "logps/chosen": -11.77645784980106, | |
| "logps/rejected": -10.591483425911125, | |
| "loss": 0.5, | |
| "rewards/chosen": 0.0, | |
| "rewards/margins": 0.0, | |
| "rewards/rejected": 0.0, | |
| "step": 1 | |
| }, | |
| { | |
| "epoch": 0.03484573502722323, | |
| "grad_norm": 0.1452127993106842, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logps/chosen": -11.32990430748981, | |
| "logps/rejected": -10.54532470703125, | |
| "loss": 0.5, | |
| "rewards/chosen": 0.0, | |
| "rewards/margins": 0.0, | |
| "rewards/rejected": 0.0, | |
| "step": 2 | |
| }, | |
| { | |
| "epoch": 0.052268602540834846, | |
| "grad_norm": 0.3266962170600891, | |
| "kl": 0.0029318034648895264, | |
| "learning_rate": 4.955357142857144e-06, | |
| "logps/chosen": -11.723511402027027, | |
| "logps/rejected": -10.767559722440327, | |
| "loss": 0.5049, | |
| "rewards/chosen": -0.000146248413098825, | |
| "rewards/margins": -0.00010965315489224665, | |
| "rewards/rejected": -3.6595258206578356e-05, | |
| "step": 3 | |
| }, | |
| { | |
| "epoch": 0.06969147005444647, | |
| "grad_norm": 0.222149059176445, | |
| "kl": 0.0028855118434876204, | |
| "learning_rate": 4.910714285714286e-06, | |
| "logps/chosen": -11.060244668496622, | |
| "logps/rejected": -10.751804677685302, | |
| "loss": 0.5038, | |
| "rewards/chosen": -6.2011333333479394e-06, | |
| "rewards/margins": 0.0002193756961875533, | |
| "rewards/rejected": -0.00022557682952090125, | |
| "step": 4 | |
| }, | |
| { | |
| "epoch": 0.08711433756805807, | |
| "grad_norm": 0.23216503858566284, | |
| "kl": 0.003043340751901269, | |
| "learning_rate": 4.866071428571429e-06, | |
| "logps/chosen": -11.470826369065504, | |
| "logps/rejected": -10.402360395951705, | |
| "loss": 0.5049, | |
| "rewards/chosen": -6.513710384472058e-05, | |
| "rewards/margins": -2.0336450510690883e-05, | |
| "rewards/rejected": -4.48006533340297e-05, | |
| "step": 5 | |
| }, | |
| { | |
| "epoch": 0.10453720508166969, | |
| "grad_norm": 0.2607983946800232, | |
| "kl": 0.0025751441717147827, | |
| "learning_rate": 4.821428571428572e-06, | |
| "logps/chosen": -11.465345632399025, | |
| "logps/rejected": -10.64196449037286, | |
| "loss": 0.5036, | |
| "rewards/chosen": 0.00024762971736594494, | |
| "rewards/margins": 0.0002127551913608797, | |
| "rewards/rejected": 3.4874526005065234e-05, | |
| "step": 6 | |
| }, | |
| { | |
| "epoch": 0.12196007259528131, | |
| "grad_norm": 0.335457444190979, | |
| "kl": 0.0026024316903203726, | |
| "learning_rate": 4.776785714285715e-06, | |
| "logps/chosen": -11.233088825579573, | |
| "logps/rejected": -10.439390931995257, | |
| "loss": 0.5047, | |
| "rewards/chosen": -1.1002241089977418e-05, | |
| "rewards/margins": 5.897991155423426e-05, | |
| "rewards/rejected": -6.998215264421168e-05, | |
| "step": 7 | |
| }, | |
| { | |
| "epoch": 0.13938294010889293, | |
| "grad_norm": 0.21088895201683044, | |
| "kl": 0.004203550051897764, | |
| "learning_rate": 4.732142857142857e-06, | |
| "logps/chosen": -11.42248280843099, | |
| "logps/rejected": -10.589916229248047, | |
| "loss": 0.5049, | |
| "rewards/chosen": -0.00014668223836148778, | |
| "rewards/margins": 8.747742200891179e-05, | |
| "rewards/rejected": -0.00023415966037039956, | |
| "step": 8 | |
| }, | |
| { | |
| "epoch": 0.15680580762250454, | |
| "grad_norm": 0.2334163337945938, | |
| "kl": 0.0023759554605931044, | |
| "learning_rate": 4.6875000000000004e-06, | |
| "logps/chosen": -11.515217052940232, | |
| "logps/rejected": -10.479249335216029, | |
| "loss": 0.5047, | |
| "rewards/chosen": 1.3712330321496433e-05, | |
| "rewards/margins": 0.00026004273796865194, | |
| "rewards/rejected": -0.0002463304076471555, | |
| "step": 9 | |
| }, | |
| { | |
| "epoch": 0.17422867513611615, | |
| "grad_norm": 0.2604079246520996, | |
| "kl": 0.002872834214940667, | |
| "learning_rate": 4.642857142857144e-06, | |
| "logps/chosen": -11.413588686342592, | |
| "logps/rejected": -10.579748568784435, | |
| "loss": 0.5047, | |
| "rewards/chosen": -1.3620075252321032e-05, | |
| "rewards/margins": 0.00010424906420423088, | |
| "rewards/rejected": -0.00011786913945655192, | |
| "step": 10 | |
| }, | |
| { | |
| "epoch": 0.19165154264972778, | |
| "grad_norm": 0.2593174874782562, | |
| "kl": 0.0022250190377235413, | |
| "learning_rate": 4.5982142857142854e-06, | |
| "logps/chosen": -11.424675119011056, | |
| "logps/rejected": -10.487614428237535, | |
| "loss": 0.5044, | |
| "rewards/chosen": 0.00014462312843641426, | |
| "rewards/margins": 0.0003136567206988408, | |
| "rewards/rejected": -0.00016903359226242657, | |
| "step": 11 | |
| }, | |
| { | |
| "epoch": 0.20907441016333939, | |
| "grad_norm": 0.1917509138584137, | |
| "kl": 0.0026314922142773867, | |
| "learning_rate": 4.553571428571429e-06, | |
| "logps/chosen": -11.320460956057007, | |
| "logps/rejected": -10.401315122928674, | |
| "loss": 0.5045, | |
| "rewards/chosen": 0.00016278723993663833, | |
| "rewards/margins": 0.0006349250786245901, | |
| "rewards/rejected": -0.0004721378386879517, | |
| "step": 12 | |
| }, | |
| { | |
| "epoch": 0.226497277676951, | |
| "grad_norm": 0.2281549721956253, | |
| "kl": 0.0019408013904467225, | |
| "learning_rate": 4.508928571428572e-06, | |
| "logps/chosen": -11.184846626695736, | |
| "logps/rejected": -10.86145371965223, | |
| "loss": 0.5041, | |
| "rewards/chosen": 0.00017400297699664605, | |
| "rewards/margins": 0.0004404290903305609, | |
| "rewards/rejected": -0.00026642611333391484, | |
| "step": 13 | |
| }, | |
| { | |
| "epoch": 0.24392014519056263, | |
| "grad_norm": 0.29345858097076416, | |
| "kl": 0.0032534776255488396, | |
| "learning_rate": 4.464285714285715e-06, | |
| "logps/chosen": -11.328454192120928, | |
| "logps/rejected": -10.352222804454607, | |
| "loss": 0.5055, | |
| "rewards/chosen": -0.00023024088532703563, | |
| "rewards/margins": 0.00022275829169079655, | |
| "rewards/rejected": -0.0004529991770178322, | |
| "step": 14 | |
| }, | |
| { | |
| "epoch": 0.2613430127041742, | |
| "grad_norm": 0.20449300110340118, | |
| "kl": 0.003026653779670596, | |
| "learning_rate": 4.419642857142857e-06, | |
| "logps/chosen": -11.203996222636421, | |
| "logps/rejected": -10.722607944100936, | |
| "loss": 0.5043, | |
| "rewards/chosen": 0.00018860261679300803, | |
| "rewards/margins": 0.0004592195771910241, | |
| "rewards/rejected": -0.0002706169603980161, | |
| "step": 15 | |
| }, | |
| { | |
| "epoch": 0.27876588021778587, | |
| "grad_norm": 0.201906219124794, | |
| "kl": 0.002389234723523259, | |
| "learning_rate": 4.3750000000000005e-06, | |
| "logps/chosen": -11.154755261479592, | |
| "logps/rejected": -10.722290688372674, | |
| "loss": 0.5042, | |
| "rewards/chosen": 0.00029305163390782414, | |
| "rewards/margins": 0.0007897253288582932, | |
| "rewards/rejected": -0.000496673694950469, | |
| "step": 16 | |
| }, | |
| { | |
| "epoch": 0.27876588021778587, | |
| "eval_kl": 0.002219570567831397, | |
| "eval_logps/chosen": -11.288407461983818, | |
| "eval_logps/rejected": -10.652853829520089, | |
| "eval_loss": 0.5038086175918579, | |
| "eval_rewards/chosen": 0.0004127062524535826, | |
| "eval_rewards/margins": 0.0008178721514663526, | |
| "eval_rewards/rejected": -0.00040516589901276996, | |
| "eval_runtime": 41.7888, | |
| "eval_samples_per_second": 10.673, | |
| "eval_steps_per_second": 0.67, | |
| "step": 16 | |
| }, | |
| { | |
| "epoch": 0.2961887477313975, | |
| "grad_norm": 0.2981337308883667, | |
| "kl": 0.0040084668435156345, | |
| "learning_rate": 4.330357142857143e-06, | |
| "logps/chosen": -11.60596080662049, | |
| "logps/rejected": -10.618717876233553, | |
| "loss": 0.5046, | |
| "rewards/chosen": 0.00017366750338642867, | |
| "rewards/margins": 0.0006427841346935287, | |
| "rewards/rejected": -0.0004691166313071, | |
| "step": 17 | |
| }, | |
| { | |
| "epoch": 0.3136116152450091, | |
| "grad_norm": 0.1905021220445633, | |
| "kl": 0.0029678388964384794, | |
| "learning_rate": 4.2857142857142855e-06, | |
| "logps/chosen": -11.358947906876567, | |
| "logps/rejected": -10.434524660188009, | |
| "loss": 0.5048, | |
| "rewards/chosen": 2.532637675752616e-05, | |
| "rewards/margins": 0.0007196114555040427, | |
| "rewards/rejected": -0.0006942850787465166, | |
| "step": 18 | |
| }, | |
| { | |
| "epoch": 0.3310344827586207, | |
| "grad_norm": 0.21423697471618652, | |
| "kl": 0.002790730679407716, | |
| "learning_rate": 4.241071428571429e-06, | |
| "logps/chosen": -11.554482736895162, | |
| "logps/rejected": -11.022931956281566, | |
| "loss": 0.504, | |
| "rewards/chosen": 0.00010277342892462207, | |
| "rewards/margins": 0.0007622132545750395, | |
| "rewards/rejected": -0.0006594398256504174, | |
| "step": 19 | |
| }, | |
| { | |
| "epoch": 0.3484573502722323, | |
| "grad_norm": 0.19702628254890442, | |
| "kl": 0.002327132970094681, | |
| "learning_rate": 4.196428571428572e-06, | |
| "logps/chosen": -11.411150614420572, | |
| "logps/rejected": -10.28190549214681, | |
| "loss": 0.504, | |
| "rewards/chosen": 0.000137356905421863, | |
| "rewards/margins": 0.0005323387255581716, | |
| "rewards/rejected": -0.00039498182013630867, | |
| "step": 20 | |
| }, | |
| { | |
| "epoch": 0.3658802177858439, | |
| "grad_norm": 0.20904229581356049, | |
| "kl": 0.002573903650045395, | |
| "learning_rate": 4.151785714285715e-06, | |
| "logps/chosen": -11.281766930404974, | |
| "logps/rejected": -10.511914841672207, | |
| "loss": 0.504, | |
| "rewards/chosen": 0.00020978350800519087, | |
| "rewards/margins": 0.0008185515050823616, | |
| "rewards/rejected": -0.0006087679970771708, | |
| "step": 21 | |
| }, | |
| { | |
| "epoch": 0.38330308529945556, | |
| "grad_norm": 0.19066323339939117, | |
| "kl": 0.004084563348442316, | |
| "learning_rate": 4.107142857142857e-06, | |
| "logps/chosen": -11.57584678589191, | |
| "logps/rejected": -11.04019266504156, | |
| "loss": 0.5042, | |
| "rewards/chosen": 0.00028499441137364435, | |
| "rewards/margins": 0.0007805728857745476, | |
| "rewards/rejected": -0.0004955784744009032, | |
| "step": 22 | |
| }, | |
| { | |
| "epoch": 0.40072595281306717, | |
| "grad_norm": 0.21532228589057922, | |
| "kl": 0.0018439519917592406, | |
| "learning_rate": 4.0625000000000005e-06, | |
| "logps/chosen": -11.395969251063482, | |
| "logps/rejected": -10.348430416126943, | |
| "loss": 0.5039, | |
| "rewards/chosen": 0.0002833760065558069, | |
| "rewards/margins": 0.000828726542101749, | |
| "rewards/rejected": -0.0005453505355459421, | |
| "step": 23 | |
| }, | |
| { | |
| "epoch": 0.41814882032667877, | |
| "grad_norm": 0.1852329671382904, | |
| "kl": 0.002771291183307767, | |
| "learning_rate": 4.017857142857143e-06, | |
| "logps/chosen": -11.074297150088029, | |
| "logps/rejected": -10.626986228813559, | |
| "loss": 0.5035, | |
| "rewards/chosen": 0.00041746350241379, | |
| "rewards/margins": 0.0010044164534086372, | |
| "rewards/rejected": -0.0005869529509948472, | |
| "step": 24 | |
| }, | |
| { | |
| "epoch": 0.4355716878402904, | |
| "grad_norm": 0.2030268907546997, | |
| "kl": 0.004407358821481466, | |
| "learning_rate": 3.9732142857142855e-06, | |
| "logps/chosen": -11.303015870474727, | |
| "logps/rejected": -10.537113018889926, | |
| "loss": 0.5033, | |
| "rewards/chosen": 0.000330610088800472, | |
| "rewards/margins": 0.0013877044688375805, | |
| "rewards/rejected": -0.0010570943800371085, | |
| "step": 25 | |
| }, | |
| { | |
| "epoch": 0.452994555353902, | |
| "grad_norm": 0.330771803855896, | |
| "kl": 0.0028855446726083755, | |
| "learning_rate": 3.928571428571429e-06, | |
| "logps/chosen": -11.326861509342784, | |
| "logps/rejected": -10.294227359169408, | |
| "loss": 0.5044, | |
| "rewards/chosen": 0.0002357424380853004, | |
| "rewards/margins": 0.0008208984113942651, | |
| "rewards/rejected": -0.0005851559733089647, | |
| "step": 26 | |
| }, | |
| { | |
| "epoch": 0.4704174228675136, | |
| "grad_norm": 0.204763263463974, | |
| "kl": 0.0029878586065024137, | |
| "learning_rate": 3.883928571428572e-06, | |
| "logps/chosen": -11.238988330696202, | |
| "logps/rejected": -10.748736097729557, | |
| "loss": 0.5038, | |
| "rewards/chosen": 0.0004333936338183246, | |
| "rewards/margins": 0.0012179940220870076, | |
| "rewards/rejected": -0.0007846003882686829, | |
| "step": 27 | |
| }, | |
| { | |
| "epoch": 0.48784029038112525, | |
| "grad_norm": 0.4104137420654297, | |
| "kl": 0.002435252070426941, | |
| "learning_rate": 3.839285714285715e-06, | |
| "logps/chosen": -11.65949259124677, | |
| "logps/rejected": -10.41943679769521, | |
| "loss": 0.505, | |
| "rewards/chosen": -4.84786946952189e-05, | |
| "rewards/margins": 0.0007131273978328765, | |
| "rewards/rejected": -0.0007616060925280954, | |
| "step": 28 | |
| }, | |
| { | |
| "epoch": 0.5052631578947369, | |
| "grad_norm": 0.1900724321603775, | |
| "kl": 0.003689997596666217, | |
| "learning_rate": 3.794642857142857e-06, | |
| "logps/chosen": -11.369253954475308, | |
| "logps/rejected": -10.650356996814738, | |
| "loss": 0.5035, | |
| "rewards/chosen": 0.0005172402034571141, | |
| "rewards/margins": 0.0012426704372049484, | |
| "rewards/rejected": -0.0007254302337478343, | |
| "step": 29 | |
| }, | |
| { | |
| "epoch": 0.5226860254083484, | |
| "grad_norm": 0.25149452686309814, | |
| "kl": 0.003089333651587367, | |
| "learning_rate": 3.7500000000000005e-06, | |
| "logps/chosen": -11.706775306138677, | |
| "logps/rejected": -10.319467447916667, | |
| "loss": 0.5048, | |
| "rewards/chosen": 0.00012126840821659293, | |
| "rewards/margins": 0.0005743338433082473, | |
| "rewards/rejected": -0.00045306543509165444, | |
| "step": 30 | |
| }, | |
| { | |
| "epoch": 0.5401088929219601, | |
| "grad_norm": 0.2375558316707611, | |
| "kl": 0.0021258096676319838, | |
| "learning_rate": 3.7053571428571434e-06, | |
| "logps/chosen": -11.040724951044387, | |
| "logps/rejected": -10.158553799715909, | |
| "loss": 0.504, | |
| "rewards/chosen": 0.00032178575158430455, | |
| "rewards/margins": 0.001046311172281009, | |
| "rewards/rejected": -0.0007245254206967044, | |
| "step": 31 | |
| }, | |
| { | |
| "epoch": 0.5575317604355717, | |
| "grad_norm": 0.20534147322177887, | |
| "kl": 0.002767733531072736, | |
| "learning_rate": 3.660714285714286e-06, | |
| "logps/chosen": -11.517271468581537, | |
| "logps/rejected": -10.606380126338161, | |
| "loss": 0.5037, | |
| "rewards/chosen": 0.0005935063821607523, | |
| "rewards/margins": 0.0013072239592998884, | |
| "rewards/rejected": -0.0007137175771391362, | |
| "step": 32 | |
| }, | |
| { | |
| "epoch": 0.5575317604355717, | |
| "eval_kl": 0.0013082654913887382, | |
| "eval_logps/chosen": -11.286455426897321, | |
| "eval_logps/rejected": -10.656499590192523, | |
| "eval_loss": 0.5032945275306702, | |
| "eval_rewards/chosen": 0.0006079860031604767, | |
| "eval_rewards/margins": 0.001377601841730731, | |
| "eval_rewards/rejected": -0.0007696158385702542, | |
| "eval_runtime": 41.5226, | |
| "eval_samples_per_second": 10.741, | |
| "eval_steps_per_second": 0.674, | |
| "step": 32 | |
| }, | |
| { | |
| "epoch": 0.5749546279491833, | |
| "grad_norm": 0.240458145737648, | |
| "kl": 0.002010853262618184, | |
| "learning_rate": 3.616071428571429e-06, | |
| "logps/chosen": -11.25503524547906, | |
| "logps/rejected": -11.051669034090908, | |
| "loss": 0.5043, | |
| "rewards/chosen": 0.00019917678711983152, | |
| "rewards/margins": 0.0007490985231613238, | |
| "rewards/rejected": -0.0005499217360414923, | |
| "step": 33 | |
| }, | |
| { | |
| "epoch": 0.592377495462795, | |
| "grad_norm": 0.18933577835559845, | |
| "kl": 0.0021181467454880476, | |
| "learning_rate": 3.5714285714285718e-06, | |
| "logps/chosen": -11.664008105974741, | |
| "logps/rejected": -10.828131391115837, | |
| "loss": 0.5032, | |
| "rewards/chosen": 0.0005863340947912147, | |
| "rewards/margins": 0.0013070002798906317, | |
| "rewards/rejected": -0.000720666185099417, | |
| "step": 34 | |
| }, | |
| { | |
| "epoch": 0.6098003629764065, | |
| "grad_norm": 0.20784388482570648, | |
| "kl": 0.002850498305633664, | |
| "learning_rate": 3.5267857142857147e-06, | |
| "logps/chosen": -11.464662464007452, | |
| "logps/rejected": -10.13366056743421, | |
| "loss": 0.5041, | |
| "rewards/chosen": 0.00017015952970277326, | |
| "rewards/margins": 0.0008830348158433109, | |
| "rewards/rejected": -0.0007128752861405376, | |
| "step": 35 | |
| }, | |
| { | |
| "epoch": 0.6272232304900182, | |
| "grad_norm": 0.1830260157585144, | |
| "kl": 0.003351332386955619, | |
| "learning_rate": 3.482142857142857e-06, | |
| "logps/chosen": -11.203702368776113, | |
| "logps/rejected": -10.977577265433986, | |
| "loss": 0.5036, | |
| "rewards/chosen": 0.00040133928356064395, | |
| "rewards/margins": 0.0009519243287689104, | |
| "rewards/rejected": -0.0005505850452082664, | |
| "step": 36 | |
| }, | |
| { | |
| "epoch": 0.6446460980036298, | |
| "grad_norm": 0.18368099629878998, | |
| "kl": 0.0037287927698343992, | |
| "learning_rate": 3.4375e-06, | |
| "logps/chosen": -11.507929849361188, | |
| "logps/rejected": -10.265622594673646, | |
| "loss": 0.503, | |
| "rewards/chosen": 0.00047660240316917883, | |
| "rewards/margins": 0.0012168469473675392, | |
| "rewards/rejected": -0.0007402445441983603, | |
| "step": 37 | |
| }, | |
| { | |
| "epoch": 0.6620689655172414, | |
| "grad_norm": 0.22299793362617493, | |
| "kl": 0.0025776519905775785, | |
| "learning_rate": 3.3928571428571435e-06, | |
| "logps/chosen": -11.776427912833121, | |
| "logps/rejected": -10.537473627590241, | |
| "loss": 0.5041, | |
| "rewards/chosen": 0.00029345284107372844, | |
| "rewards/margins": 0.0009656724311435042, | |
| "rewards/rejected": -0.0006722195900697759, | |
| "step": 38 | |
| }, | |
| { | |
| "epoch": 0.679491833030853, | |
| "grad_norm": 0.21307651698589325, | |
| "kl": 0.0029651224613189697, | |
| "learning_rate": 3.3482142857142855e-06, | |
| "logps/chosen": -11.337782663316583, | |
| "logps/rejected": -10.489964500633446, | |
| "loss": 0.5041, | |
| "rewards/chosen": 0.000353607026176836, | |
| "rewards/margins": 0.0009334429436754503, | |
| "rewards/rejected": -0.0005798359174986143, | |
| "step": 39 | |
| }, | |
| { | |
| "epoch": 0.6969147005444646, | |
| "grad_norm": 0.21936196088790894, | |
| "kl": 0.0019360833102837205, | |
| "learning_rate": 3.303571428571429e-06, | |
| "logps/chosen": -12.052929194286616, | |
| "logps/rejected": -10.637656591271842, | |
| "loss": 0.5044, | |
| "rewards/chosen": 0.000453127905575916, | |
| "rewards/margins": 0.0010775220099412956, | |
| "rewards/rejected": -0.0006243941043653796, | |
| "step": 40 | |
| }, | |
| { | |
| "epoch": 0.7143375680580762, | |
| "grad_norm": 0.5484824180603027, | |
| "kl": 0.0032773923594504595, | |
| "learning_rate": 3.258928571428572e-06, | |
| "logps/chosen": -11.568930269491792, | |
| "logps/rejected": -10.71394266108031, | |
| "loss": 0.5054, | |
| "rewards/chosen": 5.686828264533871e-05, | |
| "rewards/margins": 0.0006258111605199182, | |
| "rewards/rejected": -0.0005689428778745795, | |
| "step": 41 | |
| }, | |
| { | |
| "epoch": 0.7317604355716878, | |
| "grad_norm": 0.20393657684326172, | |
| "kl": 0.003910336643457413, | |
| "learning_rate": 3.2142857142857147e-06, | |
| "logps/chosen": -11.504363593319558, | |
| "logps/rejected": -10.101014539930556, | |
| "loss": 0.5036, | |
| "rewards/chosen": 0.00029671770974624257, | |
| "rewards/margins": 0.001082045991996541, | |
| "rewards/rejected": -0.0007853282822502984, | |
| "step": 42 | |
| }, | |
| { | |
| "epoch": 0.7491833030852995, | |
| "grad_norm": 0.16519714891910553, | |
| "kl": 0.003650219878181815, | |
| "learning_rate": 3.1696428571428572e-06, | |
| "logps/chosen": -10.816071234809028, | |
| "logps/rejected": -10.322287166819853, | |
| "loss": 0.503, | |
| "rewards/chosen": 0.0007183290190166897, | |
| "rewards/margins": 0.0015814549685303683, | |
| "rewards/rejected": -0.0008631259495136785, | |
| "step": 43 | |
| }, | |
| { | |
| "epoch": 0.7666061705989111, | |
| "grad_norm": 0.20090065896511078, | |
| "kl": 0.002337512793019414, | |
| "learning_rate": 3.125e-06, | |
| "logps/chosen": -11.918630044291339, | |
| "logps/rejected": -11.223259346737725, | |
| "loss": 0.5041, | |
| "rewards/chosen": 0.00022355469036602912, | |
| "rewards/margins": 0.0007338886578213189, | |
| "rewards/rejected": -0.0005103339674552898, | |
| "step": 44 | |
| }, | |
| { | |
| "epoch": 0.7840290381125227, | |
| "grad_norm": 0.23820927739143372, | |
| "kl": 0.0014156276592984796, | |
| "learning_rate": 3.080357142857143e-06, | |
| "logps/chosen": -11.657449557086615, | |
| "logps/rejected": -10.76135790132429, | |
| "loss": 0.5037, | |
| "rewards/chosen": 0.0006081545994350604, | |
| "rewards/margins": 0.0009453997203017225, | |
| "rewards/rejected": -0.0003372451208666622, | |
| "step": 45 | |
| }, | |
| { | |
| "epoch": 0.8014519056261343, | |
| "grad_norm": 0.2038673311471939, | |
| "kl": 0.0020267742220312357, | |
| "learning_rate": 3.0357142857142856e-06, | |
| "logps/chosen": -11.508970748546512, | |
| "logps/rejected": -10.532100327058727, | |
| "loss": 0.5043, | |
| "rewards/chosen": 0.0003054198599600977, | |
| "rewards/margins": 0.0006779901515518521, | |
| "rewards/rejected": -0.0003725702915917544, | |
| "step": 46 | |
| }, | |
| { | |
| "epoch": 0.8188747731397459, | |
| "grad_norm": 0.2023131251335144, | |
| "kl": 0.0036333007737994194, | |
| "learning_rate": 2.991071428571429e-06, | |
| "logps/chosen": -11.018499706769436, | |
| "logps/rejected": -10.779378461234177, | |
| "loss": 0.5039, | |
| "rewards/chosen": 0.00048798394586701177, | |
| "rewards/margins": 0.0010197794913363682, | |
| "rewards/rejected": -0.0005317955454693565, | |
| "step": 47 | |
| }, | |
| { | |
| "epoch": 0.8362976406533575, | |
| "grad_norm": 0.19009532034397125, | |
| "kl": 0.0020073249470442533, | |
| "learning_rate": 2.946428571428572e-06, | |
| "logps/chosen": -11.3989532127809, | |
| "logps/rejected": -11.23740068454187, | |
| "loss": 0.5035, | |
| "rewards/chosen": 0.0002772415035896087, | |
| "rewards/margins": 0.0007655857580138798, | |
| "rewards/rejected": -0.0004883442544242711, | |
| "step": 48 | |
| }, | |
| { | |
| "epoch": 0.8362976406533575, | |
| "eval_kl": 0.001558297430165112, | |
| "eval_logps/chosen": -11.289903913225446, | |
| "eval_logps/rejected": -10.654574802943639, | |
| "eval_loss": 0.5040844082832336, | |
| "eval_rewards/chosen": 0.00026316663050757986, | |
| "eval_rewards/margins": 0.0008403176741142358, | |
| "eval_rewards/rejected": -0.000577151043606656, | |
| "eval_runtime": 41.5721, | |
| "eval_samples_per_second": 10.728, | |
| "eval_steps_per_second": 0.674, | |
| "step": 48 | |
| }, | |
| { | |
| "epoch": 0.8537205081669691, | |
| "grad_norm": 0.2040846198797226, | |
| "kl": 0.0028173942118883133, | |
| "learning_rate": 2.9017857142857148e-06, | |
| "logps/chosen": -11.502028847760695, | |
| "logps/rejected": -9.920858160493337, | |
| "loss": 0.5037, | |
| "rewards/chosen": 0.0003798981720113499, | |
| "rewards/margins": 0.001099527059744577, | |
| "rewards/rejected": -0.0007196288877332271, | |
| "step": 49 | |
| }, | |
| { | |
| "epoch": 0.8711433756805808, | |
| "grad_norm": 0.46055036783218384, | |
| "kl": 0.001712739816866815, | |
| "learning_rate": 2.8571428571428573e-06, | |
| "logps/chosen": -11.66295957325691, | |
| "logps/rejected": -10.606267155827704, | |
| "loss": 0.5046, | |
| "rewards/chosen": 8.328470527826242e-05, | |
| "rewards/margins": 0.0005694089115119315, | |
| "rewards/rejected": -0.000486124206233669, | |
| "step": 50 | |
| }, | |
| { | |
| "epoch": 0.8885662431941924, | |
| "grad_norm": 0.2116948515176773, | |
| "kl": 0.0014734516153112054, | |
| "learning_rate": 2.8125e-06, | |
| "logps/chosen": -11.087164900249377, | |
| "logps/rejected": -10.773697606224456, | |
| "loss": 0.5046, | |
| "rewards/chosen": 0.0002402601545291054, | |
| "rewards/margins": 0.0008381397699308074, | |
| "rewards/rejected": -0.000597879615401702, | |
| "step": 51 | |
| }, | |
| { | |
| "epoch": 0.905989110707804, | |
| "grad_norm": 0.2204010933637619, | |
| "kl": 0.0023574617225676775, | |
| "learning_rate": 2.767857142857143e-06, | |
| "logps/chosen": -11.274248798076924, | |
| "logps/rejected": -10.153245391038359, | |
| "loss": 0.5032, | |
| "rewards/chosen": 0.0005002290392533327, | |
| "rewards/margins": 0.0011188293800006614, | |
| "rewards/rejected": -0.0006186003407473286, | |
| "step": 52 | |
| }, | |
| { | |
| "epoch": 0.9234119782214156, | |
| "grad_norm": 0.22388231754302979, | |
| "kl": 0.002206307603046298, | |
| "learning_rate": 2.7232142857142856e-06, | |
| "logps/chosen": -11.911312705592104, | |
| "logps/rejected": -10.432602086018042, | |
| "loss": 0.5041, | |
| "rewards/chosen": 0.0004630535840988159, | |
| "rewards/margins": 0.0009620043727540478, | |
| "rewards/rejected": -0.0004989507886552319, | |
| "step": 53 | |
| }, | |
| { | |
| "epoch": 0.9408348457350272, | |
| "grad_norm": 0.20113378763198853, | |
| "kl": 0.0021755837369710207, | |
| "learning_rate": 2.6785714285714285e-06, | |
| "logps/chosen": -11.37064292594178, | |
| "logps/rejected": -10.506611788539082, | |
| "loss": 0.5036, | |
| "rewards/chosen": 0.00027320436010622, | |
| "rewards/margins": 0.0005519093562507254, | |
| "rewards/rejected": -0.0002787049961445054, | |
| "step": 54 | |
| }, | |
| { | |
| "epoch": 0.9582577132486388, | |
| "grad_norm": 0.7052344083786011, | |
| "kl": 0.002290615811944008, | |
| "learning_rate": 2.633928571428572e-06, | |
| "logps/chosen": -11.785455354221204, | |
| "logps/rejected": -10.28837688228627, | |
| "loss": 0.5038, | |
| "rewards/chosen": 0.0004812158371141444, | |
| "rewards/margins": 0.0010049399622622454, | |
| "rewards/rejected": -0.0005237241251481011, | |
| "step": 55 | |
| }, | |
| { | |
| "epoch": 0.9756805807622505, | |
| "grad_norm": 0.18455247581005096, | |
| "kl": 0.001295599271543324, | |
| "learning_rate": 2.5892857142857148e-06, | |
| "logps/chosen": -11.051934689891581, | |
| "logps/rejected": -10.790322161735372, | |
| "loss": 0.5038, | |
| "rewards/chosen": 0.00037732340243397925, | |
| "rewards/margins": 0.0009394810824965168, | |
| "rewards/rejected": -0.0005621576800625375, | |
| "step": 56 | |
| }, | |
| { | |
| "epoch": 0.993103448275862, | |
| "grad_norm": 0.1891961693763733, | |
| "kl": 0.00294533371925354, | |
| "learning_rate": 2.5446428571428573e-06, | |
| "logps/chosen": -11.06938274166127, | |
| "logps/rejected": -10.141628405186518, | |
| "loss": 0.5034, | |
| "rewards/chosen": 0.00042142599357842164, | |
| "rewards/margins": 0.0006545582687216406, | |
| "rewards/rejected": -0.0002331322751432189, | |
| "step": 57 | |
| }, | |
| { | |
| "epoch": 1.0105263157894737, | |
| "grad_norm": 0.22368647158145905, | |
| "kl": 0.002181227086111903, | |
| "learning_rate": 2.5e-06, | |
| "logps/chosen": -11.29225919699367, | |
| "logps/rejected": -10.528809902815013, | |
| "loss": 0.5036, | |
| "rewards/chosen": 0.0002977928406075586, | |
| "rewards/margins": 0.0008452181976182237, | |
| "rewards/rejected": -0.000547425357010665, | |
| "step": 58 | |
| }, | |
| { | |
| "epoch": 1.0279491833030854, | |
| "grad_norm": 0.19894687831401825, | |
| "kl": 0.0022480275947600603, | |
| "learning_rate": 2.455357142857143e-06, | |
| "logps/chosen": -11.208153912401574, | |
| "logps/rejected": -10.815308563468992, | |
| "loss": 0.5037, | |
| "rewards/chosen": 0.00028297573015132913, | |
| "rewards/margins": 0.0009500956342332799, | |
| "rewards/rejected": -0.0006671199040819508, | |
| "step": 59 | |
| }, | |
| { | |
| "epoch": 1.0453720508166968, | |
| "grad_norm": 0.19452357292175293, | |
| "kl": 0.003947921562939882, | |
| "learning_rate": 2.410714285714286e-06, | |
| "logps/chosen": -11.397494791666666, | |
| "logps/rejected": -10.136647309359097, | |
| "loss": 0.5036, | |
| "rewards/chosen": 0.0006445555686950684, | |
| "rewards/margins": 0.0011657039269842872, | |
| "rewards/rejected": -0.0005211483582892187, | |
| "step": 60 | |
| }, | |
| { | |
| "epoch": 1.0627949183303085, | |
| "grad_norm": 0.2021968960762024, | |
| "kl": 0.00338422111235559, | |
| "learning_rate": 2.3660714285714285e-06, | |
| "logps/chosen": -11.75505078125, | |
| "logps/rejected": -10.338195024252544, | |
| "loss": 0.5035, | |
| "rewards/chosen": 0.0006066491603851318, | |
| "rewards/margins": 0.0008883069659614078, | |
| "rewards/rejected": -0.0002816578055762759, | |
| "step": 61 | |
| }, | |
| { | |
| "epoch": 1.0802177858439201, | |
| "grad_norm": 0.18729077279567719, | |
| "kl": 0.0022495612502098083, | |
| "learning_rate": 2.321428571428572e-06, | |
| "logps/chosen": -11.218356103012242, | |
| "logps/rejected": -10.509180972450658, | |
| "loss": 0.5034, | |
| "rewards/chosen": 0.0004960870005420803, | |
| "rewards/margins": 0.0008710139801466005, | |
| "rewards/rejected": -0.0003749269796045203, | |
| "step": 62 | |
| }, | |
| { | |
| "epoch": 1.0976406533575318, | |
| "grad_norm": 0.19914378225803375, | |
| "kl": 0.003042086260393262, | |
| "learning_rate": 2.2767857142857144e-06, | |
| "logps/chosen": -11.236100905012377, | |
| "logps/rejected": -10.941490760216347, | |
| "loss": 0.5034, | |
| "rewards/chosen": 0.0013145432613863803, | |
| "rewards/margins": 0.0017898373715980775, | |
| "rewards/rejected": -0.00047529411021169726, | |
| "step": 63 | |
| }, | |
| { | |
| "epoch": 1.1150635208711435, | |
| "grad_norm": 0.19482098519802094, | |
| "kl": 0.002518598223105073, | |
| "learning_rate": 2.2321428571428573e-06, | |
| "logps/chosen": -11.394892939814815, | |
| "logps/rejected": -10.390007762419872, | |
| "loss": 0.5037, | |
| "rewards/chosen": 0.0003466351480080337, | |
| "rewards/margins": 0.0006846176696153592, | |
| "rewards/rejected": -0.0003379825216073256, | |
| "step": 64 | |
| }, | |
| { | |
| "epoch": 1.1150635208711435, | |
| "eval_kl": 0.0016980862710624933, | |
| "eval_logps/chosen": -11.287184579031807, | |
| "eval_logps/rejected": -10.654030936104911, | |
| "eval_loss": 0.5034511089324951, | |
| "eval_rewards/chosen": 0.0005350507396672453, | |
| "eval_rewards/margins": 0.0010578856537384646, | |
| "eval_rewards/rejected": -0.0005228349140712193, | |
| "eval_runtime": 41.5198, | |
| "eval_samples_per_second": 10.742, | |
| "eval_steps_per_second": 0.674, | |
| "step": 64 | |
| }, | |
| { | |
| "epoch": 1.132486388384755, | |
| "grad_norm": 0.20668481290340424, | |
| "kl": 0.003911910578608513, | |
| "learning_rate": 2.1875000000000002e-06, | |
| "logps/chosen": -11.54857183807525, | |
| "logps/rejected": -11.033686736893785, | |
| "loss": 0.504, | |
| "rewards/chosen": 0.0006883917608071322, | |
| "rewards/margins": 0.0012867070528502448, | |
| "rewards/rejected": -0.0005983152920431127, | |
| "step": 65 | |
| }, | |
| { | |
| "epoch": 1.1499092558983666, | |
| "grad_norm": 0.20961228013038635, | |
| "kl": 0.0017986433813348413, | |
| "learning_rate": 2.1428571428571427e-06, | |
| "logps/chosen": -11.831485829454786, | |
| "logps/rejected": -10.89035345583546, | |
| "loss": 0.504, | |
| "rewards/chosen": 0.0002542631502481217, | |
| "rewards/margins": 0.0006354290375722992, | |
| "rewards/rejected": -0.0003811658873241775, | |
| "step": 66 | |
| }, | |
| { | |
| "epoch": 1.1673321234119782, | |
| "grad_norm": 0.19809454679489136, | |
| "kl": 0.002224177587777376, | |
| "learning_rate": 2.098214285714286e-06, | |
| "logps/chosen": -11.331428947029533, | |
| "logps/rejected": -10.59391437190594, | |
| "loss": 0.5034, | |
| "rewards/chosen": 0.0005333201325201727, | |
| "rewards/margins": 0.001024618166864742, | |
| "rewards/rejected": -0.0004912980343445693, | |
| "step": 67 | |
| }, | |
| { | |
| "epoch": 1.18475499092559, | |
| "grad_norm": 0.19940558075904846, | |
| "kl": 0.00197195284999907, | |
| "learning_rate": 2.0535714285714286e-06, | |
| "logps/chosen": -11.550331552282016, | |
| "logps/rejected": -10.52055773651808, | |
| "loss": 0.503, | |
| "rewards/chosen": 0.0006965530340937892, | |
| "rewards/margins": 0.0013385297657675268, | |
| "rewards/rejected": -0.0006419767316737377, | |
| "step": 68 | |
| }, | |
| { | |
| "epoch": 1.2021778584392013, | |
| "grad_norm": 0.21953897178173065, | |
| "kl": 0.0020015963818877935, | |
| "learning_rate": 2.0089285714285715e-06, | |
| "logps/chosen": -10.969684954573935, | |
| "logps/rejected": -10.10690514375847, | |
| "loss": 0.5042, | |
| "rewards/chosen": 0.0001828334385291078, | |
| "rewards/margins": 0.0006383278937348235, | |
| "rewards/rejected": -0.00045549445520571576, | |
| "step": 69 | |
| }, | |
| { | |
| "epoch": 1.219600725952813, | |
| "grad_norm": 0.19710642099380493, | |
| "kl": 0.00231179385446012, | |
| "learning_rate": 1.9642857142857144e-06, | |
| "logps/chosen": -11.595243038467848, | |
| "logps/rejected": -10.112245387193152, | |
| "loss": 0.5033, | |
| "rewards/chosen": 0.00039139722432364314, | |
| "rewards/margins": 0.0010068912661115128, | |
| "rewards/rejected": -0.0006154940417878696, | |
| "step": 70 | |
| }, | |
| { | |
| "epoch": 1.2370235934664247, | |
| "grad_norm": 0.1987324059009552, | |
| "kl": 0.002300393534824252, | |
| "learning_rate": 1.9196428571428573e-06, | |
| "logps/chosen": -11.305307889174403, | |
| "logps/rejected": -10.520050751278772, | |
| "loss": 0.5035, | |
| "rewards/chosen": 0.0004280993729750737, | |
| "rewards/margins": 0.0009470002607156162, | |
| "rewards/rejected": -0.0005189008877405425, | |
| "step": 71 | |
| }, | |
| { | |
| "epoch": 1.2544464609800363, | |
| "grad_norm": 0.18199442327022552, | |
| "kl": 0.0023830088321119547, | |
| "learning_rate": 1.8750000000000003e-06, | |
| "logps/chosen": -11.361725593834231, | |
| "logps/rejected": -10.366453233233628, | |
| "loss": 0.5029, | |
| "rewards/chosen": 0.000802040341086786, | |
| "rewards/margins": 0.0015088396506467594, | |
| "rewards/rejected": -0.0007067993095599734, | |
| "step": 72 | |
| }, | |
| { | |
| "epoch": 1.271869328493648, | |
| "grad_norm": 0.2903735339641571, | |
| "kl": 0.0014767992543056607, | |
| "learning_rate": 1.830357142857143e-06, | |
| "logps/chosen": -11.19476151596653, | |
| "logps/rejected": -10.959830972092663, | |
| "loss": 0.5034, | |
| "rewards/chosen": 0.00034413270937289043, | |
| "rewards/margins": 0.0008636754093954977, | |
| "rewards/rejected": -0.0005195427000226073, | |
| "step": 73 | |
| }, | |
| { | |
| "epoch": 1.2892921960072594, | |
| "grad_norm": 0.20160308480262756, | |
| "kl": 0.003358997404575348, | |
| "learning_rate": 1.7857142857142859e-06, | |
| "logps/chosen": -10.99349352108535, | |
| "logps/rejected": -10.71151579269255, | |
| "loss": 0.5032, | |
| "rewards/chosen": 0.0006134921023922582, | |
| "rewards/margins": 0.0010400663268667124, | |
| "rewards/rejected": -0.00042657422447445415, | |
| "step": 74 | |
| }, | |
| { | |
| "epoch": 1.306715063520871, | |
| "grad_norm": 0.23630282282829285, | |
| "kl": 0.0023285781498998404, | |
| "learning_rate": 1.7410714285714286e-06, | |
| "logps/chosen": -11.506769643439947, | |
| "logps/rejected": -10.45025872564935, | |
| "loss": 0.5036, | |
| "rewards/chosen": 0.0006310967801445147, | |
| "rewards/margins": 0.0013577277155657525, | |
| "rewards/rejected": -0.0007266309354212377, | |
| "step": 75 | |
| }, | |
| { | |
| "epoch": 1.3241379310344827, | |
| "grad_norm": 0.18052582442760468, | |
| "kl": 0.0017558973049744964, | |
| "learning_rate": 1.6964285714285717e-06, | |
| "logps/chosen": -11.271184662723785, | |
| "logps/rejected": -10.48470008910809, | |
| "loss": 0.5033, | |
| "rewards/chosen": 0.0006234512076048595, | |
| "rewards/margins": 0.0013483937273518372, | |
| "rewards/rejected": -0.0007249425197469777, | |
| "step": 76 | |
| }, | |
| { | |
| "epoch": 1.3415607985480944, | |
| "grad_norm": 0.18041782081127167, | |
| "kl": 0.0014101862907409668, | |
| "learning_rate": 1.6517857142857144e-06, | |
| "logps/chosen": -11.356517080872704, | |
| "logps/rejected": -10.676619024547804, | |
| "loss": 0.5033, | |
| "rewards/chosen": 0.0006575664979579255, | |
| "rewards/margins": 0.001282264682831324, | |
| "rewards/rejected": -0.0006246981848733986, | |
| "step": 77 | |
| }, | |
| { | |
| "epoch": 1.358983666061706, | |
| "grad_norm": 0.2689516544342041, | |
| "kl": 0.0018190672853961587, | |
| "learning_rate": 1.6071428571428574e-06, | |
| "logps/chosen": -11.690565026324728, | |
| "logps/rejected": -10.228720703125, | |
| "loss": 0.5034, | |
| "rewards/chosen": 0.0008241140648074772, | |
| "rewards/margins": 0.0013455211176820424, | |
| "rewards/rejected": -0.0005214070528745652, | |
| "step": 78 | |
| }, | |
| { | |
| "epoch": 1.3764065335753175, | |
| "grad_norm": 0.20166873931884766, | |
| "kl": 0.0017292723059654236, | |
| "learning_rate": 1.5625e-06, | |
| "logps/chosen": -11.26230097903481, | |
| "logps/rejected": -11.479227756367292, | |
| "loss": 0.5034, | |
| "rewards/chosen": 0.0006177344654179827, | |
| "rewards/margins": 0.0012838053047731855, | |
| "rewards/rejected": -0.0006660708393552028, | |
| "step": 79 | |
| }, | |
| { | |
| "epoch": 1.3938294010889292, | |
| "grad_norm": 0.19607041776180267, | |
| "kl": 0.0031144104432314634, | |
| "learning_rate": 1.5178571428571428e-06, | |
| "logps/chosen": -11.208266241253181, | |
| "logps/rejected": -10.324174479166667, | |
| "loss": 0.5036, | |
| "rewards/chosen": 0.0007915615883795663, | |
| "rewards/margins": 0.0014803941892910246, | |
| "rewards/rejected": -0.0006888326009114583, | |
| "step": 80 | |
| }, | |
| { | |
| "epoch": 1.3938294010889292, | |
| "eval_kl": 0.0010345944901928306, | |
| "eval_logps/chosen": -11.287428719656807, | |
| "eval_logps/rejected": -10.653524126325335, | |
| "eval_loss": 0.5035938024520874, | |
| "eval_rewards/chosen": 0.0005107134514089141, | |
| "eval_rewards/margins": 0.0009828850826514618, | |
| "eval_rewards/rejected": -0.0004721716312425477, | |
| "eval_runtime": 41.5877, | |
| "eval_samples_per_second": 10.724, | |
| "eval_steps_per_second": 0.673, | |
| "step": 80 | |
| }, | |
| { | |
| "epoch": 1.4112522686025408, | |
| "grad_norm": 0.2139582335948944, | |
| "kl": 0.0026578225661069155, | |
| "learning_rate": 1.473214285714286e-06, | |
| "logps/chosen": -11.404391783887467, | |
| "logps/rejected": -10.326293621518568, | |
| "loss": 0.5037, | |
| "rewards/chosen": 0.0004199028701123679, | |
| "rewards/margins": 0.0008872187001457688, | |
| "rewards/rejected": -0.00046731583003340095, | |
| "step": 81 | |
| }, | |
| { | |
| "epoch": 1.4286751361161525, | |
| "grad_norm": 0.3459619879722595, | |
| "kl": 0.001983026973903179, | |
| "learning_rate": 1.4285714285714286e-06, | |
| "logps/chosen": -11.670273977308417, | |
| "logps/rejected": -10.357556482263513, | |
| "loss": 0.5033, | |
| "rewards/chosen": 0.0005598089203762649, | |
| "rewards/margins": 0.0007428220220635673, | |
| "rewards/rejected": -0.00018301310168730245, | |
| "step": 82 | |
| }, | |
| { | |
| "epoch": 1.4460980036297642, | |
| "grad_norm": 0.21906645596027374, | |
| "kl": 0.0028725937008857727, | |
| "learning_rate": 1.3839285714285715e-06, | |
| "logps/chosen": -11.741632576689632, | |
| "logps/rejected": -10.43268595062177, | |
| "loss": 0.5033, | |
| "rewards/chosen": 0.0006233549962832234, | |
| "rewards/margins": 0.001042224365689024, | |
| "rewards/rejected": -0.0004188693694058007, | |
| "step": 83 | |
| }, | |
| { | |
| "epoch": 1.4635208711433756, | |
| "grad_norm": 0.17626065015792847, | |
| "kl": 0.00280402391217649, | |
| "learning_rate": 1.3392857142857143e-06, | |
| "logps/chosen": -11.461028108891753, | |
| "logps/rejected": -10.382724480879935, | |
| "loss": 0.5028, | |
| "rewards/chosen": 0.000817755371639409, | |
| "rewards/margins": 0.0010563352754176859, | |
| "rewards/rejected": -0.00023857990377827696, | |
| "step": 84 | |
| }, | |
| { | |
| "epoch": 1.4809437386569873, | |
| "grad_norm": 0.21237261593341827, | |
| "kl": 0.002868844196200371, | |
| "learning_rate": 1.2946428571428574e-06, | |
| "logps/chosen": -11.554072961329156, | |
| "logps/rejected": -10.606980036953727, | |
| "loss": 0.5036, | |
| "rewards/chosen": 0.00040721653476554046, | |
| "rewards/margins": 0.0007238921818244932, | |
| "rewards/rejected": -0.00031667564705895274, | |
| "step": 85 | |
| }, | |
| { | |
| "epoch": 1.498366606170599, | |
| "grad_norm": 0.21006950736045837, | |
| "kl": 0.0021779753733426332, | |
| "learning_rate": 1.25e-06, | |
| "logps/chosen": -11.395751308954486, | |
| "logps/rejected": -10.509847842022815, | |
| "loss": 0.5039, | |
| "rewards/chosen": 0.0002953462833464932, | |
| "rewards/margins": 0.0009452639913045675, | |
| "rewards/rejected": -0.0006499177079580743, | |
| "step": 86 | |
| }, | |
| { | |
| "epoch": 1.5157894736842106, | |
| "grad_norm": 0.2044811248779297, | |
| "kl": 0.0027360066305845976, | |
| "learning_rate": 1.205357142857143e-06, | |
| "logps/chosen": -11.286482900943396, | |
| "logps/rejected": -10.226171383028968, | |
| "loss": 0.5036, | |
| "rewards/chosen": 0.00042764071345007963, | |
| "rewards/margins": 0.000754331256658454, | |
| "rewards/rejected": -0.00032669054320837447, | |
| "step": 87 | |
| }, | |
| { | |
| "epoch": 1.5332123411978222, | |
| "grad_norm": 0.1994449943304062, | |
| "kl": 0.0012993334094062448, | |
| "learning_rate": 1.160714285714286e-06, | |
| "logps/chosen": -11.429487209875607, | |
| "logps/rejected": -10.525211634260884, | |
| "loss": 0.5034, | |
| "rewards/chosen": 0.0006345899649036741, | |
| "rewards/margins": 0.0010494590537087978, | |
| "rewards/rejected": -0.0004148690888051237, | |
| "step": 88 | |
| }, | |
| { | |
| "epoch": 1.5506352087114337, | |
| "grad_norm": 0.2269240915775299, | |
| "kl": 0.0028356958646327257, | |
| "learning_rate": 1.1160714285714287e-06, | |
| "logps/chosen": -11.494557624315693, | |
| "logps/rejected": -10.647711232930673, | |
| "loss": 0.5039, | |
| "rewards/chosen": 0.0005171763055806032, | |
| "rewards/margins": 0.0010993786980465312, | |
| "rewards/rejected": -0.000582202392465928, | |
| "step": 89 | |
| }, | |
| { | |
| "epoch": 1.5680580762250453, | |
| "grad_norm": 0.17898203432559967, | |
| "kl": 0.0027232880238443613, | |
| "learning_rate": 1.0714285714285714e-06, | |
| "logps/chosen": -11.28170191988032, | |
| "logps/rejected": -10.416350695551658, | |
| "loss": 0.5031, | |
| "rewards/chosen": 0.0005683141978497201, | |
| "rewards/margins": 0.0011697266601863967, | |
| "rewards/rejected": -0.0006014124623366765, | |
| "step": 90 | |
| }, | |
| { | |
| "epoch": 1.585480943738657, | |
| "grad_norm": 0.30514323711395264, | |
| "kl": 0.003049398772418499, | |
| "learning_rate": 1.0267857142857143e-06, | |
| "logps/chosen": -11.509693008814102, | |
| "logps/rejected": -10.68894675925926, | |
| "loss": 0.5045, | |
| "rewards/chosen": 0.0003556648890177409, | |
| "rewards/margins": 0.0007876505769749797, | |
| "rewards/rejected": -0.0004319856879572389, | |
| "step": 91 | |
| }, | |
| { | |
| "epoch": 1.6029038112522684, | |
| "grad_norm": 0.22825570404529572, | |
| "kl": 0.001876774593256414, | |
| "learning_rate": 9.821428571428572e-07, | |
| "logps/chosen": -11.782056884765625, | |
| "logps/rejected": -10.345401929772418, | |
| "loss": 0.5035, | |
| "rewards/chosen": 0.0005325853452086449, | |
| "rewards/margins": 0.0011105023842790853, | |
| "rewards/rejected": -0.0005779170390704404, | |
| "step": 92 | |
| }, | |
| { | |
| "epoch": 1.6203266787658803, | |
| "grad_norm": 0.1923515647649765, | |
| "kl": 0.0026991041377186775, | |
| "learning_rate": 9.375000000000001e-07, | |
| "logps/chosen": -11.152751884075126, | |
| "logps/rejected": -9.97277569514449, | |
| "loss": 0.5036, | |
| "rewards/chosen": 0.0005818240043490824, | |
| "rewards/margins": 0.0010237199423968812, | |
| "rewards/rejected": -0.0004418959380477987, | |
| "step": 93 | |
| }, | |
| { | |
| "epoch": 1.6377495462794918, | |
| "grad_norm": 0.21557894349098206, | |
| "kl": 0.001241236925125122, | |
| "learning_rate": 8.928571428571429e-07, | |
| "logps/chosen": -11.167566068672839, | |
| "logps/rejected": -11.65214332601584, | |
| "loss": 0.5034, | |
| "rewards/chosen": 0.0005508374652744811, | |
| "rewards/margins": 0.0011452616979097689, | |
| "rewards/rejected": -0.0005944242326352879, | |
| "step": 94 | |
| }, | |
| { | |
| "epoch": 1.6551724137931034, | |
| "grad_norm": 0.15490390360355377, | |
| "kl": 0.0023578277323395014, | |
| "learning_rate": 8.482142857142859e-07, | |
| "logps/chosen": -11.630532976752022, | |
| "logps/rejected": -10.874779842962846, | |
| "loss": 0.5024, | |
| "rewards/chosen": 0.001025584428458201, | |
| "rewards/margins": 0.0015268274366321843, | |
| "rewards/rejected": -0.0005012430081739834, | |
| "step": 95 | |
| }, | |
| { | |
| "epoch": 1.672595281306715, | |
| "grad_norm": 0.21339836716651917, | |
| "kl": 0.0030644694343209267, | |
| "learning_rate": 8.035714285714287e-07, | |
| "logps/chosen": -11.422396998881075, | |
| "logps/rejected": -10.11446374544098, | |
| "loss": 0.5032, | |
| "rewards/chosen": 0.0006199539698603208, | |
| "rewards/margins": 0.0011987829361300795, | |
| "rewards/rejected": -0.0005788289662697587, | |
| "step": 96 | |
| }, | |
| { | |
| "epoch": 1.672595281306715, | |
| "eval_kl": 0.0012286369455978274, | |
| "eval_logps/chosen": -11.286660330636161, | |
| "eval_logps/rejected": -10.654095241001674, | |
| "eval_loss": 0.503455400466919, | |
| "eval_rewards/chosen": 0.0005875181938920702, | |
| "eval_rewards/margins": 0.0011167954320886306, | |
| "eval_rewards/rejected": -0.0005292772381965603, | |
| "eval_runtime": 41.5081, | |
| "eval_samples_per_second": 10.745, | |
| "eval_steps_per_second": 0.675, | |
| "step": 96 | |
| }, | |
| { | |
| "epoch": 1.6900181488203265, | |
| "grad_norm": 0.1822512149810791, | |
| "kl": 0.0018086197087541223, | |
| "learning_rate": 7.589285714285714e-07, | |
| "logps/chosen": -11.162734347797702, | |
| "logps/rejected": -10.495764249694377, | |
| "loss": 0.5024, | |
| "rewards/chosen": 0.0007518338957868911, | |
| "rewards/margins": 0.0012704190070132039, | |
| "rewards/rejected": -0.0005185851112263127, | |
| "step": 97 | |
| }, | |
| { | |
| "epoch": 1.7074410163339384, | |
| "grad_norm": 0.25983884930610657, | |
| "kl": 0.001475659548304975, | |
| "learning_rate": 7.142857142857143e-07, | |
| "logps/chosen": -11.400912198153408, | |
| "logps/rejected": -10.339601098037347, | |
| "loss": 0.5035, | |
| "rewards/chosen": 0.0009949224916371432, | |
| "rewards/margins": 0.0013851757258375045, | |
| "rewards/rejected": -0.00039025323420036127, | |
| "step": 98 | |
| }, | |
| { | |
| "epoch": 1.7248638838475499, | |
| "grad_norm": 0.18693022429943085, | |
| "kl": 0.002064686268568039, | |
| "learning_rate": 6.696428571428571e-07, | |
| "logps/chosen": -11.613883205191799, | |
| "logps/rejected": -10.563515374599358, | |
| "loss": 0.5033, | |
| "rewards/chosen": 0.0007188759310535653, | |
| "rewards/margins": 0.0014389748514110983, | |
| "rewards/rejected": -0.000720098920357533, | |
| "step": 99 | |
| }, | |
| { | |
| "epoch": 1.7422867513611615, | |
| "grad_norm": 0.2225840985774994, | |
| "kl": 0.0023758753668516874, | |
| "learning_rate": 6.25e-07, | |
| "logps/chosen": -11.64061211658971, | |
| "logps/rejected": -10.474972761688624, | |
| "loss": 0.503, | |
| "rewards/chosen": 0.0006008470986323495, | |
| "rewards/margins": 0.0013913730829674177, | |
| "rewards/rejected": -0.0007905259843350682, | |
| "step": 100 | |
| }, | |
| { | |
| "epoch": 1.7597096188747732, | |
| "grad_norm": 0.20069453120231628, | |
| "kl": 0.001968987053260207, | |
| "learning_rate": 5.80357142857143e-07, | |
| "logps/chosen": -11.33251953125, | |
| "logps/rejected": -10.533562193525599, | |
| "loss": 0.5042, | |
| "rewards/chosen": 0.0002462785234864877, | |
| "rewards/margins": 0.0005758832525453824, | |
| "rewards/rejected": -0.0003296047290588947, | |
| "step": 101 | |
| }, | |
| { | |
| "epoch": 1.7771324863883846, | |
| "grad_norm": 0.1894039809703827, | |
| "kl": 0.003226263215765357, | |
| "learning_rate": 5.357142857142857e-07, | |
| "logps/chosen": -11.40087890625, | |
| "logps/rejected": -10.571763671875, | |
| "loss": 0.504, | |
| "rewards/chosen": 0.00039598065174869607, | |
| "rewards/margins": 0.0011867985482737612, | |
| "rewards/rejected": -0.0007908178965250651, | |
| "step": 102 | |
| }, | |
| { | |
| "epoch": 1.7945553539019965, | |
| "grad_norm": 0.16288532316684723, | |
| "kl": 0.002215030835941434, | |
| "learning_rate": 4.910714285714286e-07, | |
| "logps/chosen": -11.702183131167763, | |
| "logps/rejected": -10.757870389014176, | |
| "loss": 0.5027, | |
| "rewards/chosen": 0.0007084523376665617, | |
| "rewards/margins": 0.001386926632437186, | |
| "rewards/rejected": -0.0006784742947706242, | |
| "step": 103 | |
| }, | |
| { | |
| "epoch": 1.811978221415608, | |
| "grad_norm": 0.18168872594833374, | |
| "kl": 0.0029251668602228165, | |
| "learning_rate": 4.4642857142857147e-07, | |
| "logps/chosen": -11.455708379330842, | |
| "logps/rejected": -11.033857421875, | |
| "loss": 0.5029, | |
| "rewards/chosen": 0.0005165521136444548, | |
| "rewards/margins": 0.0011331064335030059, | |
| "rewards/rejected": -0.0006165543198585511, | |
| "step": 104 | |
| }, | |
| { | |
| "epoch": 1.8294010889292196, | |
| "grad_norm": 0.17783309519290924, | |
| "kl": 0.0027887674514204264, | |
| "learning_rate": 4.0178571428571434e-07, | |
| "logps/chosen": -11.572852601396276, | |
| "logps/rejected": -9.61759107940051, | |
| "loss": 0.5029, | |
| "rewards/chosen": 0.0006706785965473093, | |
| "rewards/margins": 0.001759196485623659, | |
| "rewards/rejected": -0.0010885178890763497, | |
| "step": 105 | |
| }, | |
| { | |
| "epoch": 1.8468239564428313, | |
| "grad_norm": 0.18048368394374847, | |
| "kl": 0.0018043291056528687, | |
| "learning_rate": 3.5714285714285716e-07, | |
| "logps/chosen": -11.548997631962436, | |
| "logps/rejected": -11.18207649910013, | |
| "loss": 0.5038, | |
| "rewards/chosen": 0.00041619074468167953, | |
| "rewards/margins": 0.0005713092191551835, | |
| "rewards/rejected": -0.00015511847447350388, | |
| "step": 106 | |
| }, | |
| { | |
| "epoch": 1.8642468239564427, | |
| "grad_norm": 0.4144538938999176, | |
| "kl": 0.0020457797218114138, | |
| "learning_rate": 3.125e-07, | |
| "logps/chosen": -10.976453261876685, | |
| "logps/rejected": -10.985627066278338, | |
| "loss": 0.5048, | |
| "rewards/chosen": 0.00022244479537331512, | |
| "rewards/margins": 0.0006958120098764712, | |
| "rewards/rejected": -0.00047336721450315616, | |
| "step": 107 | |
| }, | |
| { | |
| "epoch": 1.8816696914700546, | |
| "grad_norm": 0.19312834739685059, | |
| "kl": 0.0014575794339179993, | |
| "learning_rate": 2.6785714285714284e-07, | |
| "logps/chosen": -11.37527155066906, | |
| "logps/rejected": -10.887072595373377, | |
| "loss": 0.5035, | |
| "rewards/chosen": 0.0006041409686091052, | |
| "rewards/margins": 0.0013794575971519186, | |
| "rewards/rejected": -0.0007753166285428134, | |
| "step": 108 | |
| }, | |
| { | |
| "epoch": 1.899092558983666, | |
| "grad_norm": 0.24906954169273376, | |
| "kl": 0.002399616641923785, | |
| "learning_rate": 2.2321428571428574e-07, | |
| "logps/chosen": -11.331646469763092, | |
| "logps/rejected": -10.6643505460235, | |
| "loss": 0.5046, | |
| "rewards/chosen": 0.0009270152843503881, | |
| "rewards/margins": 0.001616924349477436, | |
| "rewards/rejected": -0.0006899090651270479, | |
| "step": 109 | |
| }, | |
| { | |
| "epoch": 1.9165154264972777, | |
| "grad_norm": 0.1880314201116562, | |
| "kl": 0.0026240963488817215, | |
| "learning_rate": 1.7857142857142858e-07, | |
| "logps/chosen": -11.29613597972973, | |
| "logps/rejected": -10.615358285568467, | |
| "loss": 0.5031, | |
| "rewards/chosen": 0.0010521258051330979, | |
| "rewards/margins": 0.0013450532624429453, | |
| "rewards/rejected": -0.0002929274573098475, | |
| "step": 110 | |
| }, | |
| { | |
| "epoch": 1.9339382940108893, | |
| "grad_norm": 0.18609453737735748, | |
| "kl": 0.002484673634171486, | |
| "learning_rate": 1.3392857142857142e-07, | |
| "logps/chosen": -11.604450023938023, | |
| "logps/rejected": -10.78327236781785, | |
| "loss": 0.5028, | |
| "rewards/chosen": 0.0006836153836635494, | |
| "rewards/margins": 0.001532551150396621, | |
| "rewards/rejected": -0.0008489357667330717, | |
| "step": 111 | |
| }, | |
| { | |
| "epoch": 1.9513611615245008, | |
| "grad_norm": 0.20338238775730133, | |
| "kl": 0.0025711385533213615, | |
| "learning_rate": 8.928571428571429e-08, | |
| "logps/chosen": -11.439998413705585, | |
| "logps/rejected": -10.535859296666109, | |
| "loss": 0.5036, | |
| "rewards/chosen": 0.0005479781760782154, | |
| "rewards/margins": 0.0011845395663098528, | |
| "rewards/rejected": -0.0006365613902316374, | |
| "step": 112 | |
| }, | |
| { | |
| "epoch": 1.9513611615245008, | |
| "eval_kl": 0.000932498718611896, | |
| "eval_logps/chosen": -11.286882672991071, | |
| "eval_logps/rejected": -10.654556274414062, | |
| "eval_loss": 0.5036724805831909, | |
| "eval_rewards/chosen": 0.0005651861429214478, | |
| "eval_rewards/margins": 0.0011406249499746732, | |
| "eval_rewards/rejected": -0.0005754388070532254, | |
| "eval_runtime": 41.5261, | |
| "eval_samples_per_second": 10.74, | |
| "eval_steps_per_second": 0.674, | |
| "step": 112 | |
| }, | |
| { | |
| "epoch": 1.9687840290381127, | |
| "grad_norm": 0.18693436682224274, | |
| "kl": 0.0030837843660265207, | |
| "learning_rate": 4.4642857142857145e-08, | |
| "logps/chosen": -11.600828793174342, | |
| "logps/rejected": -10.943336722777062, | |
| "loss": 0.5031, | |
| "rewards/chosen": 0.0006475970541176044, | |
| "rewards/margins": 0.0013402001844336014, | |
| "rewards/rejected": -0.0006926031303159969, | |
| "step": 113 | |
| }, | |
| { | |
| "epoch": 1.986206896551724, | |
| "grad_norm": 0.1876712143421173, | |
| "kl": 0.0018025139579549432, | |
| "learning_rate": 0.0, | |
| "logps/chosen": -11.169455056662088, | |
| "logps/rejected": -10.222750522122524, | |
| "loss": 0.5029, | |
| "rewards/chosen": 0.0007852382221064725, | |
| "rewards/margins": 0.0014990826276489785, | |
| "rewards/rejected": -0.0007138444055425059, | |
| "step": 114 | |
| } | |
| ], | |
| "logging_steps": 1, | |
| "max_steps": 114, | |
| "num_input_tokens_seen": 0, | |
| "num_train_epochs": 2, | |
| "save_steps": 500, | |
| "stateful_callbacks": { | |
| "TrainerControl": { | |
| "args": { | |
| "should_epoch_stop": false, | |
| "should_evaluate": false, | |
| "should_log": false, | |
| "should_save": true, | |
| "should_training_stop": true | |
| }, | |
| "attributes": {} | |
| } | |
| }, | |
| "total_flos": 2.0165964255461376e+18, | |
| "train_batch_size": 2, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |