|
{ |
|
"best_metric": null, |
|
"best_model_checkpoint": null, |
|
"epoch": 0.10626587637595258, |
|
"global_step": 251, |
|
"is_hyper_param_search": false, |
|
"is_local_process_zero": true, |
|
"is_world_process_zero": true, |
|
"log_history": [ |
|
{ |
|
"epoch": 0.0, |
|
"learning_rate": 4.166666666666667e-06, |
|
"loss": 3.0643, |
|
"theoretical_loss": 3.321567680436603, |
|
"tokens_seen": 2990538752 |
|
}, |
|
{ |
|
"epoch": 0.0, |
|
"learning_rate": 8.333333333333334e-06, |
|
"loss": 3.0798, |
|
"theoretical_loss": 3.3215564803546, |
|
"tokens_seen": 2990669824 |
|
}, |
|
{ |
|
"epoch": 0.0, |
|
"learning_rate": 1.25e-05, |
|
"loss": 2.9318, |
|
"theoretical_loss": 3.321545280900887, |
|
"tokens_seen": 2990800896 |
|
}, |
|
{ |
|
"epoch": 0.0, |
|
"learning_rate": 1.6666666666666667e-05, |
|
"loss": 2.8098, |
|
"theoretical_loss": 3.3215340820754022, |
|
"tokens_seen": 2990931968 |
|
}, |
|
{ |
|
"epoch": 0.0, |
|
"learning_rate": 2.0833333333333336e-05, |
|
"loss": 2.7055, |
|
"theoretical_loss": 3.3215228838780817, |
|
"tokens_seen": 2991063040 |
|
}, |
|
{ |
|
"epoch": 0.0, |
|
"learning_rate": 2.5e-05, |
|
"loss": 2.9762, |
|
"theoretical_loss": 3.3215116863088636, |
|
"tokens_seen": 2991194112 |
|
}, |
|
{ |
|
"epoch": 0.0, |
|
"learning_rate": 2.916666666666667e-05, |
|
"loss": 2.8724, |
|
"theoretical_loss": 3.3215004893676854, |
|
"tokens_seen": 2991325184 |
|
}, |
|
{ |
|
"epoch": 0.0, |
|
"learning_rate": 3.3333333333333335e-05, |
|
"loss": 3.0452, |
|
"theoretical_loss": 3.321489293054483, |
|
"tokens_seen": 2991456256 |
|
}, |
|
{ |
|
"epoch": 0.0, |
|
"learning_rate": 3.7500000000000003e-05, |
|
"loss": 2.8676, |
|
"theoretical_loss": 3.321478097369195, |
|
"tokens_seen": 2991587328 |
|
}, |
|
{ |
|
"epoch": 0.0, |
|
"learning_rate": 4.166666666666667e-05, |
|
"loss": 2.8343, |
|
"theoretical_loss": 3.321466902311758, |
|
"tokens_seen": 2991718400 |
|
}, |
|
{ |
|
"epoch": 0.0, |
|
"learning_rate": 4.5833333333333334e-05, |
|
"loss": 2.743, |
|
"theoretical_loss": 3.3214557078821096, |
|
"tokens_seen": 2991849472 |
|
}, |
|
{ |
|
"epoch": 0.01, |
|
"learning_rate": 5e-05, |
|
"loss": 2.5867, |
|
"theoretical_loss": 3.321444514080187, |
|
"tokens_seen": 2991980544 |
|
}, |
|
{ |
|
"epoch": 0.01, |
|
"objective/train/docs_used": 1640856, |
|
"objective/train/instantaneous_batch_size": 16, |
|
"objective/train/instantaneous_microbatch_size": 16384, |
|
"objective/train/original_loss": 2.7297048568725586, |
|
"objective/train/theoretical_loss": 3.321438917414603, |
|
"objective/train/tokens_used": 22097376, |
|
"theoretical_loss": 3.321438917414603, |
|
"tokens_seen": 2992046080 |
|
}, |
|
{ |
|
"epoch": 0.01, |
|
"learning_rate": 5.4166666666666664e-05, |
|
"loss": 2.7262, |
|
"theoretical_loss": 3.321433320905927, |
|
"tokens_seen": 2992111616 |
|
}, |
|
{ |
|
"epoch": 0.01, |
|
"learning_rate": 5.833333333333334e-05, |
|
"loss": 2.6517, |
|
"theoretical_loss": 3.3214221283592678, |
|
"tokens_seen": 2992242688 |
|
}, |
|
{ |
|
"epoch": 0.01, |
|
"learning_rate": 6.25e-05, |
|
"loss": 2.9399, |
|
"theoretical_loss": 3.321410936440146, |
|
"tokens_seen": 2992373760 |
|
}, |
|
{ |
|
"epoch": 0.01, |
|
"learning_rate": 6.666666666666667e-05, |
|
"loss": 2.7939, |
|
"theoretical_loss": 3.3213997451485, |
|
"tokens_seen": 2992504832 |
|
}, |
|
{ |
|
"epoch": 0.01, |
|
"learning_rate": 7.083333333333334e-05, |
|
"loss": 2.5715, |
|
"theoretical_loss": 3.3213885544842654, |
|
"tokens_seen": 2992635904 |
|
}, |
|
{ |
|
"epoch": 0.01, |
|
"learning_rate": 7.500000000000001e-05, |
|
"loss": 2.6047, |
|
"theoretical_loss": 3.321377364447381, |
|
"tokens_seen": 2992766976 |
|
}, |
|
{ |
|
"epoch": 0.01, |
|
"learning_rate": 7.916666666666666e-05, |
|
"loss": 2.6736, |
|
"theoretical_loss": 3.3213661750377836, |
|
"tokens_seen": 2992898048 |
|
}, |
|
{ |
|
"epoch": 0.01, |
|
"learning_rate": 8.333333333333334e-05, |
|
"loss": 2.6853, |
|
"theoretical_loss": 3.3213549862554106, |
|
"tokens_seen": 2993029120 |
|
}, |
|
{ |
|
"epoch": 0.01, |
|
"learning_rate": 8.75e-05, |
|
"loss": 2.3665, |
|
"theoretical_loss": 3.3213437981001994, |
|
"tokens_seen": 2993160192 |
|
}, |
|
{ |
|
"epoch": 0.01, |
|
"learning_rate": 9.166666666666667e-05, |
|
"loss": 2.618, |
|
"theoretical_loss": 3.3213326105720875, |
|
"tokens_seen": 2993291264 |
|
}, |
|
{ |
|
"epoch": 0.01, |
|
"learning_rate": 9.583333333333334e-05, |
|
"loss": 2.6614, |
|
"theoretical_loss": 3.3213214236710122, |
|
"tokens_seen": 2993422336 |
|
}, |
|
{ |
|
"epoch": 0.01, |
|
"learning_rate": 0.0001, |
|
"loss": 2.6163, |
|
"theoretical_loss": 3.321310237396911, |
|
"tokens_seen": 2993553408 |
|
}, |
|
{ |
|
"epoch": 0.01, |
|
"objective/train/docs_used": 1641461, |
|
"objective/train/instantaneous_batch_size": 16, |
|
"objective/train/instantaneous_microbatch_size": 16384, |
|
"objective/train/original_loss": 2.5642035007476807, |
|
"objective/train/theoretical_loss": 3.3212990517497207, |
|
"objective/train/tokens_used": 23735776, |
|
"theoretical_loss": 3.3212990517497207, |
|
"tokens_seen": 2993684480 |
|
}, |
|
{ |
|
"epoch": 0.01, |
|
"learning_rate": 9.995722840034217e-05, |
|
"loss": 2.7218, |
|
"theoretical_loss": 3.3212990517497207, |
|
"tokens_seen": 2993684480 |
|
}, |
|
{ |
|
"epoch": 0.01, |
|
"learning_rate": 9.991445680068435e-05, |
|
"loss": 2.5137, |
|
"theoretical_loss": 3.3212878667293797, |
|
"tokens_seen": 2993815552 |
|
}, |
|
{ |
|
"epoch": 0.01, |
|
"learning_rate": 9.987168520102653e-05, |
|
"loss": 2.5283, |
|
"theoretical_loss": 3.321276682335825, |
|
"tokens_seen": 2993946624 |
|
}, |
|
{ |
|
"epoch": 0.01, |
|
"learning_rate": 9.98289136013687e-05, |
|
"loss": 2.6367, |
|
"theoretical_loss": 3.3212654985689936, |
|
"tokens_seen": 2994077696 |
|
}, |
|
{ |
|
"epoch": 0.01, |
|
"learning_rate": 9.978614200171087e-05, |
|
"loss": 2.5822, |
|
"theoretical_loss": 3.3212543154288237, |
|
"tokens_seen": 2994208768 |
|
}, |
|
{ |
|
"epoch": 0.01, |
|
"learning_rate": 9.974337040205303e-05, |
|
"loss": 2.6296, |
|
"theoretical_loss": 3.3212431329152525, |
|
"tokens_seen": 2994339840 |
|
}, |
|
{ |
|
"epoch": 0.01, |
|
"learning_rate": 9.970059880239521e-05, |
|
"loss": 2.5596, |
|
"theoretical_loss": 3.321231951028217, |
|
"tokens_seen": 2994470912 |
|
}, |
|
{ |
|
"epoch": 0.01, |
|
"learning_rate": 9.965782720273739e-05, |
|
"loss": 2.5663, |
|
"theoretical_loss": 3.3212207697676552, |
|
"tokens_seen": 2994601984 |
|
}, |
|
{ |
|
"epoch": 0.01, |
|
"learning_rate": 9.961505560307956e-05, |
|
"loss": 2.5138, |
|
"theoretical_loss": 3.3212095891335043, |
|
"tokens_seen": 2994733056 |
|
}, |
|
{ |
|
"epoch": 0.01, |
|
"learning_rate": 9.957228400342173e-05, |
|
"loss": 2.5938, |
|
"theoretical_loss": 3.321198409125702, |
|
"tokens_seen": 2994864128 |
|
}, |
|
{ |
|
"epoch": 0.01, |
|
"learning_rate": 9.95295124037639e-05, |
|
"loss": 2.4583, |
|
"theoretical_loss": 3.321187229744186, |
|
"tokens_seen": 2994995200 |
|
}, |
|
{ |
|
"epoch": 0.02, |
|
"learning_rate": 9.948674080410608e-05, |
|
"loss": 2.5265, |
|
"theoretical_loss": 3.321176050988893, |
|
"tokens_seen": 2995126272 |
|
}, |
|
{ |
|
"epoch": 0.02, |
|
"learning_rate": 9.944396920444825e-05, |
|
"loss": 2.7376, |
|
"theoretical_loss": 3.3211648728597614, |
|
"tokens_seen": 2995257344 |
|
}, |
|
{ |
|
"epoch": 0.02, |
|
"objective/train/docs_used": 1642666, |
|
"objective/train/instantaneous_batch_size": 16, |
|
"objective/train/instantaneous_microbatch_size": 16384, |
|
"objective/train/original_loss": 2.7178521156311035, |
|
"objective/train/theoretical_loss": 3.3211592840299864, |
|
"objective/train/tokens_used": 25374176, |
|
"theoretical_loss": 3.3211592840299864, |
|
"tokens_seen": 2995322880 |
|
}, |
|
{ |
|
"epoch": 0.02, |
|
"learning_rate": 9.940119760479042e-05, |
|
"loss": 2.4464, |
|
"theoretical_loss": 3.3211536953567284, |
|
"tokens_seen": 2995388416 |
|
}, |
|
{ |
|
"epoch": 0.02, |
|
"learning_rate": 9.93584260051326e-05, |
|
"loss": 2.657, |
|
"theoretical_loss": 3.321142518479731, |
|
"tokens_seen": 2995519488 |
|
}, |
|
{ |
|
"epoch": 0.02, |
|
"learning_rate": 9.931565440547476e-05, |
|
"loss": 2.5525, |
|
"theoretical_loss": 3.321131342228708, |
|
"tokens_seen": 2995650560 |
|
}, |
|
{ |
|
"epoch": 0.02, |
|
"learning_rate": 9.927288280581694e-05, |
|
"loss": 2.6099, |
|
"theoretical_loss": 3.321120166603596, |
|
"tokens_seen": 2995781632 |
|
}, |
|
{ |
|
"epoch": 0.02, |
|
"learning_rate": 9.923011120615912e-05, |
|
"loss": 2.4712, |
|
"theoretical_loss": 3.3211089916043326, |
|
"tokens_seen": 2995912704 |
|
}, |
|
{ |
|
"epoch": 0.02, |
|
"learning_rate": 9.918733960650128e-05, |
|
"loss": 2.514, |
|
"theoretical_loss": 3.3210978172308554, |
|
"tokens_seen": 2996043776 |
|
}, |
|
{ |
|
"epoch": 0.02, |
|
"learning_rate": 9.914456800684346e-05, |
|
"loss": 2.4037, |
|
"theoretical_loss": 3.3210866434831026, |
|
"tokens_seen": 2996174848 |
|
}, |
|
{ |
|
"epoch": 0.02, |
|
"learning_rate": 9.910179640718563e-05, |
|
"loss": 2.4715, |
|
"theoretical_loss": 3.3210754703610106, |
|
"tokens_seen": 2996305920 |
|
}, |
|
{ |
|
"epoch": 0.02, |
|
"learning_rate": 9.90590248075278e-05, |
|
"loss": 2.4805, |
|
"theoretical_loss": 3.321064297864518, |
|
"tokens_seen": 2996436992 |
|
}, |
|
{ |
|
"epoch": 0.02, |
|
"learning_rate": 9.901625320786998e-05, |
|
"loss": 2.4755, |
|
"theoretical_loss": 3.3210531259935627, |
|
"tokens_seen": 2996568064 |
|
}, |
|
{ |
|
"epoch": 0.02, |
|
"learning_rate": 9.897348160821215e-05, |
|
"loss": 2.5174, |
|
"theoretical_loss": 3.321041954748081, |
|
"tokens_seen": 2996699136 |
|
}, |
|
{ |
|
"epoch": 0.02, |
|
"learning_rate": 9.893071000855433e-05, |
|
"loss": 2.5996, |
|
"theoretical_loss": 3.321030784128012, |
|
"tokens_seen": 2996830208 |
|
}, |
|
{ |
|
"epoch": 0.02, |
|
"objective/train/docs_used": 1643300, |
|
"objective/train/instantaneous_batch_size": 16, |
|
"objective/train/instantaneous_microbatch_size": 16384, |
|
"objective/train/original_loss": 2.879695415496826, |
|
"objective/train/theoretical_loss": 3.321019614133292, |
|
"objective/train/tokens_used": 27012576, |
|
"theoretical_loss": 3.321019614133292, |
|
"tokens_seen": 2996961280 |
|
}, |
|
{ |
|
"epoch": 0.02, |
|
"learning_rate": 9.888793840889649e-05, |
|
"loss": 2.5696, |
|
"theoretical_loss": 3.321019614133292, |
|
"tokens_seen": 2996961280 |
|
}, |
|
{ |
|
"epoch": 0.02, |
|
"learning_rate": 9.884516680923867e-05, |
|
"loss": 2.607, |
|
"theoretical_loss": 3.3210084447638595, |
|
"tokens_seen": 2997092352 |
|
}, |
|
{ |
|
"epoch": 0.02, |
|
"learning_rate": 9.880239520958085e-05, |
|
"loss": 2.4604, |
|
"theoretical_loss": 3.320997276019652, |
|
"tokens_seen": 2997223424 |
|
}, |
|
{ |
|
"epoch": 0.02, |
|
"learning_rate": 9.875962360992301e-05, |
|
"loss": 2.4603, |
|
"theoretical_loss": 3.3209861079006067, |
|
"tokens_seen": 2997354496 |
|
}, |
|
{ |
|
"epoch": 0.02, |
|
"learning_rate": 9.871685201026519e-05, |
|
"loss": 2.4374, |
|
"theoretical_loss": 3.320974940406662, |
|
"tokens_seen": 2997485568 |
|
}, |
|
{ |
|
"epoch": 0.02, |
|
"learning_rate": 9.867408041060736e-05, |
|
"loss": 2.5285, |
|
"theoretical_loss": 3.320963773537755, |
|
"tokens_seen": 2997616640 |
|
}, |
|
{ |
|
"epoch": 0.02, |
|
"learning_rate": 9.863130881094953e-05, |
|
"loss": 2.5895, |
|
"theoretical_loss": 3.320952607293824, |
|
"tokens_seen": 2997747712 |
|
}, |
|
{ |
|
"epoch": 0.02, |
|
"learning_rate": 9.858853721129171e-05, |
|
"loss": 2.5835, |
|
"theoretical_loss": 3.320941441674806, |
|
"tokens_seen": 2997878784 |
|
}, |
|
{ |
|
"epoch": 0.02, |
|
"learning_rate": 9.854576561163388e-05, |
|
"loss": 2.7362, |
|
"theoretical_loss": 3.320930276680639, |
|
"tokens_seen": 2998009856 |
|
}, |
|
{ |
|
"epoch": 0.02, |
|
"learning_rate": 9.850299401197606e-05, |
|
"loss": 2.5553, |
|
"theoretical_loss": 3.3209191123112607, |
|
"tokens_seen": 2998140928 |
|
}, |
|
{ |
|
"epoch": 0.03, |
|
"learning_rate": 9.846022241231822e-05, |
|
"loss": 2.6403, |
|
"theoretical_loss": 3.320907948566609, |
|
"tokens_seen": 2998272000 |
|
}, |
|
{ |
|
"epoch": 0.03, |
|
"learning_rate": 9.84174508126604e-05, |
|
"loss": 2.5129, |
|
"theoretical_loss": 3.3208967854466214, |
|
"tokens_seen": 2998403072 |
|
}, |
|
{ |
|
"epoch": 0.03, |
|
"learning_rate": 9.837467921300258e-05, |
|
"loss": 2.4238, |
|
"theoretical_loss": 3.3208856229512356, |
|
"tokens_seen": 2998534144 |
|
}, |
|
{ |
|
"epoch": 0.03, |
|
"objective/train/docs_used": 1644380, |
|
"objective/train/instantaneous_batch_size": 16, |
|
"objective/train/instantaneous_microbatch_size": 16384, |
|
"objective/train/original_loss": 2.4041128158569336, |
|
"objective/train/theoretical_loss": 3.320880041937749, |
|
"objective/train/tokens_used": 28650976, |
|
"theoretical_loss": 3.320880041937749, |
|
"tokens_seen": 2998599680 |
|
}, |
|
{ |
|
"epoch": 0.03, |
|
"learning_rate": 9.833190761334474e-05, |
|
"loss": 2.3705, |
|
"theoretical_loss": 3.3208744610803898, |
|
"tokens_seen": 2998665216 |
|
}, |
|
{ |
|
"epoch": 0.03, |
|
"learning_rate": 9.828913601368692e-05, |
|
"loss": 2.4602, |
|
"theoretical_loss": 3.320863299834021, |
|
"tokens_seen": 2998796288 |
|
}, |
|
{ |
|
"epoch": 0.03, |
|
"learning_rate": 9.824636441402908e-05, |
|
"loss": 2.5492, |
|
"theoretical_loss": 3.320852139212068, |
|
"tokens_seen": 2998927360 |
|
}, |
|
{ |
|
"epoch": 0.03, |
|
"learning_rate": 9.820359281437126e-05, |
|
"loss": 2.426, |
|
"theoretical_loss": 3.3208409792144677, |
|
"tokens_seen": 2999058432 |
|
}, |
|
{ |
|
"epoch": 0.03, |
|
"learning_rate": 9.816082121471344e-05, |
|
"loss": 2.4079, |
|
"theoretical_loss": 3.320829819841158, |
|
"tokens_seen": 2999189504 |
|
}, |
|
{ |
|
"epoch": 0.03, |
|
"learning_rate": 9.81180496150556e-05, |
|
"loss": 2.5189, |
|
"theoretical_loss": 3.320818661092077, |
|
"tokens_seen": 2999320576 |
|
}, |
|
{ |
|
"epoch": 0.03, |
|
"learning_rate": 9.807527801539777e-05, |
|
"loss": 2.4085, |
|
"theoretical_loss": 3.3208075029671624, |
|
"tokens_seen": 2999451648 |
|
}, |
|
{ |
|
"epoch": 0.03, |
|
"learning_rate": 9.803250641573995e-05, |
|
"loss": 2.4031, |
|
"theoretical_loss": 3.320796345466352, |
|
"tokens_seen": 2999582720 |
|
}, |
|
{ |
|
"epoch": 0.03, |
|
"learning_rate": 9.798973481608213e-05, |
|
"loss": 2.5067, |
|
"theoretical_loss": 3.320785188589584, |
|
"tokens_seen": 2999713792 |
|
}, |
|
{ |
|
"epoch": 0.03, |
|
"learning_rate": 9.79469632164243e-05, |
|
"loss": 2.4357, |
|
"theoretical_loss": 3.3207740323367956, |
|
"tokens_seen": 2999844864 |
|
}, |
|
{ |
|
"epoch": 0.03, |
|
"learning_rate": 9.790419161676647e-05, |
|
"loss": 2.4929, |
|
"theoretical_loss": 3.3207628767079242, |
|
"tokens_seen": 2999975936 |
|
}, |
|
{ |
|
"epoch": 0.03, |
|
"learning_rate": 9.786142001710863e-05, |
|
"loss": 2.4052, |
|
"theoretical_loss": 3.3207517217029094, |
|
"tokens_seen": 3000107008 |
|
}, |
|
{ |
|
"epoch": 0.03, |
|
"objective/train/docs_used": 1645056, |
|
"objective/train/instantaneous_batch_size": 16, |
|
"objective/train/instantaneous_microbatch_size": 16384, |
|
"objective/train/original_loss": 2.3271520137786865, |
|
"objective/train/theoretical_loss": 3.3207405673216877, |
|
"objective/train/tokens_used": 30289376, |
|
"theoretical_loss": 3.3207405673216877, |
|
"tokens_seen": 3000238080 |
|
}, |
|
{ |
|
"epoch": 0.03, |
|
"learning_rate": 9.781864841745081e-05, |
|
"loss": 2.5201, |
|
"theoretical_loss": 3.3207405673216877, |
|
"tokens_seen": 3000238080 |
|
}, |
|
{ |
|
"epoch": 0.03, |
|
"learning_rate": 9.777587681779299e-05, |
|
"loss": 2.5431, |
|
"theoretical_loss": 3.320729413564197, |
|
"tokens_seen": 3000369152 |
|
}, |
|
{ |
|
"epoch": 0.03, |
|
"learning_rate": 9.773310521813517e-05, |
|
"loss": 2.5359, |
|
"theoretical_loss": 3.3207182604303753, |
|
"tokens_seen": 3000500224 |
|
}, |
|
{ |
|
"epoch": 0.03, |
|
"learning_rate": 9.769033361847733e-05, |
|
"loss": 2.1929, |
|
"theoretical_loss": 3.320707107920161, |
|
"tokens_seen": 3000631296 |
|
}, |
|
{ |
|
"epoch": 0.03, |
|
"learning_rate": 9.76475620188195e-05, |
|
"loss": 2.5196, |
|
"theoretical_loss": 3.3206959560334917, |
|
"tokens_seen": 3000762368 |
|
}, |
|
{ |
|
"epoch": 0.03, |
|
"learning_rate": 9.760479041916169e-05, |
|
"loss": 2.3645, |
|
"theoretical_loss": 3.320684804770305, |
|
"tokens_seen": 3000893440 |
|
}, |
|
{ |
|
"epoch": 0.03, |
|
"learning_rate": 9.756201881950386e-05, |
|
"loss": 2.5329, |
|
"theoretical_loss": 3.3206736541305393, |
|
"tokens_seen": 3001024512 |
|
}, |
|
{ |
|
"epoch": 0.03, |
|
"learning_rate": 9.751924721984602e-05, |
|
"loss": 2.533, |
|
"theoretical_loss": 3.3206625041141318, |
|
"tokens_seen": 3001155584 |
|
}, |
|
{ |
|
"epoch": 0.04, |
|
"learning_rate": 9.74764756201882e-05, |
|
"loss": 2.4702, |
|
"theoretical_loss": 3.3206513547210212, |
|
"tokens_seen": 3001286656 |
|
}, |
|
{ |
|
"epoch": 0.04, |
|
"learning_rate": 9.743370402053036e-05, |
|
"loss": 2.6255, |
|
"theoretical_loss": 3.320640205951145, |
|
"tokens_seen": 3001417728 |
|
}, |
|
{ |
|
"epoch": 0.04, |
|
"learning_rate": 9.739093242087256e-05, |
|
"loss": 2.5139, |
|
"theoretical_loss": 3.3206290578044415, |
|
"tokens_seen": 3001548800 |
|
}, |
|
{ |
|
"epoch": 0.04, |
|
"learning_rate": 9.734816082121472e-05, |
|
"loss": 2.3796, |
|
"theoretical_loss": 3.3206179102808484, |
|
"tokens_seen": 3001679872 |
|
}, |
|
{ |
|
"epoch": 0.04, |
|
"learning_rate": 9.730538922155689e-05, |
|
"loss": 2.4997, |
|
"theoretical_loss": 3.3206067633803036, |
|
"tokens_seen": 3001810944 |
|
}, |
|
{ |
|
"epoch": 0.04, |
|
"objective/train/docs_used": 1646327, |
|
"objective/train/instantaneous_batch_size": 16, |
|
"objective/train/instantaneous_microbatch_size": 16384, |
|
"objective/train/original_loss": 2.398160219192505, |
|
"objective/train/theoretical_loss": 3.320601190163655, |
|
"objective/train/tokens_used": 31927776, |
|
"theoretical_loss": 3.320601190163655, |
|
"tokens_seen": 3001876480 |
|
}, |
|
{ |
|
"epoch": 0.04, |
|
"learning_rate": 9.726261762189906e-05, |
|
"loss": 2.3607, |
|
"theoretical_loss": 3.320595617102745, |
|
"tokens_seen": 3001942016 |
|
}, |
|
{ |
|
"epoch": 0.04, |
|
"learning_rate": 9.721984602224123e-05, |
|
"loss": 2.4803, |
|
"theoretical_loss": 3.320584471448111, |
|
"tokens_seen": 3002073088 |
|
}, |
|
{ |
|
"epoch": 0.04, |
|
"learning_rate": 9.717707442258342e-05, |
|
"loss": 2.348, |
|
"theoretical_loss": 3.3205733264163393, |
|
"tokens_seen": 3002204160 |
|
}, |
|
{ |
|
"epoch": 0.04, |
|
"learning_rate": 9.713430282292558e-05, |
|
"loss": 2.2828, |
|
"theoretical_loss": 3.320562182007368, |
|
"tokens_seen": 3002335232 |
|
}, |
|
{ |
|
"epoch": 0.04, |
|
"learning_rate": 9.709153122326775e-05, |
|
"loss": 2.4615, |
|
"theoretical_loss": 3.320551038221135, |
|
"tokens_seen": 3002466304 |
|
}, |
|
{ |
|
"epoch": 0.04, |
|
"learning_rate": 9.704875962360993e-05, |
|
"loss": 2.3988, |
|
"theoretical_loss": 3.3205398950575784, |
|
"tokens_seen": 3002597376 |
|
}, |
|
{ |
|
"epoch": 0.04, |
|
"learning_rate": 9.700598802395209e-05, |
|
"loss": 2.5582, |
|
"theoretical_loss": 3.320528752516636, |
|
"tokens_seen": 3002728448 |
|
}, |
|
{ |
|
"epoch": 0.04, |
|
"learning_rate": 9.696321642429428e-05, |
|
"loss": 2.3266, |
|
"theoretical_loss": 3.3205176105982463, |
|
"tokens_seen": 3002859520 |
|
}, |
|
{ |
|
"epoch": 0.04, |
|
"learning_rate": 9.692044482463645e-05, |
|
"loss": 2.5922, |
|
"theoretical_loss": 3.320506469302347, |
|
"tokens_seen": 3002990592 |
|
}, |
|
{ |
|
"epoch": 0.04, |
|
"learning_rate": 9.687767322497861e-05, |
|
"loss": 2.4959, |
|
"theoretical_loss": 3.3204953286288763, |
|
"tokens_seen": 3003121664 |
|
}, |
|
{ |
|
"epoch": 0.04, |
|
"learning_rate": 9.683490162532079e-05, |
|
"loss": 2.6068, |
|
"theoretical_loss": 3.3204841885777725, |
|
"tokens_seen": 3003252736 |
|
}, |
|
{ |
|
"epoch": 0.04, |
|
"learning_rate": 9.679213002566297e-05, |
|
"loss": 2.3996, |
|
"theoretical_loss": 3.3204730491489727, |
|
"tokens_seen": 3003383808 |
|
}, |
|
{ |
|
"epoch": 0.04, |
|
"objective/train/docs_used": 1647543, |
|
"objective/train/instantaneous_batch_size": 16, |
|
"objective/train/instantaneous_microbatch_size": 16384, |
|
"objective/train/original_loss": 2.3885486125946045, |
|
"objective/train/theoretical_loss": 3.3204619103424164, |
|
"objective/train/tokens_used": 33566176, |
|
"theoretical_loss": 3.3204619103424164, |
|
"tokens_seen": 3003514880 |
|
}, |
|
{ |
|
"epoch": 0.04, |
|
"learning_rate": 9.674935842600514e-05, |
|
"loss": 2.4692, |
|
"theoretical_loss": 3.3204619103424164, |
|
"tokens_seen": 3003514880 |
|
}, |
|
{ |
|
"epoch": 0.04, |
|
"learning_rate": 9.670658682634731e-05, |
|
"loss": 2.35, |
|
"theoretical_loss": 3.3204507721580403, |
|
"tokens_seen": 3003645952 |
|
}, |
|
{ |
|
"epoch": 0.04, |
|
"learning_rate": 9.666381522668948e-05, |
|
"loss": 2.3168, |
|
"theoretical_loss": 3.3204396345957834, |
|
"tokens_seen": 3003777024 |
|
}, |
|
{ |
|
"epoch": 0.04, |
|
"learning_rate": 9.662104362703166e-05, |
|
"loss": 2.4671, |
|
"theoretical_loss": 3.320428497655584, |
|
"tokens_seen": 3003908096 |
|
}, |
|
{ |
|
"epoch": 0.04, |
|
"learning_rate": 9.657827202737383e-05, |
|
"loss": 2.552, |
|
"theoretical_loss": 3.320417361337379, |
|
"tokens_seen": 3004039168 |
|
}, |
|
{ |
|
"epoch": 0.04, |
|
"learning_rate": 9.6535500427716e-05, |
|
"loss": 2.3902, |
|
"theoretical_loss": 3.3204062256411078, |
|
"tokens_seen": 3004170240 |
|
}, |
|
{ |
|
"epoch": 0.04, |
|
"learning_rate": 9.649272882805818e-05, |
|
"loss": 2.491, |
|
"theoretical_loss": 3.320395090566708, |
|
"tokens_seen": 3004301312 |
|
}, |
|
{ |
|
"epoch": 0.05, |
|
"learning_rate": 9.644995722840034e-05, |
|
"loss": 2.4628, |
|
"theoretical_loss": 3.3203839561141173, |
|
"tokens_seen": 3004432384 |
|
}, |
|
{ |
|
"epoch": 0.05, |
|
"learning_rate": 9.640718562874252e-05, |
|
"loss": 2.4858, |
|
"theoretical_loss": 3.320372822283275, |
|
"tokens_seen": 3004563456 |
|
}, |
|
{ |
|
"epoch": 0.05, |
|
"learning_rate": 9.63644140290847e-05, |
|
"loss": 2.5038, |
|
"theoretical_loss": 3.3203616890741183, |
|
"tokens_seen": 3004694528 |
|
}, |
|
{ |
|
"epoch": 0.05, |
|
"learning_rate": 9.632164242942686e-05, |
|
"loss": 2.2651, |
|
"theoretical_loss": 3.3203505564865856, |
|
"tokens_seen": 3004825600 |
|
}, |
|
{ |
|
"epoch": 0.05, |
|
"learning_rate": 9.627887082976904e-05, |
|
"loss": 2.4591, |
|
"theoretical_loss": 3.3203394245206153, |
|
"tokens_seen": 3004956672 |
|
}, |
|
{ |
|
"epoch": 0.05, |
|
"learning_rate": 9.623609923011121e-05, |
|
"loss": 2.4003, |
|
"theoretical_loss": 3.320328293176145, |
|
"tokens_seen": 3005087744 |
|
}, |
|
{ |
|
"epoch": 0.05, |
|
"objective/train/docs_used": 1648109, |
|
"objective/train/instantaneous_batch_size": 16, |
|
"objective/train/instantaneous_microbatch_size": 16384, |
|
"objective/train/original_loss": 2.3964030742645264, |
|
"objective/train/theoretical_loss": 3.3203227277369534, |
|
"objective/train/tokens_used": 35204576, |
|
"theoretical_loss": 3.3203227277369534, |
|
"tokens_seen": 3005153280 |
|
}, |
|
{ |
|
"epoch": 0.05, |
|
"learning_rate": 9.619332763045337e-05, |
|
"loss": 2.3952, |
|
"theoretical_loss": 3.320317162453114, |
|
"tokens_seen": 3005218816 |
|
}, |
|
{ |
|
"epoch": 0.05, |
|
"learning_rate": 9.615055603079556e-05, |
|
"loss": 2.4367, |
|
"theoretical_loss": 3.3203060323514593, |
|
"tokens_seen": 3005349888 |
|
}, |
|
{ |
|
"epoch": 0.05, |
|
"learning_rate": 9.610778443113773e-05, |
|
"loss": 2.4468, |
|
"theoretical_loss": 3.3202949028711197, |
|
"tokens_seen": 3005480960 |
|
}, |
|
{ |
|
"epoch": 0.05, |
|
"learning_rate": 9.60650128314799e-05, |
|
"loss": 2.3551, |
|
"theoretical_loss": 3.3202837740120335, |
|
"tokens_seen": 3005612032 |
|
}, |
|
{ |
|
"epoch": 0.05, |
|
"learning_rate": 9.602224123182207e-05, |
|
"loss": 2.3886, |
|
"theoretical_loss": 3.3202726457741387, |
|
"tokens_seen": 3005743104 |
|
}, |
|
{ |
|
"epoch": 0.05, |
|
"learning_rate": 9.597946963216424e-05, |
|
"loss": 2.4953, |
|
"theoretical_loss": 3.320261518157374, |
|
"tokens_seen": 3005874176 |
|
}, |
|
{ |
|
"epoch": 0.05, |
|
"learning_rate": 9.593669803250643e-05, |
|
"loss": 2.3074, |
|
"theoretical_loss": 3.3202503911616765, |
|
"tokens_seen": 3006005248 |
|
}, |
|
{ |
|
"epoch": 0.05, |
|
"learning_rate": 9.589392643284859e-05, |
|
"loss": 2.4135, |
|
"theoretical_loss": 3.320239264786986, |
|
"tokens_seen": 3006136320 |
|
}, |
|
{ |
|
"epoch": 0.05, |
|
"learning_rate": 9.585115483319077e-05, |
|
"loss": 2.431, |
|
"theoretical_loss": 3.3202281390332393, |
|
"tokens_seen": 3006267392 |
|
}, |
|
{ |
|
"epoch": 0.05, |
|
"learning_rate": 9.580838323353294e-05, |
|
"loss": 2.3277, |
|
"theoretical_loss": 3.320217013900376, |
|
"tokens_seen": 3006398464 |
|
}, |
|
{ |
|
"epoch": 0.05, |
|
"learning_rate": 9.576561163387511e-05, |
|
"loss": 2.5083, |
|
"theoretical_loss": 3.3202058893883333, |
|
"tokens_seen": 3006529536 |
|
}, |
|
{ |
|
"epoch": 0.05, |
|
"learning_rate": 9.572284003421729e-05, |
|
"loss": 2.4582, |
|
"theoretical_loss": 3.3201947654970505, |
|
"tokens_seen": 3006660608 |
|
}, |
|
{ |
|
"epoch": 0.05, |
|
"objective/train/docs_used": 1649212, |
|
"objective/train/instantaneous_batch_size": 16, |
|
"objective/train/instantaneous_microbatch_size": 16384, |
|
"objective/train/original_loss": 2.5212416648864746, |
|
"objective/train/theoretical_loss": 3.320183642226465, |
|
"objective/train/tokens_used": 36842976, |
|
"theoretical_loss": 3.320183642226465, |
|
"tokens_seen": 3006791680 |
|
}, |
|
{ |
|
"epoch": 0.05, |
|
"learning_rate": 9.568006843455946e-05, |
|
"loss": 2.3596, |
|
"theoretical_loss": 3.320183642226465, |
|
"tokens_seen": 3006791680 |
|
}, |
|
{ |
|
"epoch": 0.05, |
|
"learning_rate": 9.563729683490164e-05, |
|
"loss": 2.3923, |
|
"theoretical_loss": 3.3201725195765155, |
|
"tokens_seen": 3006922752 |
|
}, |
|
{ |
|
"epoch": 0.05, |
|
"learning_rate": 9.55945252352438e-05, |
|
"loss": 2.3563, |
|
"theoretical_loss": 3.3201613975471402, |
|
"tokens_seen": 3007053824 |
|
}, |
|
{ |
|
"epoch": 0.05, |
|
"learning_rate": 9.555175363558598e-05, |
|
"loss": 2.4285, |
|
"theoretical_loss": 3.3201502761382775, |
|
"tokens_seen": 3007184896 |
|
}, |
|
{ |
|
"epoch": 0.05, |
|
"learning_rate": 9.550898203592816e-05, |
|
"loss": 2.2928, |
|
"theoretical_loss": 3.320139155349866, |
|
"tokens_seen": 3007315968 |
|
}, |
|
{ |
|
"epoch": 0.06, |
|
"learning_rate": 9.546621043627032e-05, |
|
"loss": 2.4617, |
|
"theoretical_loss": 3.3201280351818436, |
|
"tokens_seen": 3007447040 |
|
}, |
|
{ |
|
"epoch": 0.06, |
|
"learning_rate": 9.542343883661249e-05, |
|
"loss": 2.4107, |
|
"theoretical_loss": 3.320116915634149, |
|
"tokens_seen": 3007578112 |
|
}, |
|
{ |
|
"epoch": 0.06, |
|
"learning_rate": 9.538066723695466e-05, |
|
"loss": 2.5715, |
|
"theoretical_loss": 3.3201057967067205, |
|
"tokens_seen": 3007709184 |
|
}, |
|
{ |
|
"epoch": 0.06, |
|
"learning_rate": 9.533789563729684e-05, |
|
"loss": 2.4217, |
|
"theoretical_loss": 3.3200946783994962, |
|
"tokens_seen": 3007840256 |
|
}, |
|
{ |
|
"epoch": 0.06, |
|
"learning_rate": 9.529512403763902e-05, |
|
"loss": 2.4315, |
|
"theoretical_loss": 3.3200835607124146, |
|
"tokens_seen": 3007971328 |
|
}, |
|
{ |
|
"epoch": 0.06, |
|
"learning_rate": 9.525235243798119e-05, |
|
"loss": 2.4307, |
|
"theoretical_loss": 3.3200724436454143, |
|
"tokens_seen": 3008102400 |
|
}, |
|
{ |
|
"epoch": 0.06, |
|
"learning_rate": 9.520958083832335e-05, |
|
"loss": 2.5032, |
|
"theoretical_loss": 3.3200613271984336, |
|
"tokens_seen": 3008233472 |
|
}, |
|
{ |
|
"epoch": 0.06, |
|
"learning_rate": 9.516680923866553e-05, |
|
"loss": 2.4379, |
|
"theoretical_loss": 3.3200502113714108, |
|
"tokens_seen": 3008364544 |
|
}, |
|
{ |
|
"epoch": 0.06, |
|
"objective/train/docs_used": 1649940, |
|
"objective/train/instantaneous_batch_size": 16, |
|
"objective/train/instantaneous_microbatch_size": 16384, |
|
"objective/train/original_loss": 2.399945020675659, |
|
"objective/train/theoretical_loss": 3.3200446536903643, |
|
"objective/train/tokens_used": 38481376, |
|
"theoretical_loss": 3.3200446536903643, |
|
"tokens_seen": 3008430080 |
|
}, |
|
{ |
|
"epoch": 0.06, |
|
"learning_rate": 9.512403763900771e-05, |
|
"loss": 2.5107, |
|
"theoretical_loss": 3.3200390961642845, |
|
"tokens_seen": 3008495616 |
|
}, |
|
{ |
|
"epoch": 0.06, |
|
"learning_rate": 9.508126603934989e-05, |
|
"loss": 2.4659, |
|
"theoretical_loss": 3.3200279815769926, |
|
"tokens_seen": 3008626688 |
|
}, |
|
{ |
|
"epoch": 0.06, |
|
"learning_rate": 9.503849443969205e-05, |
|
"loss": 2.4327, |
|
"theoretical_loss": 3.3200168676094743, |
|
"tokens_seen": 3008757760 |
|
}, |
|
{ |
|
"epoch": 0.06, |
|
"learning_rate": 9.499572284003421e-05, |
|
"loss": 2.2681, |
|
"theoretical_loss": 3.320005754261668, |
|
"tokens_seen": 3008888832 |
|
}, |
|
{ |
|
"epoch": 0.06, |
|
"learning_rate": 9.49529512403764e-05, |
|
"loss": 2.3802, |
|
"theoretical_loss": 3.319994641533511, |
|
"tokens_seen": 3009019904 |
|
}, |
|
{ |
|
"epoch": 0.06, |
|
"learning_rate": 9.491017964071857e-05, |
|
"loss": 2.5461, |
|
"theoretical_loss": 3.319983529424943, |
|
"tokens_seen": 3009150976 |
|
}, |
|
{ |
|
"epoch": 0.06, |
|
"learning_rate": 9.486740804106075e-05, |
|
"loss": 2.4021, |
|
"theoretical_loss": 3.3199724179359027, |
|
"tokens_seen": 3009282048 |
|
}, |
|
{ |
|
"epoch": 0.06, |
|
"learning_rate": 9.482463644140291e-05, |
|
"loss": 2.4038, |
|
"theoretical_loss": 3.319961307066327, |
|
"tokens_seen": 3009413120 |
|
}, |
|
{ |
|
"epoch": 0.06, |
|
"learning_rate": 9.478186484174508e-05, |
|
"loss": 2.4687, |
|
"theoretical_loss": 3.3199501968161558, |
|
"tokens_seen": 3009544192 |
|
}, |
|
{ |
|
"epoch": 0.06, |
|
"learning_rate": 9.473909324208726e-05, |
|
"loss": 2.4401, |
|
"theoretical_loss": 3.319939087185327, |
|
"tokens_seen": 3009675264 |
|
}, |
|
{ |
|
"epoch": 0.06, |
|
"learning_rate": 9.469632164242944e-05, |
|
"loss": 2.4778, |
|
"theoretical_loss": 3.3199279781737796, |
|
"tokens_seen": 3009806336 |
|
}, |
|
{ |
|
"epoch": 0.06, |
|
"learning_rate": 9.46535500427716e-05, |
|
"loss": 2.4166, |
|
"theoretical_loss": 3.3199168697814514, |
|
"tokens_seen": 3009937408 |
|
}, |
|
{ |
|
"epoch": 0.06, |
|
"objective/train/docs_used": 1651249, |
|
"objective/train/instantaneous_batch_size": 16, |
|
"objective/train/instantaneous_microbatch_size": 16384, |
|
"objective/train/original_loss": 2.412529230117798, |
|
"objective/train/theoretical_loss": 3.3199057620082812, |
|
"objective/train/tokens_used": 40119776, |
|
"theoretical_loss": 3.3199057620082812, |
|
"tokens_seen": 3010068480 |
|
}, |
|
{ |
|
"epoch": 0.06, |
|
"learning_rate": 9.461077844311378e-05, |
|
"loss": 2.3839, |
|
"theoretical_loss": 3.3199057620082812, |
|
"tokens_seen": 3010068480 |
|
}, |
|
{ |
|
"epoch": 0.06, |
|
"learning_rate": 9.456800684345594e-05, |
|
"loss": 2.5879, |
|
"theoretical_loss": 3.319894654854208, |
|
"tokens_seen": 3010199552 |
|
}, |
|
{ |
|
"epoch": 0.06, |
|
"learning_rate": 9.452523524379812e-05, |
|
"loss": 2.444, |
|
"theoretical_loss": 3.3198835483191695, |
|
"tokens_seen": 3010330624 |
|
}, |
|
{ |
|
"epoch": 0.06, |
|
"learning_rate": 9.44824636441403e-05, |
|
"loss": 2.2849, |
|
"theoretical_loss": 3.319872442403105, |
|
"tokens_seen": 3010461696 |
|
}, |
|
{ |
|
"epoch": 0.07, |
|
"learning_rate": 9.443969204448247e-05, |
|
"loss": 2.568, |
|
"theoretical_loss": 3.3198613371059524, |
|
"tokens_seen": 3010592768 |
|
}, |
|
{ |
|
"epoch": 0.07, |
|
"learning_rate": 9.439692044482464e-05, |
|
"loss": 2.4131, |
|
"theoretical_loss": 3.319850232427651, |
|
"tokens_seen": 3010723840 |
|
}, |
|
{ |
|
"epoch": 0.07, |
|
"learning_rate": 9.435414884516681e-05, |
|
"loss": 2.4222, |
|
"theoretical_loss": 3.3198391283681383, |
|
"tokens_seen": 3010854912 |
|
}, |
|
{ |
|
"epoch": 0.07, |
|
"learning_rate": 9.431137724550899e-05, |
|
"loss": 2.3993, |
|
"theoretical_loss": 3.3198280249273546, |
|
"tokens_seen": 3010985984 |
|
}, |
|
{ |
|
"epoch": 0.07, |
|
"learning_rate": 9.426860564585116e-05, |
|
"loss": 2.5294, |
|
"theoretical_loss": 3.319816922105237, |
|
"tokens_seen": 3011117056 |
|
}, |
|
{ |
|
"epoch": 0.07, |
|
"learning_rate": 9.422583404619333e-05, |
|
"loss": 2.4529, |
|
"theoretical_loss": 3.319805819901724, |
|
"tokens_seen": 3011248128 |
|
}, |
|
{ |
|
"epoch": 0.07, |
|
"learning_rate": 9.418306244653551e-05, |
|
"loss": 2.4924, |
|
"theoretical_loss": 3.3197947183167553, |
|
"tokens_seen": 3011379200 |
|
}, |
|
{ |
|
"epoch": 0.07, |
|
"learning_rate": 9.414029084687767e-05, |
|
"loss": 2.6054, |
|
"theoretical_loss": 3.319783617350269, |
|
"tokens_seen": 3011510272 |
|
}, |
|
{ |
|
"epoch": 0.07, |
|
"learning_rate": 9.409751924721985e-05, |
|
"loss": 2.388, |
|
"theoretical_loss": 3.319772517002204, |
|
"tokens_seen": 3011641344 |
|
}, |
|
{ |
|
"epoch": 0.07, |
|
"objective/train/docs_used": 1651905, |
|
"objective/train/instantaneous_batch_size": 16, |
|
"objective/train/instantaneous_microbatch_size": 16384, |
|
"objective/train/original_loss": 2.8164331912994385, |
|
"objective/train/theoretical_loss": 3.31976696706006, |
|
"objective/train/tokens_used": 41758176, |
|
"theoretical_loss": 3.31976696706006, |
|
"tokens_seen": 3011706880 |
|
}, |
|
{ |
|
"epoch": 0.07, |
|
"learning_rate": 9.405474764756203e-05, |
|
"loss": 2.4623, |
|
"theoretical_loss": 3.319761417272498, |
|
"tokens_seen": 3011772416 |
|
}, |
|
{ |
|
"epoch": 0.07, |
|
"learning_rate": 9.40119760479042e-05, |
|
"loss": 2.3955, |
|
"theoretical_loss": 3.319750318161091, |
|
"tokens_seen": 3011903488 |
|
}, |
|
{ |
|
"epoch": 0.07, |
|
"learning_rate": 9.396920444824637e-05, |
|
"loss": 2.2424, |
|
"theoretical_loss": 3.3197392196679205, |
|
"tokens_seen": 3012034560 |
|
}, |
|
{ |
|
"epoch": 0.07, |
|
"learning_rate": 9.392643284858854e-05, |
|
"loss": 2.3568, |
|
"theoretical_loss": 3.3197281217929255, |
|
"tokens_seen": 3012165632 |
|
}, |
|
{ |
|
"epoch": 0.07, |
|
"learning_rate": 9.388366124893072e-05, |
|
"loss": 2.3788, |
|
"theoretical_loss": 3.319717024536045, |
|
"tokens_seen": 3012296704 |
|
}, |
|
{ |
|
"epoch": 0.07, |
|
"learning_rate": 9.38408896492729e-05, |
|
"loss": 2.3081, |
|
"theoretical_loss": 3.3197059278972176, |
|
"tokens_seen": 3012427776 |
|
}, |
|
{ |
|
"epoch": 0.07, |
|
"learning_rate": 9.379811804961506e-05, |
|
"loss": 2.4277, |
|
"theoretical_loss": 3.3196948318763817, |
|
"tokens_seen": 3012558848 |
|
}, |
|
{ |
|
"epoch": 0.07, |
|
"learning_rate": 9.375534644995724e-05, |
|
"loss": 2.4135, |
|
"theoretical_loss": 3.319683736473476, |
|
"tokens_seen": 3012689920 |
|
}, |
|
{ |
|
"epoch": 0.07, |
|
"learning_rate": 9.37125748502994e-05, |
|
"loss": 2.3474, |
|
"theoretical_loss": 3.3196726416884395, |
|
"tokens_seen": 3012820992 |
|
}, |
|
{ |
|
"epoch": 0.07, |
|
"learning_rate": 9.366980325064158e-05, |
|
"loss": 2.5489, |
|
"theoretical_loss": 3.3196615475212106, |
|
"tokens_seen": 3012952064 |
|
}, |
|
{ |
|
"epoch": 0.07, |
|
"learning_rate": 9.362703165098376e-05, |
|
"loss": 2.4998, |
|
"theoretical_loss": 3.3196504539717284, |
|
"tokens_seen": 3013083136 |
|
}, |
|
{ |
|
"epoch": 0.07, |
|
"learning_rate": 9.358426005132592e-05, |
|
"loss": 2.5438, |
|
"theoretical_loss": 3.3196393610399317, |
|
"tokens_seen": 3013214208 |
|
}, |
|
{ |
|
"epoch": 0.07, |
|
"objective/train/docs_used": 1652881, |
|
"objective/train/instantaneous_batch_size": 16, |
|
"objective/train/instantaneous_microbatch_size": 16384, |
|
"objective/train/original_loss": 2.3386740684509277, |
|
"objective/train/theoretical_loss": 3.3196282687257583, |
|
"objective/train/tokens_used": 43396576, |
|
"theoretical_loss": 3.3196282687257583, |
|
"tokens_seen": 3013345280 |
|
}, |
|
{ |
|
"epoch": 0.07, |
|
"learning_rate": 9.35414884516681e-05, |
|
"loss": 2.5351, |
|
"theoretical_loss": 3.3196282687257583, |
|
"tokens_seen": 3013345280 |
|
}, |
|
{ |
|
"epoch": 0.07, |
|
"learning_rate": 9.349871685201027e-05, |
|
"loss": 2.3127, |
|
"theoretical_loss": 3.3196171770291483, |
|
"tokens_seen": 3013476352 |
|
}, |
|
{ |
|
"epoch": 0.07, |
|
"learning_rate": 9.345594525235244e-05, |
|
"loss": 2.435, |
|
"theoretical_loss": 3.3196060859500394, |
|
"tokens_seen": 3013607424 |
|
}, |
|
{ |
|
"epoch": 0.08, |
|
"learning_rate": 9.341317365269462e-05, |
|
"loss": 2.3466, |
|
"theoretical_loss": 3.319594995488371, |
|
"tokens_seen": 3013738496 |
|
}, |
|
{ |
|
"epoch": 0.08, |
|
"learning_rate": 9.337040205303679e-05, |
|
"loss": 2.5683, |
|
"theoretical_loss": 3.3195839056440812, |
|
"tokens_seen": 3013869568 |
|
}, |
|
{ |
|
"epoch": 0.08, |
|
"learning_rate": 9.332763045337895e-05, |
|
"loss": 2.4838, |
|
"theoretical_loss": 3.3195728164171094, |
|
"tokens_seen": 3014000640 |
|
}, |
|
{ |
|
"epoch": 0.08, |
|
"learning_rate": 9.328485885372113e-05, |
|
"loss": 2.5372, |
|
"theoretical_loss": 3.319561727807394, |
|
"tokens_seen": 3014131712 |
|
}, |
|
{ |
|
"epoch": 0.08, |
|
"learning_rate": 9.324208725406331e-05, |
|
"loss": 2.4055, |
|
"theoretical_loss": 3.3195506398148744, |
|
"tokens_seen": 3014262784 |
|
}, |
|
{ |
|
"epoch": 0.08, |
|
"learning_rate": 9.319931565440549e-05, |
|
"loss": 2.5566, |
|
"theoretical_loss": 3.319539552439489, |
|
"tokens_seen": 3014393856 |
|
}, |
|
{ |
|
"epoch": 0.08, |
|
"learning_rate": 9.315654405474765e-05, |
|
"loss": 2.4051, |
|
"theoretical_loss": 3.3195284656811763, |
|
"tokens_seen": 3014524928 |
|
}, |
|
{ |
|
"epoch": 0.08, |
|
"learning_rate": 9.311377245508982e-05, |
|
"loss": 2.5032, |
|
"theoretical_loss": 3.319517379539876, |
|
"tokens_seen": 3014656000 |
|
}, |
|
{ |
|
"epoch": 0.08, |
|
"learning_rate": 9.3071000855432e-05, |
|
"loss": 2.427, |
|
"theoretical_loss": 3.3195062940155258, |
|
"tokens_seen": 3014787072 |
|
}, |
|
{ |
|
"epoch": 0.08, |
|
"learning_rate": 9.302822925577417e-05, |
|
"loss": 2.4468, |
|
"theoretical_loss": 3.3194952091080654, |
|
"tokens_seen": 3014918144 |
|
}, |
|
{ |
|
"epoch": 0.08, |
|
"objective/train/docs_used": 1653310, |
|
"objective/train/instantaneous_batch_size": 16, |
|
"objective/train/instantaneous_microbatch_size": 16384, |
|
"objective/train/original_loss": 2.2789435386657715, |
|
"objective/train/theoretical_loss": 3.3194896668856497, |
|
"objective/train/tokens_used": 45034976, |
|
"theoretical_loss": 3.3194896668856497, |
|
"tokens_seen": 3014983680 |
|
}, |
|
{ |
|
"epoch": 0.08, |
|
"learning_rate": 9.298545765611635e-05, |
|
"loss": 2.3447, |
|
"theoretical_loss": 3.3194841248174334, |
|
"tokens_seen": 3015049216 |
|
}, |
|
{ |
|
"epoch": 0.08, |
|
"learning_rate": 9.294268605645852e-05, |
|
"loss": 2.3587, |
|
"theoretical_loss": 3.3194730411435684, |
|
"tokens_seen": 3015180288 |
|
}, |
|
{ |
|
"epoch": 0.08, |
|
"learning_rate": 9.289991445680068e-05, |
|
"loss": 2.6264, |
|
"theoretical_loss": 3.3194619580864098, |
|
"tokens_seen": 3015311360 |
|
}, |
|
{ |
|
"epoch": 0.08, |
|
"learning_rate": 9.285714285714286e-05, |
|
"loss": 2.57, |
|
"theoretical_loss": 3.3194508756458965, |
|
"tokens_seen": 3015442432 |
|
}, |
|
{ |
|
"epoch": 0.08, |
|
"learning_rate": 9.281437125748504e-05, |
|
"loss": 2.3972, |
|
"theoretical_loss": 3.319439793821967, |
|
"tokens_seen": 3015573504 |
|
}, |
|
{ |
|
"epoch": 0.08, |
|
"learning_rate": 9.27715996578272e-05, |
|
"loss": 2.4522, |
|
"theoretical_loss": 3.3194287126145596, |
|
"tokens_seen": 3015704576 |
|
}, |
|
{ |
|
"epoch": 0.08, |
|
"learning_rate": 9.272882805816938e-05, |
|
"loss": 2.4546, |
|
"theoretical_loss": 3.3194176320236144, |
|
"tokens_seen": 3015835648 |
|
}, |
|
{ |
|
"epoch": 0.08, |
|
"learning_rate": 9.268605645851154e-05, |
|
"loss": 2.6088, |
|
"theoretical_loss": 3.31940655204907, |
|
"tokens_seen": 3015966720 |
|
}, |
|
{ |
|
"epoch": 0.08, |
|
"learning_rate": 9.264328485885372e-05, |
|
"loss": 2.4454, |
|
"theoretical_loss": 3.319395472690865, |
|
"tokens_seen": 3016097792 |
|
}, |
|
{ |
|
"epoch": 0.08, |
|
"learning_rate": 9.26005132591959e-05, |
|
"loss": 2.3876, |
|
"theoretical_loss": 3.3193843939489382, |
|
"tokens_seen": 3016228864 |
|
}, |
|
{ |
|
"epoch": 0.08, |
|
"learning_rate": 9.255774165953807e-05, |
|
"loss": 2.4971, |
|
"theoretical_loss": 3.319373315823229, |
|
"tokens_seen": 3016359936 |
|
}, |
|
{ |
|
"epoch": 0.08, |
|
"learning_rate": 9.251497005988024e-05, |
|
"loss": 2.5668, |
|
"theoretical_loss": 3.3193622383136763, |
|
"tokens_seen": 3016491008 |
|
}, |
|
{ |
|
"epoch": 0.08, |
|
"objective/train/docs_used": 1654644, |
|
"objective/train/instantaneous_batch_size": 16, |
|
"objective/train/instantaneous_microbatch_size": 16384, |
|
"objective/train/original_loss": 2.148563861846924, |
|
"objective/train/theoretical_loss": 3.3193511614202187, |
|
"objective/train/tokens_used": 46673376, |
|
"theoretical_loss": 3.3193511614202187, |
|
"tokens_seen": 3016622080 |
|
}, |
|
{ |
|
"epoch": 0.08, |
|
"learning_rate": 9.247219846022241e-05, |
|
"loss": 2.3435, |
|
"theoretical_loss": 3.3193511614202187, |
|
"tokens_seen": 3016622080 |
|
}, |
|
{ |
|
"epoch": 0.09, |
|
"learning_rate": 9.242942686056459e-05, |
|
"loss": 2.5671, |
|
"theoretical_loss": 3.319340085142796, |
|
"tokens_seen": 3016753152 |
|
}, |
|
{ |
|
"epoch": 0.09, |
|
"learning_rate": 9.238665526090677e-05, |
|
"loss": 2.455, |
|
"theoretical_loss": 3.319329009481346, |
|
"tokens_seen": 3016884224 |
|
}, |
|
{ |
|
"epoch": 0.09, |
|
"learning_rate": 9.234388366124893e-05, |
|
"loss": 2.5706, |
|
"theoretical_loss": 3.3193179344358086, |
|
"tokens_seen": 3017015296 |
|
}, |
|
{ |
|
"epoch": 0.09, |
|
"learning_rate": 9.230111206159111e-05, |
|
"loss": 2.3164, |
|
"theoretical_loss": 3.319306860006122, |
|
"tokens_seen": 3017146368 |
|
}, |
|
{ |
|
"epoch": 0.09, |
|
"learning_rate": 9.225834046193327e-05, |
|
"loss": 2.317, |
|
"theoretical_loss": 3.319295786192226, |
|
"tokens_seen": 3017277440 |
|
}, |
|
{ |
|
"epoch": 0.09, |
|
"learning_rate": 9.221556886227547e-05, |
|
"loss": 2.3955, |
|
"theoretical_loss": 3.319284712994059, |
|
"tokens_seen": 3017408512 |
|
}, |
|
{ |
|
"epoch": 0.09, |
|
"learning_rate": 9.217279726261763e-05, |
|
"loss": 2.4648, |
|
"theoretical_loss": 3.3192736404115606, |
|
"tokens_seen": 3017539584 |
|
}, |
|
{ |
|
"epoch": 0.09, |
|
"learning_rate": 9.21300256629598e-05, |
|
"loss": 2.3474, |
|
"theoretical_loss": 3.3192625684446693, |
|
"tokens_seen": 3017670656 |
|
}, |
|
{ |
|
"epoch": 0.09, |
|
"learning_rate": 9.208725406330197e-05, |
|
"loss": 2.2225, |
|
"theoretical_loss": 3.3192514970933242, |
|
"tokens_seen": 3017801728 |
|
}, |
|
{ |
|
"epoch": 0.09, |
|
"learning_rate": 9.204448246364414e-05, |
|
"loss": 2.4497, |
|
"theoretical_loss": 3.319240426357465, |
|
"tokens_seen": 3017932800 |
|
}, |
|
{ |
|
"epoch": 0.09, |
|
"learning_rate": 9.200171086398632e-05, |
|
"loss": 2.4471, |
|
"theoretical_loss": 3.31922935623703, |
|
"tokens_seen": 3018063872 |
|
}, |
|
{ |
|
"epoch": 0.09, |
|
"learning_rate": 9.19589392643285e-05, |
|
"loss": 2.3816, |
|
"theoretical_loss": 3.3192182867319584, |
|
"tokens_seen": 3018194944 |
|
}, |
|
{ |
|
"epoch": 0.09, |
|
"objective/train/docs_used": 1655335, |
|
"objective/train/instantaneous_batch_size": 16, |
|
"objective/train/instantaneous_microbatch_size": 16384, |
|
"objective/train/original_loss": 2.1034493446350098, |
|
"objective/train/theoretical_loss": 3.319212752210165, |
|
"objective/train/tokens_used": 48311776, |
|
"theoretical_loss": 3.319212752210165, |
|
"tokens_seen": 3018260480 |
|
}, |
|
{ |
|
"epoch": 0.09, |
|
"learning_rate": 9.191616766467066e-05, |
|
"loss": 2.3561, |
|
"theoretical_loss": 3.3192072178421896, |
|
"tokens_seen": 3018326016 |
|
}, |
|
{ |
|
"epoch": 0.09, |
|
"learning_rate": 9.187339606501284e-05, |
|
"loss": 2.4573, |
|
"theoretical_loss": 3.319196149567662, |
|
"tokens_seen": 3018457088 |
|
}, |
|
{ |
|
"epoch": 0.09, |
|
"learning_rate": 9.1830624465355e-05, |
|
"loss": 2.3837, |
|
"theoretical_loss": 3.3191850819083157, |
|
"tokens_seen": 3018588160 |
|
}, |
|
{ |
|
"epoch": 0.09, |
|
"learning_rate": 9.178785286569718e-05, |
|
"loss": 2.4923, |
|
"theoretical_loss": 3.319174014864089, |
|
"tokens_seen": 3018719232 |
|
}, |
|
{ |
|
"epoch": 0.09, |
|
"learning_rate": 9.174508126603936e-05, |
|
"loss": 2.6121, |
|
"theoretical_loss": 3.319162948434921, |
|
"tokens_seen": 3018850304 |
|
}, |
|
{ |
|
"epoch": 0.09, |
|
"learning_rate": 9.170230966638152e-05, |
|
"loss": 2.5103, |
|
"theoretical_loss": 3.319151882620752, |
|
"tokens_seen": 3018981376 |
|
}, |
|
{ |
|
"epoch": 0.09, |
|
"learning_rate": 9.16595380667237e-05, |
|
"loss": 2.3644, |
|
"theoretical_loss": 3.3191408174215193, |
|
"tokens_seen": 3019112448 |
|
}, |
|
{ |
|
"epoch": 0.09, |
|
"learning_rate": 9.161676646706587e-05, |
|
"loss": 2.5582, |
|
"theoretical_loss": 3.3191297528371635, |
|
"tokens_seen": 3019243520 |
|
}, |
|
{ |
|
"epoch": 0.09, |
|
"learning_rate": 9.157399486740804e-05, |
|
"loss": 2.3977, |
|
"theoretical_loss": 3.319118688867623, |
|
"tokens_seen": 3019374592 |
|
}, |
|
{ |
|
"epoch": 0.09, |
|
"learning_rate": 9.153122326775022e-05, |
|
"loss": 2.3253, |
|
"theoretical_loss": 3.319107625512837, |
|
"tokens_seen": 3019505664 |
|
}, |
|
{ |
|
"epoch": 0.09, |
|
"learning_rate": 9.148845166809239e-05, |
|
"loss": 2.5582, |
|
"theoretical_loss": 3.3190965627727445, |
|
"tokens_seen": 3019636736 |
|
}, |
|
{ |
|
"epoch": 0.09, |
|
"learning_rate": 9.144568006843457e-05, |
|
"loss": 2.402, |
|
"theoretical_loss": 3.3190855006472857, |
|
"tokens_seen": 3019767808 |
|
}, |
|
{ |
|
"epoch": 0.09, |
|
"objective/train/docs_used": 1656670, |
|
"objective/train/instantaneous_batch_size": 16, |
|
"objective/train/instantaneous_microbatch_size": 16384, |
|
"objective/train/original_loss": 2.027528762817383, |
|
"objective/train/theoretical_loss": 3.3190744391363984, |
|
"objective/train/tokens_used": 49950176, |
|
"theoretical_loss": 3.3190744391363984, |
|
"tokens_seen": 3019898880 |
|
}, |
|
{ |
|
"epoch": 0.1, |
|
"learning_rate": 9.140290846877674e-05, |
|
"loss": 2.4387, |
|
"theoretical_loss": 3.3190744391363984, |
|
"tokens_seen": 3019898880 |
|
}, |
|
{ |
|
"epoch": 0.1, |
|
"learning_rate": 9.136013686911891e-05, |
|
"loss": 2.5059, |
|
"theoretical_loss": 3.3190633782400223, |
|
"tokens_seen": 3020029952 |
|
}, |
|
{ |
|
"epoch": 0.1, |
|
"learning_rate": 9.131736526946109e-05, |
|
"loss": 2.3856, |
|
"theoretical_loss": 3.3190523179580973, |
|
"tokens_seen": 3020161024 |
|
}, |
|
{ |
|
"epoch": 0.1, |
|
"learning_rate": 9.127459366980325e-05, |
|
"loss": 2.5568, |
|
"theoretical_loss": 3.3190412582905617, |
|
"tokens_seen": 3020292096 |
|
}, |
|
{ |
|
"epoch": 0.1, |
|
"learning_rate": 9.123182207014542e-05, |
|
"loss": 2.5093, |
|
"theoretical_loss": 3.319030199237355, |
|
"tokens_seen": 3020423168 |
|
}, |
|
{ |
|
"epoch": 0.1, |
|
"learning_rate": 9.118905047048761e-05, |
|
"loss": 2.3476, |
|
"theoretical_loss": 3.3190191407984164, |
|
"tokens_seen": 3020554240 |
|
}, |
|
{ |
|
"epoch": 0.1, |
|
"learning_rate": 9.114627887082977e-05, |
|
"loss": 2.4233, |
|
"theoretical_loss": 3.3190080829736854, |
|
"tokens_seen": 3020685312 |
|
}, |
|
{ |
|
"epoch": 0.1, |
|
"learning_rate": 9.110350727117195e-05, |
|
"loss": 2.4302, |
|
"theoretical_loss": 3.318997025763101, |
|
"tokens_seen": 3020816384 |
|
}, |
|
{ |
|
"epoch": 0.1, |
|
"learning_rate": 9.106073567151412e-05, |
|
"loss": 2.5779, |
|
"theoretical_loss": 3.318985969166602, |
|
"tokens_seen": 3020947456 |
|
}, |
|
{ |
|
"epoch": 0.1, |
|
"learning_rate": 9.101796407185628e-05, |
|
"loss": 2.4611, |
|
"theoretical_loss": 3.3189749131841286, |
|
"tokens_seen": 3021078528 |
|
}, |
|
{ |
|
"epoch": 0.1, |
|
"learning_rate": 9.097519247219847e-05, |
|
"loss": 2.3387, |
|
"theoretical_loss": 3.3189638578156195, |
|
"tokens_seen": 3021209600 |
|
}, |
|
{ |
|
"epoch": 0.1, |
|
"learning_rate": 9.093242087254064e-05, |
|
"loss": 2.6853, |
|
"theoretical_loss": 3.3189528030610136, |
|
"tokens_seen": 3021340672 |
|
}, |
|
{ |
|
"epoch": 0.1, |
|
"learning_rate": 9.088964927288282e-05, |
|
"loss": 2.4887, |
|
"theoretical_loss": 3.318941748920251, |
|
"tokens_seen": 3021471744 |
|
}, |
|
{ |
|
"epoch": 0.1, |
|
"objective/train/docs_used": 1657192, |
|
"objective/train/instantaneous_batch_size": 16, |
|
"objective/train/instantaneous_microbatch_size": 16384, |
|
"objective/train/original_loss": 2.7123944759368896, |
|
"objective/train/theoretical_loss": 3.318936222080042, |
|
"objective/train/tokens_used": 51588576, |
|
"theoretical_loss": 3.318936222080042, |
|
"tokens_seen": 3021537280 |
|
}, |
|
{ |
|
"epoch": 0.1, |
|
"learning_rate": 9.084687767322498e-05, |
|
"loss": 2.4619, |
|
"theoretical_loss": 3.318930695393271, |
|
"tokens_seen": 3021602816 |
|
}, |
|
{ |
|
"epoch": 0.1, |
|
"learning_rate": 9.080410607356715e-05, |
|
"loss": 2.4821, |
|
"theoretical_loss": 3.3189196424800116, |
|
"tokens_seen": 3021733888 |
|
}, |
|
{ |
|
"epoch": 0.1, |
|
"learning_rate": 9.076133447390934e-05, |
|
"loss": 2.4471, |
|
"theoretical_loss": 3.3189085901804134, |
|
"tokens_seen": 3021864960 |
|
}, |
|
{ |
|
"epoch": 0.1, |
|
"learning_rate": 9.07185628742515e-05, |
|
"loss": 2.3988, |
|
"theoretical_loss": 3.3188975384944155, |
|
"tokens_seen": 3021996032 |
|
}, |
|
{ |
|
"epoch": 0.1, |
|
"learning_rate": 9.067579127459367e-05, |
|
"loss": 2.469, |
|
"theoretical_loss": 3.318886487421957, |
|
"tokens_seen": 3022127104 |
|
}, |
|
{ |
|
"epoch": 0.1, |
|
"learning_rate": 9.063301967493585e-05, |
|
"loss": 2.5398, |
|
"theoretical_loss": 3.318875436962977, |
|
"tokens_seen": 3022258176 |
|
}, |
|
{ |
|
"epoch": 0.1, |
|
"learning_rate": 9.059024807527801e-05, |
|
"loss": 2.3125, |
|
"theoretical_loss": 3.3188643871174155, |
|
"tokens_seen": 3022389248 |
|
}, |
|
{ |
|
"epoch": 0.1, |
|
"learning_rate": 9.05474764756202e-05, |
|
"loss": 2.3502, |
|
"theoretical_loss": 3.318853337885211, |
|
"tokens_seen": 3022520320 |
|
}, |
|
{ |
|
"epoch": 0.1, |
|
"learning_rate": 9.050470487596237e-05, |
|
"loss": 2.4073, |
|
"theoretical_loss": 3.318842289266304, |
|
"tokens_seen": 3022651392 |
|
}, |
|
{ |
|
"epoch": 0.1, |
|
"learning_rate": 9.046193327630453e-05, |
|
"loss": 2.3899, |
|
"theoretical_loss": 3.3188312412606327, |
|
"tokens_seen": 3022782464 |
|
}, |
|
{ |
|
"epoch": 0.1, |
|
"learning_rate": 9.041916167664671e-05, |
|
"loss": 2.3516, |
|
"theoretical_loss": 3.3188201938681368, |
|
"tokens_seen": 3022913536 |
|
}, |
|
{ |
|
"epoch": 0.11, |
|
"learning_rate": 9.037639007698889e-05, |
|
"loss": 2.3968, |
|
"theoretical_loss": 3.318809147088756, |
|
"tokens_seen": 3023044608 |
|
}, |
|
{ |
|
"epoch": 0.11, |
|
"objective/train/docs_used": 1658380, |
|
"objective/train/instantaneous_batch_size": 16, |
|
"objective/train/instantaneous_microbatch_size": 16384, |
|
"objective/train/original_loss": 2.6505117416381836, |
|
"objective/train/theoretical_loss": 3.3187981009224297, |
|
"objective/train/tokens_used": 53226976, |
|
"theoretical_loss": 3.3187981009224297, |
|
"tokens_seen": 3023175680 |
|
}, |
|
{ |
|
"epoch": 0.11, |
|
"learning_rate": 9.033361847733107e-05, |
|
"loss": 2.4709, |
|
"theoretical_loss": 3.3187981009224297, |
|
"tokens_seen": 3023175680 |
|
}, |
|
{ |
|
"epoch": 0.11, |
|
"learning_rate": 9.029084687767323e-05, |
|
"loss": 2.3951, |
|
"theoretical_loss": 3.3187870553690972, |
|
"tokens_seen": 3023306752 |
|
} |
|
], |
|
"max_steps": 2362, |
|
"num_train_epochs": 9223372036854775807, |
|
"total_flos": 1.6789580808192e+16, |
|
"trial_name": null, |
|
"trial_params": null |
|
} |
|
|