{ "best_metric": -0.9551481008529663, "best_model_checkpoint": "./qwen2.5-0.5b/qwen2.5-0.5b-expo-DPO-ES-TRY/checkpoint-36", "epoch": 0.8162494095418045, "eval_steps": 36, "global_step": 144, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "dpo_loss": 0.6931471824645996, "dpo_wo_beta": -0.6931471824645996, "epoch": 0.005668398677373642, "grad_norm": 13.433596947813236, "learning_rate": 1.3888888888888887e-08, "logits": -1.3874311447143555, "logps": -88.43561553955078, "loss": 0.6931, "objective": 0.6931471824645996, "ranking_idealized": 0.625, "ranking_idealized_expo": 0.5208333134651184, "ranking_simple": 0.5208333134651184, "regularize": 0.6931471824645996, "step": 1 }, { "dpo_loss": 0.6931712031364441, "dpo_wo_beta": -0.6935088634490967, "epoch": 0.02834199338686821, "grad_norm": 13.470599581187907, "learning_rate": 6.944444444444444e-08, "logits": -1.408589482307434, "logps": -84.63459777832031, "loss": 0.6931, "objective": 0.6931712031364441, "ranking_idealized": 0.6197916865348816, "ranking_idealized_expo": 0.546875, "ranking_simple": 0.546875, "regularize": 0.6931712031364441, "step": 5 }, { "dpo_loss": 0.6912738680839539, "dpo_wo_beta": -0.6787264943122864, "epoch": 0.05668398677373642, "grad_norm": 12.891994338762009, "learning_rate": 1.3888888888888888e-07, "logits": -1.4766913652420044, "logps": -83.30558776855469, "loss": 0.6922, "objective": 0.6912738680839539, "ranking_idealized": 0.6000000238418579, "ranking_idealized_expo": 0.5083333253860474, "ranking_simple": 0.5083333253860474, "regularize": 0.6912738680839539, "step": 10 }, { "dpo_loss": 0.6917670369148254, "dpo_wo_beta": -0.6925874352455139, "epoch": 0.08502598016060463, "grad_norm": 12.616091674274701, "learning_rate": 2.0833333333333333e-07, "logits": -1.4627829790115356, "logps": -83.51612854003906, "loss": 0.691, "objective": 0.6917670369148254, "ranking_idealized": 0.5916666388511658, "ranking_idealized_expo": 0.5041666626930237, "ranking_simple": 0.512499988079071, "regularize": 0.6917670369148254, "step": 15 }, { "dpo_loss": 0.6909351348876953, "dpo_wo_beta": -0.7156455516815186, "epoch": 0.11336797354747284, "grad_norm": 13.131384103632906, "learning_rate": 2.7777777777777776e-07, "logits": -1.4652808904647827, "logps": -83.81787109375, "loss": 0.6882, "objective": 0.6909351348876953, "ranking_idealized": 0.625, "ranking_idealized_expo": 0.5375000238418579, "ranking_simple": 0.5375000238418579, "regularize": 0.6909351348876953, "step": 20 }, { "dpo_loss": 0.6833630800247192, "dpo_wo_beta": -0.6689243316650391, "epoch": 0.14170996693434104, "grad_norm": 12.965314552644642, "learning_rate": 3.472222222222222e-07, "logits": -1.5261986255645752, "logps": -83.36495971679688, "loss": 0.6842, "objective": 0.6833630800247192, "ranking_idealized": 0.5958333611488342, "ranking_idealized_expo": 0.4958333373069763, "ranking_simple": 0.5, "regularize": 0.6833630800247192, "step": 25 }, { "dpo_loss": 0.6835373044013977, "dpo_wo_beta": -0.7437445521354675, "epoch": 0.17005196032120926, "grad_norm": 13.693441774251875, "learning_rate": 4.1666666666666667e-07, "logits": -1.512542724609375, "logps": -83.833984375, "loss": 0.6812, "objective": 0.6835373044013977, "ranking_idealized": 0.6041666865348816, "ranking_idealized_expo": 0.5291666388511658, "ranking_simple": 0.5333333611488342, "regularize": 0.6835373044013977, "step": 30 }, { "dpo_loss": 0.6732779145240784, "dpo_wo_beta": -0.7419623136520386, "epoch": 0.19839395370807747, "grad_norm": 13.409062026931796, "learning_rate": 4.861111111111111e-07, "logits": -1.4930462837219238, "logps": -82.56928253173828, "loss": 0.672, "objective": 0.6732779145240784, "ranking_idealized": 0.612500011920929, "ranking_idealized_expo": 0.5041666626930237, "ranking_simple": 0.512499988079071, "regularize": 0.6732779145240784, "step": 35 }, { "epoch": 0.20406235238545112, "eval_dpo_loss": 0.6925506591796875, "eval_dpo_wo_beta": -0.9551481008529663, "eval_logits": -1.5338705778121948, "eval_logps": -91.41157531738281, "eval_loss": 0.6866393685340881, "eval_objective": 0.6925506591796875, "eval_ranking_idealized": 0.5888429880142212, "eval_ranking_idealized_expo": 0.5092975497245789, "eval_ranking_simple": 0.5051652789115906, "eval_regularize": 0.6925506591796875, "eval_runtime": 211.8538, "eval_samples_per_second": 27.33, "eval_steps_per_second": 1.142, "step": 36 }, { "dpo_loss": 0.6632875800132751, "dpo_wo_beta": -0.7245739102363586, "epoch": 0.22673594709494568, "grad_norm": 15.824803800139307, "learning_rate": 4.998023493068254e-07, "logits": -1.4679725170135498, "logps": -83.45999908447266, "loss": 0.6728, "objective": 0.6632875800132751, "ranking_idealized": 0.6000000238418579, "ranking_idealized_expo": 0.5208333134651184, "ranking_simple": 0.5208333134651184, "regularize": 0.6632875800132751, "step": 40 }, { "dpo_loss": 0.6583938002586365, "dpo_wo_beta": -0.7636463642120361, "epoch": 0.25507794048181387, "grad_norm": 13.750505573599494, "learning_rate": 4.989999289644991e-07, "logits": -1.594675898551941, "logps": -84.99964141845703, "loss": 0.6655, "objective": 0.6583938002586365, "ranking_idealized": 0.6208333373069763, "ranking_idealized_expo": 0.5375000238418579, "ranking_simple": 0.5458333492279053, "regularize": 0.6583938002586365, "step": 45 }, { "dpo_loss": 0.6567264795303345, "dpo_wo_beta": -0.863508403301239, "epoch": 0.2834199338686821, "grad_norm": 13.413740152325811, "learning_rate": 4.975823666181255e-07, "logits": -1.5660439729690552, "logps": -82.82308959960938, "loss": 0.6644, "objective": 0.6567264795303345, "ranking_idealized": 0.5416666865348816, "ranking_idealized_expo": 0.42500001192092896, "ranking_simple": 0.4749999940395355, "regularize": 0.6567264795303345, "step": 50 }, { "dpo_loss": 0.6495455503463745, "dpo_wo_beta": -0.7752765417098999, "epoch": 0.3117619272555503, "grad_norm": 14.577952034074448, "learning_rate": 4.955531642853403e-07, "logits": -1.5825331211090088, "logps": -84.16141510009766, "loss": 0.653, "objective": 0.6495455503463745, "ranking_idealized": 0.625, "ranking_idealized_expo": 0.5416666865348816, "ranking_simple": 0.550000011920929, "regularize": 0.6495455503463745, "step": 55 }, { "dpo_loss": 0.6498515605926514, "dpo_wo_beta": -0.736571729183197, "epoch": 0.3401039206424185, "grad_norm": 13.788809377020446, "learning_rate": 4.929173350101024e-07, "logits": -1.5849006175994873, "logps": -83.91230773925781, "loss": 0.6514, "objective": 0.6498515605926514, "ranking_idealized": 0.6333333253860474, "ranking_idealized_expo": 0.5083333253860474, "ranking_simple": 0.550000011920929, "regularize": 0.6498515605926514, "step": 60 }, { "dpo_loss": 0.6450857520103455, "dpo_wo_beta": -0.8086111545562744, "epoch": 0.3684459140292867, "grad_norm": 13.799818527893999, "learning_rate": 4.896813904782162e-07, "logits": -1.5256472826004028, "logps": -84.62181091308594, "loss": 0.6495, "objective": 0.6450857520103455, "ranking_idealized": 0.5708333253860474, "ranking_idealized_expo": 0.5083333253860474, "ranking_simple": 0.5291666388511658, "regularize": 0.6450857520103455, "step": 65 }, { "dpo_loss": 0.644597053527832, "dpo_wo_beta": -0.891067385673523, "epoch": 0.39678790741615494, "grad_norm": 13.789576838493705, "learning_rate": 4.858533249305336e-07, "logits": -1.6211374998092651, "logps": -86.41259002685547, "loss": 0.6533, "objective": 0.644597053527832, "ranking_idealized": 0.6000000238418579, "ranking_idealized_expo": 0.5166666507720947, "ranking_simple": 0.5208333134651184, "regularize": 0.644597053527832, "step": 70 }, { "epoch": 0.40812470477090224, "eval_dpo_loss": 0.6884610056877136, "eval_dpo_wo_beta": -1.1473097801208496, "eval_logits": -1.6310690641403198, "eval_logps": -92.47576904296875, "eval_loss": 0.6768966913223267, "eval_objective": 0.6884610056877136, "eval_ranking_idealized": 0.5888429880142212, "eval_ranking_idealized_expo": 0.5092975497245789, "eval_ranking_simple": 0.5175619721412659, "eval_regularize": 0.6884610056877136, "eval_runtime": 210.9564, "eval_samples_per_second": 27.446, "eval_steps_per_second": 1.147, "step": 72 }, { "dpo_loss": 0.6458336114883423, "dpo_wo_beta": -0.847459614276886, "epoch": 0.42512990080302315, "grad_norm": 13.350435766130868, "learning_rate": 4.814425954135785e-07, "logits": -1.6625837087631226, "logps": -86.4749755859375, "loss": 0.636, "objective": 0.6458336114883423, "ranking_idealized": 0.625, "ranking_idealized_expo": 0.5458333492279053, "ranking_simple": 0.5833333134651184, "regularize": 0.6458336114883423, "step": 75 }, { "dpo_loss": 0.6464484930038452, "dpo_wo_beta": -0.9348036050796509, "epoch": 0.45347189418989137, "grad_norm": 13.762513071445667, "learning_rate": 4.764600984163808e-07, "logits": -1.7147225141525269, "logps": -86.26012420654297, "loss": 0.6326, "objective": 0.6464484930038452, "ranking_idealized": 0.5833333134651184, "ranking_idealized_expo": 0.5166666507720947, "ranking_simple": 0.5458333492279053, "regularize": 0.6464484930038452, "step": 80 }, { "dpo_loss": 0.6432295441627502, "dpo_wo_beta": -1.0359190702438354, "epoch": 0.4818138875767596, "grad_norm": 13.87750405485222, "learning_rate": 4.70918142951239e-07, "logits": -1.6870180368423462, "logps": -85.32923889160156, "loss": 0.6336, "objective": 0.6432295441627502, "ranking_idealized": 0.6000000238418579, "ranking_idealized_expo": 0.5249999761581421, "ranking_simple": 0.5458333492279053, "regularize": 0.6432295441627502, "step": 85 }, { "dpo_loss": 0.6311187744140625, "dpo_wo_beta": -0.9493275284767151, "epoch": 0.5101558809636277, "grad_norm": 14.792020412361724, "learning_rate": 4.6483042014491527e-07, "logits": -1.5992980003356934, "logps": -88.2070083618164, "loss": 0.6216, "objective": 0.6311187744140625, "ranking_idealized": 0.5458333492279053, "ranking_idealized_expo": 0.4749999940395355, "ranking_simple": 0.5041666626930237, "regularize": 0.6311187744140625, "step": 90 }, { "dpo_loss": 0.595143735408783, "dpo_wo_beta": -0.8239250183105469, "epoch": 0.538497874350496, "grad_norm": 16.242182565245834, "learning_rate": 4.5821196941538334e-07, "logits": -1.6257656812667847, "logps": -87.32308197021484, "loss": 0.6157, "objective": 0.595143735408783, "ranking_idealized": 0.6291666626930237, "ranking_idealized_expo": 0.5458333492279053, "ranking_simple": 0.6041666865348816, "regularize": 0.595143735408783, "step": 95 }, { "dpo_loss": 0.6018479466438293, "dpo_wo_beta": -1.0667775869369507, "epoch": 0.5668398677373642, "grad_norm": 16.965716669559736, "learning_rate": 4.510791413176912e-07, "logits": -1.6872950792312622, "logps": -88.73619842529297, "loss": 0.623, "objective": 0.6018479466438293, "ranking_idealized": 0.612500011920929, "ranking_idealized_expo": 0.5416666865348816, "ranking_simple": 0.5625, "regularize": 0.6018479466438293, "step": 100 }, { "dpo_loss": 0.5836849808692932, "dpo_wo_beta": -0.7914173007011414, "epoch": 0.5951818611242324, "grad_norm": 14.695711016472089, "learning_rate": 4.434495571507234e-07, "logits": -1.606242299079895, "logps": -87.9868392944336, "loss": 0.604, "objective": 0.5836849808692932, "ranking_idealized": 0.625, "ranking_idealized_expo": 0.5333333611488342, "ranking_simple": 0.5833333134651184, "regularize": 0.5836849808692932, "step": 105 }, { "epoch": 0.6121870571563534, "eval_dpo_loss": 0.6853068470954895, "eval_dpo_wo_beta": -1.3960636854171753, "eval_logits": -1.769094467163086, "eval_logps": -94.05838012695312, "eval_loss": 0.677278995513916, "eval_objective": 0.6853068470954895, "eval_ranking_idealized": 0.5888429880142212, "eval_ranking_idealized_expo": 0.5092975497245789, "eval_ranking_simple": 0.5185950398445129, "eval_regularize": 0.6853068470954895, "eval_runtime": 210.425, "eval_samples_per_second": 27.516, "eval_steps_per_second": 1.15, "step": 108 }, { "dpo_loss": 0.6007120013237, "dpo_wo_beta": -0.8422455191612244, "epoch": 0.6235238545111006, "grad_norm": 15.750957422421227, "learning_rate": 4.353420654246546e-07, "logits": -1.7943415641784668, "logps": -86.3438491821289, "loss": 0.6054, "objective": 0.6007120013237, "ranking_idealized": 0.5708333253860474, "ranking_idealized_expo": 0.4541666805744171, "ranking_simple": 0.5166666507720947, "regularize": 0.6007120013237, "step": 110 }, { "dpo_loss": 0.6109593510627747, "dpo_wo_beta": -1.0094329118728638, "epoch": 0.6518658478979689, "grad_norm": 16.233880140590145, "learning_rate": 4.2677669529663686e-07, "logits": -1.7633204460144043, "logps": -86.51942443847656, "loss": 0.6046, "objective": 0.6109593510627747, "ranking_idealized": 0.574999988079071, "ranking_idealized_expo": 0.4833333194255829, "ranking_simple": 0.5166666507720947, "regularize": 0.6109593510627747, "step": 115 }, { "dpo_loss": 0.5903018712997437, "dpo_wo_beta": -0.9045132994651794, "epoch": 0.680207841284837, "grad_norm": 17.133679937634657, "learning_rate": 4.177746070897592e-07, "logits": -1.7159924507141113, "logps": -87.81356811523438, "loss": 0.5999, "objective": 0.5903018712997437, "ranking_idealized": 0.5874999761581421, "ranking_idealized_expo": 0.5291666388511658, "ranking_simple": 0.5791666507720947, "regularize": 0.5903018712997437, "step": 120 }, { "dpo_loss": 0.6066262722015381, "dpo_wo_beta": -1.2728170156478882, "epoch": 0.7085498346717053, "grad_norm": 16.15790530794251, "learning_rate": 4.083580400175153e-07, "logits": -1.698543667793274, "logps": -88.70279693603516, "loss": 0.6139, "objective": 0.6066262722015381, "ranking_idealized": 0.625, "ranking_idealized_expo": 0.550000011920929, "ranking_simple": 0.5874999761581421, "regularize": 0.6066262722015381, "step": 125 }, { "dpo_loss": 0.6065606474876404, "dpo_wo_beta": -1.1573857069015503, "epoch": 0.7368918280585735, "grad_norm": 16.556049529132082, "learning_rate": 3.9855025724292763e-07, "logits": -1.8358159065246582, "logps": -87.11023712158203, "loss": 0.6035, "objective": 0.6065606474876404, "ranking_idealized": 0.5416666865348816, "ranking_idealized_expo": 0.44999998807907104, "ranking_simple": 0.5166666507720947, "regularize": 0.6065606474876404, "step": 130 }, { "dpo_loss": 0.6158966422080994, "dpo_wo_beta": -1.1439249515533447, "epoch": 0.7652338214454416, "grad_norm": 14.197353075079507, "learning_rate": 3.883754884080539e-07, "logits": -1.7928814888000488, "logps": -89.14883422851562, "loss": 0.594, "objective": 0.6158966422080994, "ranking_idealized": 0.5874999761581421, "ranking_idealized_expo": 0.5249999761581421, "ranking_simple": 0.5458333492279053, "regularize": 0.6158966422080994, "step": 135 }, { "dpo_loss": 0.61494380235672, "dpo_wo_beta": -1.0414694547653198, "epoch": 0.7935758148323099, "grad_norm": 15.670852477907466, "learning_rate": 3.7785886977585555e-07, "logits": -1.711374282836914, "logps": -88.56974792480469, "loss": 0.5911, "objective": 0.61494380235672, "ranking_idealized": 0.5791666507720947, "ranking_idealized_expo": 0.5041666626930237, "ranking_simple": 0.5416666865348816, "regularize": 0.61494380235672, "step": 140 }, { "epoch": 0.8162494095418045, "eval_dpo_loss": 0.6885914206504822, "eval_dpo_wo_beta": -1.5153368711471558, "eval_logits": -1.816495418548584, "eval_logps": -95.75775146484375, "eval_loss": 0.6774819493293762, "eval_objective": 0.6885914206504822, "eval_ranking_idealized": 0.5888429880142212, "eval_ranking_idealized_expo": 0.5092975497245789, "eval_ranking_simple": 0.5185950398445129, "eval_regularize": 0.6885914206504822, "eval_runtime": 210.8957, "eval_samples_per_second": 27.454, "eval_steps_per_second": 1.147, "step": 144 }, { "epoch": 0.8162494095418045, "step": 144, "total_flos": 0.0, "train_loss": 0.6411769655015733, "train_runtime": 5149.1432, "train_samples_per_second": 19.732, "train_steps_per_second": 0.068 } ], "logging_steps": 5, "max_steps": 352, "num_input_tokens_seen": 0, "num_train_epochs": 2, "save_steps": 36, "stateful_callbacks": { "EarlyStoppingCallback": { "args": { "early_stopping_patience": 3, "early_stopping_threshold": 0.0 }, "attributes": { "early_stopping_patience_counter": 0 } }, "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": true, "should_training_stop": true }, "attributes": {} } }, "total_flos": 0.0, "train_batch_size": 4, "trial_name": null, "trial_params": null }