qwen2.5-0.5b-expo-DPO-ES-TRY0 / trainer_state.json
hZzy's picture
Model save
a1a29fe verified
{
"best_metric": -0.9551481008529663,
"best_model_checkpoint": "./qwen2.5-0.5b/qwen2.5-0.5b-expo-DPO-ES-TRY/checkpoint-36",
"epoch": 0.8162494095418045,
"eval_steps": 36,
"global_step": 144,
"is_hyper_param_search": false,
"is_local_process_zero": true,
"is_world_process_zero": true,
"log_history": [
{
"dpo_loss": 0.6931471824645996,
"dpo_wo_beta": -0.6931471824645996,
"epoch": 0.005668398677373642,
"grad_norm": 13.433596947813236,
"learning_rate": 1.3888888888888887e-08,
"logits": -1.3874311447143555,
"logps": -88.43561553955078,
"loss": 0.6931,
"objective": 0.6931471824645996,
"ranking_idealized": 0.625,
"ranking_idealized_expo": 0.5208333134651184,
"ranking_simple": 0.5208333134651184,
"regularize": 0.6931471824645996,
"step": 1
},
{
"dpo_loss": 0.6931712031364441,
"dpo_wo_beta": -0.6935088634490967,
"epoch": 0.02834199338686821,
"grad_norm": 13.470599581187907,
"learning_rate": 6.944444444444444e-08,
"logits": -1.408589482307434,
"logps": -84.63459777832031,
"loss": 0.6931,
"objective": 0.6931712031364441,
"ranking_idealized": 0.6197916865348816,
"ranking_idealized_expo": 0.546875,
"ranking_simple": 0.546875,
"regularize": 0.6931712031364441,
"step": 5
},
{
"dpo_loss": 0.6912738680839539,
"dpo_wo_beta": -0.6787264943122864,
"epoch": 0.05668398677373642,
"grad_norm": 12.891994338762009,
"learning_rate": 1.3888888888888888e-07,
"logits": -1.4766913652420044,
"logps": -83.30558776855469,
"loss": 0.6922,
"objective": 0.6912738680839539,
"ranking_idealized": 0.6000000238418579,
"ranking_idealized_expo": 0.5083333253860474,
"ranking_simple": 0.5083333253860474,
"regularize": 0.6912738680839539,
"step": 10
},
{
"dpo_loss": 0.6917670369148254,
"dpo_wo_beta": -0.6925874352455139,
"epoch": 0.08502598016060463,
"grad_norm": 12.616091674274701,
"learning_rate": 2.0833333333333333e-07,
"logits": -1.4627829790115356,
"logps": -83.51612854003906,
"loss": 0.691,
"objective": 0.6917670369148254,
"ranking_idealized": 0.5916666388511658,
"ranking_idealized_expo": 0.5041666626930237,
"ranking_simple": 0.512499988079071,
"regularize": 0.6917670369148254,
"step": 15
},
{
"dpo_loss": 0.6909351348876953,
"dpo_wo_beta": -0.7156455516815186,
"epoch": 0.11336797354747284,
"grad_norm": 13.131384103632906,
"learning_rate": 2.7777777777777776e-07,
"logits": -1.4652808904647827,
"logps": -83.81787109375,
"loss": 0.6882,
"objective": 0.6909351348876953,
"ranking_idealized": 0.625,
"ranking_idealized_expo": 0.5375000238418579,
"ranking_simple": 0.5375000238418579,
"regularize": 0.6909351348876953,
"step": 20
},
{
"dpo_loss": 0.6833630800247192,
"dpo_wo_beta": -0.6689243316650391,
"epoch": 0.14170996693434104,
"grad_norm": 12.965314552644642,
"learning_rate": 3.472222222222222e-07,
"logits": -1.5261986255645752,
"logps": -83.36495971679688,
"loss": 0.6842,
"objective": 0.6833630800247192,
"ranking_idealized": 0.5958333611488342,
"ranking_idealized_expo": 0.4958333373069763,
"ranking_simple": 0.5,
"regularize": 0.6833630800247192,
"step": 25
},
{
"dpo_loss": 0.6835373044013977,
"dpo_wo_beta": -0.7437445521354675,
"epoch": 0.17005196032120926,
"grad_norm": 13.693441774251875,
"learning_rate": 4.1666666666666667e-07,
"logits": -1.512542724609375,
"logps": -83.833984375,
"loss": 0.6812,
"objective": 0.6835373044013977,
"ranking_idealized": 0.6041666865348816,
"ranking_idealized_expo": 0.5291666388511658,
"ranking_simple": 0.5333333611488342,
"regularize": 0.6835373044013977,
"step": 30
},
{
"dpo_loss": 0.6732779145240784,
"dpo_wo_beta": -0.7419623136520386,
"epoch": 0.19839395370807747,
"grad_norm": 13.409062026931796,
"learning_rate": 4.861111111111111e-07,
"logits": -1.4930462837219238,
"logps": -82.56928253173828,
"loss": 0.672,
"objective": 0.6732779145240784,
"ranking_idealized": 0.612500011920929,
"ranking_idealized_expo": 0.5041666626930237,
"ranking_simple": 0.512499988079071,
"regularize": 0.6732779145240784,
"step": 35
},
{
"epoch": 0.20406235238545112,
"eval_dpo_loss": 0.6925506591796875,
"eval_dpo_wo_beta": -0.9551481008529663,
"eval_logits": -1.5338705778121948,
"eval_logps": -91.41157531738281,
"eval_loss": 0.6866393685340881,
"eval_objective": 0.6925506591796875,
"eval_ranking_idealized": 0.5888429880142212,
"eval_ranking_idealized_expo": 0.5092975497245789,
"eval_ranking_simple": 0.5051652789115906,
"eval_regularize": 0.6925506591796875,
"eval_runtime": 211.8538,
"eval_samples_per_second": 27.33,
"eval_steps_per_second": 1.142,
"step": 36
},
{
"dpo_loss": 0.6632875800132751,
"dpo_wo_beta": -0.7245739102363586,
"epoch": 0.22673594709494568,
"grad_norm": 15.824803800139307,
"learning_rate": 4.998023493068254e-07,
"logits": -1.4679725170135498,
"logps": -83.45999908447266,
"loss": 0.6728,
"objective": 0.6632875800132751,
"ranking_idealized": 0.6000000238418579,
"ranking_idealized_expo": 0.5208333134651184,
"ranking_simple": 0.5208333134651184,
"regularize": 0.6632875800132751,
"step": 40
},
{
"dpo_loss": 0.6583938002586365,
"dpo_wo_beta": -0.7636463642120361,
"epoch": 0.25507794048181387,
"grad_norm": 13.750505573599494,
"learning_rate": 4.989999289644991e-07,
"logits": -1.594675898551941,
"logps": -84.99964141845703,
"loss": 0.6655,
"objective": 0.6583938002586365,
"ranking_idealized": 0.6208333373069763,
"ranking_idealized_expo": 0.5375000238418579,
"ranking_simple": 0.5458333492279053,
"regularize": 0.6583938002586365,
"step": 45
},
{
"dpo_loss": 0.6567264795303345,
"dpo_wo_beta": -0.863508403301239,
"epoch": 0.2834199338686821,
"grad_norm": 13.413740152325811,
"learning_rate": 4.975823666181255e-07,
"logits": -1.5660439729690552,
"logps": -82.82308959960938,
"loss": 0.6644,
"objective": 0.6567264795303345,
"ranking_idealized": 0.5416666865348816,
"ranking_idealized_expo": 0.42500001192092896,
"ranking_simple": 0.4749999940395355,
"regularize": 0.6567264795303345,
"step": 50
},
{
"dpo_loss": 0.6495455503463745,
"dpo_wo_beta": -0.7752765417098999,
"epoch": 0.3117619272555503,
"grad_norm": 14.577952034074448,
"learning_rate": 4.955531642853403e-07,
"logits": -1.5825331211090088,
"logps": -84.16141510009766,
"loss": 0.653,
"objective": 0.6495455503463745,
"ranking_idealized": 0.625,
"ranking_idealized_expo": 0.5416666865348816,
"ranking_simple": 0.550000011920929,
"regularize": 0.6495455503463745,
"step": 55
},
{
"dpo_loss": 0.6498515605926514,
"dpo_wo_beta": -0.736571729183197,
"epoch": 0.3401039206424185,
"grad_norm": 13.788809377020446,
"learning_rate": 4.929173350101024e-07,
"logits": -1.5849006175994873,
"logps": -83.91230773925781,
"loss": 0.6514,
"objective": 0.6498515605926514,
"ranking_idealized": 0.6333333253860474,
"ranking_idealized_expo": 0.5083333253860474,
"ranking_simple": 0.550000011920929,
"regularize": 0.6498515605926514,
"step": 60
},
{
"dpo_loss": 0.6450857520103455,
"dpo_wo_beta": -0.8086111545562744,
"epoch": 0.3684459140292867,
"grad_norm": 13.799818527893999,
"learning_rate": 4.896813904782162e-07,
"logits": -1.5256472826004028,
"logps": -84.62181091308594,
"loss": 0.6495,
"objective": 0.6450857520103455,
"ranking_idealized": 0.5708333253860474,
"ranking_idealized_expo": 0.5083333253860474,
"ranking_simple": 0.5291666388511658,
"regularize": 0.6450857520103455,
"step": 65
},
{
"dpo_loss": 0.644597053527832,
"dpo_wo_beta": -0.891067385673523,
"epoch": 0.39678790741615494,
"grad_norm": 13.789576838493705,
"learning_rate": 4.858533249305336e-07,
"logits": -1.6211374998092651,
"logps": -86.41259002685547,
"loss": 0.6533,
"objective": 0.644597053527832,
"ranking_idealized": 0.6000000238418579,
"ranking_idealized_expo": 0.5166666507720947,
"ranking_simple": 0.5208333134651184,
"regularize": 0.644597053527832,
"step": 70
},
{
"epoch": 0.40812470477090224,
"eval_dpo_loss": 0.6884610056877136,
"eval_dpo_wo_beta": -1.1473097801208496,
"eval_logits": -1.6310690641403198,
"eval_logps": -92.47576904296875,
"eval_loss": 0.6768966913223267,
"eval_objective": 0.6884610056877136,
"eval_ranking_idealized": 0.5888429880142212,
"eval_ranking_idealized_expo": 0.5092975497245789,
"eval_ranking_simple": 0.5175619721412659,
"eval_regularize": 0.6884610056877136,
"eval_runtime": 210.9564,
"eval_samples_per_second": 27.446,
"eval_steps_per_second": 1.147,
"step": 72
},
{
"dpo_loss": 0.6458336114883423,
"dpo_wo_beta": -0.847459614276886,
"epoch": 0.42512990080302315,
"grad_norm": 13.350435766130868,
"learning_rate": 4.814425954135785e-07,
"logits": -1.6625837087631226,
"logps": -86.4749755859375,
"loss": 0.636,
"objective": 0.6458336114883423,
"ranking_idealized": 0.625,
"ranking_idealized_expo": 0.5458333492279053,
"ranking_simple": 0.5833333134651184,
"regularize": 0.6458336114883423,
"step": 75
},
{
"dpo_loss": 0.6464484930038452,
"dpo_wo_beta": -0.9348036050796509,
"epoch": 0.45347189418989137,
"grad_norm": 13.762513071445667,
"learning_rate": 4.764600984163808e-07,
"logits": -1.7147225141525269,
"logps": -86.26012420654297,
"loss": 0.6326,
"objective": 0.6464484930038452,
"ranking_idealized": 0.5833333134651184,
"ranking_idealized_expo": 0.5166666507720947,
"ranking_simple": 0.5458333492279053,
"regularize": 0.6464484930038452,
"step": 80
},
{
"dpo_loss": 0.6432295441627502,
"dpo_wo_beta": -1.0359190702438354,
"epoch": 0.4818138875767596,
"grad_norm": 13.87750405485222,
"learning_rate": 4.70918142951239e-07,
"logits": -1.6870180368423462,
"logps": -85.32923889160156,
"loss": 0.6336,
"objective": 0.6432295441627502,
"ranking_idealized": 0.6000000238418579,
"ranking_idealized_expo": 0.5249999761581421,
"ranking_simple": 0.5458333492279053,
"regularize": 0.6432295441627502,
"step": 85
},
{
"dpo_loss": 0.6311187744140625,
"dpo_wo_beta": -0.9493275284767151,
"epoch": 0.5101558809636277,
"grad_norm": 14.792020412361724,
"learning_rate": 4.6483042014491527e-07,
"logits": -1.5992980003356934,
"logps": -88.2070083618164,
"loss": 0.6216,
"objective": 0.6311187744140625,
"ranking_idealized": 0.5458333492279053,
"ranking_idealized_expo": 0.4749999940395355,
"ranking_simple": 0.5041666626930237,
"regularize": 0.6311187744140625,
"step": 90
},
{
"dpo_loss": 0.595143735408783,
"dpo_wo_beta": -0.8239250183105469,
"epoch": 0.538497874350496,
"grad_norm": 16.242182565245834,
"learning_rate": 4.5821196941538334e-07,
"logits": -1.6257656812667847,
"logps": -87.32308197021484,
"loss": 0.6157,
"objective": 0.595143735408783,
"ranking_idealized": 0.6291666626930237,
"ranking_idealized_expo": 0.5458333492279053,
"ranking_simple": 0.6041666865348816,
"regularize": 0.595143735408783,
"step": 95
},
{
"dpo_loss": 0.6018479466438293,
"dpo_wo_beta": -1.0667775869369507,
"epoch": 0.5668398677373642,
"grad_norm": 16.965716669559736,
"learning_rate": 4.510791413176912e-07,
"logits": -1.6872950792312622,
"logps": -88.73619842529297,
"loss": 0.623,
"objective": 0.6018479466438293,
"ranking_idealized": 0.612500011920929,
"ranking_idealized_expo": 0.5416666865348816,
"ranking_simple": 0.5625,
"regularize": 0.6018479466438293,
"step": 100
},
{
"dpo_loss": 0.5836849808692932,
"dpo_wo_beta": -0.7914173007011414,
"epoch": 0.5951818611242324,
"grad_norm": 14.695711016472089,
"learning_rate": 4.434495571507234e-07,
"logits": -1.606242299079895,
"logps": -87.9868392944336,
"loss": 0.604,
"objective": 0.5836849808692932,
"ranking_idealized": 0.625,
"ranking_idealized_expo": 0.5333333611488342,
"ranking_simple": 0.5833333134651184,
"regularize": 0.5836849808692932,
"step": 105
},
{
"epoch": 0.6121870571563534,
"eval_dpo_loss": 0.6853068470954895,
"eval_dpo_wo_beta": -1.3960636854171753,
"eval_logits": -1.769094467163086,
"eval_logps": -94.05838012695312,
"eval_loss": 0.677278995513916,
"eval_objective": 0.6853068470954895,
"eval_ranking_idealized": 0.5888429880142212,
"eval_ranking_idealized_expo": 0.5092975497245789,
"eval_ranking_simple": 0.5185950398445129,
"eval_regularize": 0.6853068470954895,
"eval_runtime": 210.425,
"eval_samples_per_second": 27.516,
"eval_steps_per_second": 1.15,
"step": 108
},
{
"dpo_loss": 0.6007120013237,
"dpo_wo_beta": -0.8422455191612244,
"epoch": 0.6235238545111006,
"grad_norm": 15.750957422421227,
"learning_rate": 4.353420654246546e-07,
"logits": -1.7943415641784668,
"logps": -86.3438491821289,
"loss": 0.6054,
"objective": 0.6007120013237,
"ranking_idealized": 0.5708333253860474,
"ranking_idealized_expo": 0.4541666805744171,
"ranking_simple": 0.5166666507720947,
"regularize": 0.6007120013237,
"step": 110
},
{
"dpo_loss": 0.6109593510627747,
"dpo_wo_beta": -1.0094329118728638,
"epoch": 0.6518658478979689,
"grad_norm": 16.233880140590145,
"learning_rate": 4.2677669529663686e-07,
"logits": -1.7633204460144043,
"logps": -86.51942443847656,
"loss": 0.6046,
"objective": 0.6109593510627747,
"ranking_idealized": 0.574999988079071,
"ranking_idealized_expo": 0.4833333194255829,
"ranking_simple": 0.5166666507720947,
"regularize": 0.6109593510627747,
"step": 115
},
{
"dpo_loss": 0.5903018712997437,
"dpo_wo_beta": -0.9045132994651794,
"epoch": 0.680207841284837,
"grad_norm": 17.133679937634657,
"learning_rate": 4.177746070897592e-07,
"logits": -1.7159924507141113,
"logps": -87.81356811523438,
"loss": 0.5999,
"objective": 0.5903018712997437,
"ranking_idealized": 0.5874999761581421,
"ranking_idealized_expo": 0.5291666388511658,
"ranking_simple": 0.5791666507720947,
"regularize": 0.5903018712997437,
"step": 120
},
{
"dpo_loss": 0.6066262722015381,
"dpo_wo_beta": -1.2728170156478882,
"epoch": 0.7085498346717053,
"grad_norm": 16.15790530794251,
"learning_rate": 4.083580400175153e-07,
"logits": -1.698543667793274,
"logps": -88.70279693603516,
"loss": 0.6139,
"objective": 0.6066262722015381,
"ranking_idealized": 0.625,
"ranking_idealized_expo": 0.550000011920929,
"ranking_simple": 0.5874999761581421,
"regularize": 0.6066262722015381,
"step": 125
},
{
"dpo_loss": 0.6065606474876404,
"dpo_wo_beta": -1.1573857069015503,
"epoch": 0.7368918280585735,
"grad_norm": 16.556049529132082,
"learning_rate": 3.9855025724292763e-07,
"logits": -1.8358159065246582,
"logps": -87.11023712158203,
"loss": 0.6035,
"objective": 0.6065606474876404,
"ranking_idealized": 0.5416666865348816,
"ranking_idealized_expo": 0.44999998807907104,
"ranking_simple": 0.5166666507720947,
"regularize": 0.6065606474876404,
"step": 130
},
{
"dpo_loss": 0.6158966422080994,
"dpo_wo_beta": -1.1439249515533447,
"epoch": 0.7652338214454416,
"grad_norm": 14.197353075079507,
"learning_rate": 3.883754884080539e-07,
"logits": -1.7928814888000488,
"logps": -89.14883422851562,
"loss": 0.594,
"objective": 0.6158966422080994,
"ranking_idealized": 0.5874999761581421,
"ranking_idealized_expo": 0.5249999761581421,
"ranking_simple": 0.5458333492279053,
"regularize": 0.6158966422080994,
"step": 135
},
{
"dpo_loss": 0.61494380235672,
"dpo_wo_beta": -1.0414694547653198,
"epoch": 0.7935758148323099,
"grad_norm": 15.670852477907466,
"learning_rate": 3.7785886977585555e-07,
"logits": -1.711374282836914,
"logps": -88.56974792480469,
"loss": 0.5911,
"objective": 0.61494380235672,
"ranking_idealized": 0.5791666507720947,
"ranking_idealized_expo": 0.5041666626930237,
"ranking_simple": 0.5416666865348816,
"regularize": 0.61494380235672,
"step": 140
},
{
"epoch": 0.8162494095418045,
"eval_dpo_loss": 0.6885914206504822,
"eval_dpo_wo_beta": -1.5153368711471558,
"eval_logits": -1.816495418548584,
"eval_logps": -95.75775146484375,
"eval_loss": 0.6774819493293762,
"eval_objective": 0.6885914206504822,
"eval_ranking_idealized": 0.5888429880142212,
"eval_ranking_idealized_expo": 0.5092975497245789,
"eval_ranking_simple": 0.5185950398445129,
"eval_regularize": 0.6885914206504822,
"eval_runtime": 210.8957,
"eval_samples_per_second": 27.454,
"eval_steps_per_second": 1.147,
"step": 144
},
{
"epoch": 0.8162494095418045,
"step": 144,
"total_flos": 0.0,
"train_loss": 0.6411769655015733,
"train_runtime": 5149.1432,
"train_samples_per_second": 19.732,
"train_steps_per_second": 0.068
}
],
"logging_steps": 5,
"max_steps": 352,
"num_input_tokens_seen": 0,
"num_train_epochs": 2,
"save_steps": 36,
"stateful_callbacks": {
"EarlyStoppingCallback": {
"args": {
"early_stopping_patience": 3,
"early_stopping_threshold": 0.0
},
"attributes": {
"early_stopping_patience_counter": 0
}
},
"TrainerControl": {
"args": {
"should_epoch_stop": false,
"should_evaluate": false,
"should_log": false,
"should_save": true,
"should_training_stop": true
},
"attributes": {}
}
},
"total_flos": 0.0,
"train_batch_size": 4,
"trial_name": null,
"trial_params": null
}