Training in progress, step 351, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0809bafbece4702b690076073d24e954afb4de759d2258a07d2ced1ca7d828ec
+oid sha256:049ec0ada0813997e586979480e5c26282234a3e55448657344416c90be4b443
 size 550593856

last-checkpoint/global_step351/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d47520d68a34fea7b59d08fb29119a1ee7cf60c9dcc38af2d461972a9048513d
+size 243591168

last-checkpoint/global_step351/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:75c9062ade3c410712aea545d4c8225dec40b61c91ba419a0e810bd5548dbcd5
+size 243591168

last-checkpoint/global_step351/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c29fcb5aa1969ed584d0132e62648df263773a5deea7782a49b4f7d7d544d877
+size 243591168

last-checkpoint/global_step351/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:89048b4aafab116843f6ac627f9f7e16d1dc2dcc70ffae5553e43839655c7227
+size 243591168

last-checkpoint/global_step351/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5ea0fb8acfc0d14afc7df300706b4e00cf47093b3e7ec83b3316107877ce4b51
+size 243591168

last-checkpoint/global_step351/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d73cea9cd303ef921a1cafa7f42287c41862b8166c07075aa94b708385730e98
+size 243591168

last-checkpoint/global_step351/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c01386ea76c02f985434f5721dc8e0b68d1146e1b5dd080d0f92aef8cbdd7856
+size 243591168

last-checkpoint/global_step351/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f8f777796c3242494e22af251b1bc8e1e2fc4a717e81e4b570bfdb251ec44e55
+size 243591168

last-checkpoint/global_step351/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8995aabcd4b6e9742412a808c926eab9786a2d6f245f494aa6ea0dcbf2c093b8
+size 211435686

last-checkpoint/global_step351/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8df642eecd1ed86817a38a932e19f9c2058cce1a341153c2d0fbdb9ab73bc81a
+size 211435686

last-checkpoint/global_step351/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5bf99ead9d6526b5da49ed12bf60aa817186b62c92a8e3914544ae9191c342c0
+size 211435686

last-checkpoint/global_step351/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f76a8e21348e535d739f6d1ac0a4767d7d92a89a4cbec2d92a94410011b4ed6f
+size 211435686

last-checkpoint/global_step351/zero_pp_rank_4_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0765706d29af686fb0fb59f87e74e78d9f7d68a11a741cc6ec9d2ab538e2eef2
+size 211435686

last-checkpoint/global_step351/zero_pp_rank_5_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aa336096e2796613b15b2357492cb9525a20c10f7dce9ae9522127817578506a
+size 211435686

last-checkpoint/global_step351/zero_pp_rank_6_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6872a1e1763d577b225b7c9f547de78e60d18697cda8af4111b6bbe74d9b1354
+size 211435686

last-checkpoint/global_step351/zero_pp_rank_7_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ca144d0c56353acfd2da1d43386849a4d9e20c88b087d3c19d8f31420f046557
+size 211435686

last-checkpoint/latest

@@ -1 +1 @@
-global_step312
+global_step351

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9662f90ac871284e42fb85b7bc6f2a2f3759cac7bb678faaa94f777fedabb313
+oid sha256:0fec0d859a8870bb3863562a0fbaa6ebb33536c9a365b6abc0e8f09aacd3377c
 size 15920

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b2f3b2288ad04448267691f0a9a926d6ea4e1872e2d6a6114ff6a455551a1a14
+oid sha256:212dc5192b40486002c7fe7e08f770847069213c90b44b8eeb5c8c552aa09d2d
 size 15920

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d7054584e190f07a5be6417b3d068feba1c297bfef11d44338e024157eac1eea
+oid sha256:ef0f0879c31278fe1422ca799f1e946f52627895e6b6b450451e9b838670c583
 size 15920

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:699ca77f5f867262084cafd79c14bfb3ebb441808dca86d851f008f9e856702e
+oid sha256:66c283d16fff6ad5c32bb8063ebb0876ab2d3331a701287a5518f6611da12f69
 size 15920

last-checkpoint/rng_state_4.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6107f40d55050f71acd0cf64cee3cb55f7a5f2f65beb01564a3368189c88b78a
+oid sha256:1d45b771d912fd42fd57bf4de4aad3e9035242ea784f2ed87aba4c621e5cd51a
 size 15920

last-checkpoint/rng_state_5.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:34860f3d27e6296937a819f44272b9427aaea2bb9384f4f850d4d368c07caa62
+oid sha256:5016a6096b206da65160f901789323144aa46e697f4c7af7e972b16bb657ce2c
 size 15920

last-checkpoint/rng_state_6.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0486011b840a4dd1cc3653fce36b9eeabdeaf4aaa824c495c5201d1b4398ad03
+oid sha256:b70b7d3a0a245937c94c1dfab4a674840cffae712e68cd2ec85b8111cd19b6ae
 size 15920

last-checkpoint/rng_state_7.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8865eac23bdd917b7f787c6c0a4ad6441de4e3e64c1376b83d0e8ca0e795d614
+oid sha256:894e3dc130b31d1ff09091d3fba8c4756c7829f76a26489876f6acf7ddf58730
 size 15920

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:cedf4b34e868c4fb23dc007fcfc750ae8b8c1963085e5e7d64434ad8b0f1b971
+oid sha256:77a901e68fe508c580838a0a83e25dcc491921acb23119eaf14194c3e5bc3346
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.8125,
+  "epoch": 0.9140625,
   "eval_steps": 39,
-  "global_step": 312,
+  "global_step": 351,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -2263,6 +2263,287 @@
       "eval_samples_per_second": 1.222,
       "eval_steps_per_second": 0.153,
       "step": 312
+    },
+    {
+      "epoch": 0.8151041666666666,
+      "grad_norm": 0.13210051743324183,
+      "learning_rate": 1.8187729717774925e-05,
+      "loss": 2.2753,
+      "step": 313
+    },
+    {
+      "epoch": 0.8177083333333334,
+      "grad_norm": 0.1474646492548143,
+      "learning_rate": 1.7965726034785466e-05,
+      "loss": 2.3933,
+      "step": 314
+    },
+    {
+      "epoch": 0.8203125,
+      "grad_norm": 0.16871013478557104,
+      "learning_rate": 1.7746481011411416e-05,
+      "loss": 2.3614,
+      "step": 315
+    },
+    {
+      "epoch": 0.8229166666666666,
+      "grad_norm": 0.15193041380102754,
+      "learning_rate": 1.753001097907572e-05,
+      "loss": 2.3525,
+      "step": 316
+    },
+    {
+      "epoch": 0.8255208333333334,
+      "grad_norm": 0.14955167945389547,
+      "learning_rate": 1.7316332062494016e-05,
+      "loss": 2.4257,
+      "step": 317
+    },
+    {
+      "epoch": 0.828125,
+      "grad_norm": 0.1572944020219996,
+      "learning_rate": 1.710546017847347e-05,
+      "loss": 2.5182,
+      "step": 318
+    },
+    {
+      "epoch": 0.8307291666666666,
+      "grad_norm": 0.1493341360803509,
+      "learning_rate": 1.6897411034727218e-05,
+      "loss": 2.3794,
+      "step": 319
+    },
+    {
+      "epoch": 0.8333333333333334,
+      "grad_norm": 0.13561727835199758,
+      "learning_rate": 1.66922001287042e-05,
+      "loss": 2.2277,
+      "step": 320
+    },
+    {
+      "epoch": 0.8359375,
+      "grad_norm": 0.13052163343368234,
+      "learning_rate": 1.648984274643487e-05,
+      "loss": 2.309,
+      "step": 321
+    },
+    {
+      "epoch": 0.8385416666666666,
+      "grad_norm": 0.1578566178286769,
+      "learning_rate": 1.629035396139247e-05,
+      "loss": 2.4832,
+      "step": 322
+    },
+    {
+      "epoch": 0.8411458333333334,
+      "grad_norm": 0.16163117665249763,
+      "learning_rate": 1.6093748633370295e-05,
+      "loss": 2.3515,
+      "step": 323
+    },
+    {
+      "epoch": 0.84375,
+      "grad_norm": 0.16589478114748782,
+      "learning_rate": 1.5900041407374708e-05,
+      "loss": 2.5438,
+      "step": 324
+    },
+    {
+      "epoch": 0.8463541666666666,
+      "grad_norm": 0.13933316472296842,
+      "learning_rate": 1.5709246712534315e-05,
+      "loss": 2.4306,
+      "step": 325
+    },
+    {
+      "epoch": 0.8489583333333334,
+      "grad_norm": 0.1606181863885322,
+      "learning_rate": 1.5521378761025113e-05,
+      "loss": 2.5248,
+      "step": 326
+    },
+    {
+      "epoch": 0.8515625,
+      "grad_norm": 0.15583835916491617,
+      "learning_rate": 1.5336451547011838e-05,
+      "loss": 2.57,
+      "step": 327
+    },
+    {
+      "epoch": 0.8541666666666666,
+      "grad_norm": 0.13760770777048018,
+      "learning_rate": 1.515447884560556e-05,
+      "loss": 2.4217,
+      "step": 328
+    },
+    {
+      "epoch": 0.8567708333333334,
+      "grad_norm": 0.14254277680719865,
+      "learning_rate": 1.4975474211837561e-05,
+      "loss": 2.5538,
+      "step": 329
+    },
+    {
+      "epoch": 0.859375,
+      "grad_norm": 0.15022329502854323,
+      "learning_rate": 1.479945097964967e-05,
+      "loss": 2.4875,
+      "step": 330
+    },
+    {
+      "epoch": 0.8619791666666666,
+      "grad_norm": 0.14442101795365278,
+      "learning_rate": 1.4626422260900962e-05,
+      "loss": 2.4053,
+      "step": 331
+    },
+    {
+      "epoch": 0.8645833333333334,
+      "grad_norm": 0.1339112894355035,
+      "learning_rate": 1.4456400944391146e-05,
+      "loss": 2.3938,
+      "step": 332
+    },
+    {
+      "epoch": 0.8671875,
+      "grad_norm": 0.15065063782655086,
+      "learning_rate": 1.4289399694900398e-05,
+      "loss": 2.4288,
+      "step": 333
+    },
+    {
+      "epoch": 0.8697916666666666,
+      "grad_norm": 0.15054714911892625,
+      "learning_rate": 1.4125430952246071e-05,
+      "loss": 2.4214,
+      "step": 334
+    },
+    {
+      "epoch": 0.8723958333333334,
+      "grad_norm": 0.1560638225083581,
+      "learning_rate": 1.3964506930355947e-05,
+      "loss": 2.3693,
+      "step": 335
+    },
+    {
+      "epoch": 0.875,
+      "grad_norm": 0.17170476718899333,
+      "learning_rate": 1.380663961635852e-05,
+      "loss": 2.3902,
+      "step": 336
+    },
+    {
+      "epoch": 0.8776041666666666,
+      "grad_norm": 0.14853468036224993,
+      "learning_rate": 1.3651840769690028e-05,
+      "loss": 2.5229,
+      "step": 337
+    },
+    {
+      "epoch": 0.8802083333333334,
+      "grad_norm": 0.13862935132651075,
+      "learning_rate": 1.350012192121854e-05,
+      "loss": 2.272,
+      "step": 338
+    },
+    {
+      "epoch": 0.8828125,
+      "grad_norm": 0.16359619179880716,
+      "learning_rate": 1.3351494372384995e-05,
+      "loss": 2.5031,
+      "step": 339
+    },
+    {
+      "epoch": 0.8854166666666666,
+      "grad_norm": 0.15198032362712752,
+      "learning_rate": 1.3205969194361395e-05,
+      "loss": 2.355,
+      "step": 340
+    },
+    {
+      "epoch": 0.8880208333333334,
+      "grad_norm": 0.12892013869101834,
+      "learning_rate": 1.3063557227226094e-05,
+      "loss": 2.3602,
+      "step": 341
+    },
+    {
+      "epoch": 0.890625,
+      "grad_norm": 0.1498170376596413,
+      "learning_rate": 1.292426907915634e-05,
+      "loss": 2.3912,
+      "step": 342
+    },
+    {
+      "epoch": 0.8932291666666666,
+      "grad_norm": 0.15638364831022664,
+      "learning_rate": 1.2788115125638068e-05,
+      "loss": 2.4359,
+      "step": 343
+    },
+    {
+      "epoch": 0.8958333333333334,
+      "grad_norm": 0.151291537071441,
+      "learning_rate": 1.2655105508693065e-05,
+      "loss": 2.3082,
+      "step": 344
+    },
+    {
+      "epoch": 0.8984375,
+      "grad_norm": 0.14865959228520678,
+      "learning_rate": 1.252525013612346e-05,
+      "loss": 2.3877,
+      "step": 345
+    },
+    {
+      "epoch": 0.9010416666666666,
+      "grad_norm": 0.1459248998632338,
+      "learning_rate": 1.2398558680773736e-05,
+      "loss": 2.3293,
+      "step": 346
+    },
+    {
+      "epoch": 0.9036458333333334,
+      "grad_norm": 0.14220792400443616,
+      "learning_rate": 1.227504057981016e-05,
+      "loss": 2.4427,
+      "step": 347
+    },
+    {
+      "epoch": 0.90625,
+      "grad_norm": 0.15415554255450412,
+      "learning_rate": 1.2154705034017866e-05,
+      "loss": 2.383,
+      "step": 348
+    },
+    {
+      "epoch": 0.9088541666666666,
+      "grad_norm": 0.1457198339110242,
+      "learning_rate": 1.203756100711545e-05,
+      "loss": 2.3751,
+      "step": 349
+    },
+    {
+      "epoch": 0.9114583333333334,
+      "grad_norm": 0.15986391066488098,
+      "learning_rate": 1.1923617225087293e-05,
+      "loss": 2.3891,
+      "step": 350
+    },
+    {
+      "epoch": 0.9140625,
+      "grad_norm": 0.13681607679501942,
+      "learning_rate": 1.1812882175533564e-05,
+      "loss": 2.2878,
+      "step": 351
+    },
+    {
+      "epoch": 0.9140625,
+      "eval_loss": 2.397103786468506,
+      "eval_runtime": 65.1202,
+      "eval_samples_per_second": 1.228,
+      "eval_steps_per_second": 0.154,
+      "step": 351
     }
   ],
   "logging_steps": 1,
@@ -2282,7 +2563,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 103140418387968.0,
+  "total_flos": 116032970686464.0,
   "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null