Model save

README.md

@@ -1,15 +1,9 @@
 ---
 base_model: dmis-lab/selfbiorag_7b
 tags:
-- alignment-handbook
 - trl
 - dpo
 - generated_from_trainer
-- trl
-- dpo
-- generated_from_trainer
-datasets:
-- HuggingFaceH4/ultrafeedback_binarized
 model-index:
 - name: selfbiorag-7b-dpo-full-wo-healthsearch_qa-ep3
   results: []
@@ -20,17 +14,17 @@ should probably proofread and complete it, then remove this comment. -->
 
 # selfbiorag-7b-dpo-full-wo-healthsearch_qa-ep3
 
-This model is a fine-tuned version of [dmis-lab/selfbiorag_7b](https://huggingface.co/dmis-lab/selfbiorag_7b) on the HuggingFaceH4/ultrafeedback_binarized dataset.
+This model is a fine-tuned version of [dmis-lab/selfbiorag_7b](https://huggingface.co/dmis-lab/selfbiorag_7b) on an unknown dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.5131
-- Rewards/chosen: 0.3874
-- Rewards/rejected: -0.0179
-- Rewards/accuracies: 1.0
-- Rewards/margins: 0.4053
-- Logps/rejected: -92.8718
-- Logps/chosen: -349.6196
-- Logits/rejected: -1.7621
-- Logits/chosen: -1.6801
+- Logits/chosen: -1.6968
+- Logits/rejected: -1.6723
+- Logps/chosen: -158.8701
+- Logps/rejected: -170.2428
+- Loss: 0.6691
+- Rewards/accuracies: 0.6941
+- Rewards/chosen: 0.0706
+- Rewards/margins: 0.0503
+- Rewards/rejected: 0.0202
 
 ## Model description
 
@@ -54,22 +48,25 @@ The following hyperparameters were used during training:
 - eval_batch_size: 8
 - seed: 42
 - distributed_type: multi-GPU
-- num_devices: 8
+- num_devices: 4
 - gradient_accumulation_steps: 2
-- total_train_batch_size: 128
-- total_eval_batch_size: 64
+- total_train_batch_size: 64
+- total_eval_batch_size: 32
 - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
 - lr_scheduler_type: cosine
 - lr_scheduler_warmup_ratio: 0.1
-- num_epochs: 3
+- num_epochs: 1
 
 ### Training results
 
+| Training Loss | Epoch | Step | Logits/chosen | Logits/rejected | Logps/chosen | Logps/rejected | Validation Loss | Rewards/accuracies | Rewards/chosen | Rewards/margins | Rewards/rejected |
+|:-------------:|:-----:|:----:|:-------------:|:---------------:|:------------:|:--------------:|:---------------:|:------------------:|:--------------:|:---------------:|:----------------:|
+| 0.6508        | 0.61  | 100  | -1.6968       | -1.6723         | -158.8701    | -170.2428      | 0.6691          | 0.6941             | 0.0706         | 0.0503          | 0.0202           |
 
 
 ### Framework versions
 
 - Transformers 4.39.0.dev0
-- Pytorch 2.2.1+cu121
+- Pytorch 2.1.2
 - Datasets 2.14.6
 - Tokenizers 0.15.2

all_results.json

@@ -1,21 +1,8 @@
 {
-    "epoch": 3.0,
-    "eval_logits/chosen": -1.680073618888855,
-    "eval_logits/rejected": -1.7620524168014526,
-    "eval_logps/chosen": -349.61962890625,
-    "eval_logps/rejected": -92.87176513671875,
-    "eval_loss": 0.5131469964981079,
-    "eval_rewards/accuracies": 1.0,
-    "eval_rewards/chosen": 0.3873787820339203,
-    "eval_rewards/margins": 0.40528222918510437,
-    "eval_rewards/rejected": -0.017903532832860947,
-    "eval_runtime": 67.3461,
-    "eval_samples": 3077,
-    "eval_samples_per_second": 45.689,
-    "eval_steps_per_second": 0.728,
-    "train_loss": 0.5981406688690185,
-    "train_runtime": 346.5059,
-    "train_samples": 1885,
-    "train_samples_per_second": 16.32,
-    "train_steps_per_second": 0.13
+    "epoch": 1.0,
+    "train_loss": 0.2519006322069866,
+    "train_runtime": 787.0698,
+    "train_samples": 10477,
+    "train_samples_per_second": 13.311,
+    "train_steps_per_second": 0.208
 }

model-00001-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c510f855c5a77fa0a39abce44f3906dd30865ea22922bfa3b4c0f5d175b65c9b
+oid sha256:7e3bf12878dd219177ecbd7042879cc4e9c7ee0478427bc28d5cd74360bc1c66
 size 4939116424

model-00002-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c100741650f0af22f89c2e95607cfb10da99f616315a183657a64cce801c9e28
+oid sha256:dfe885f61c314c6938f605dc911c1de2fe4bbbb4426c99f25c33f776e6ac3dcc
 size 4947390880

model-00003-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ff7646b79096dc96e6fc68d15c163e5fbb19476cf9f8dbf71cae1dfbf52ca7ed
+oid sha256:4779ffacf1ced93b0fae7c156d31864dc9b11c079ff1599f2ad368083d507fef
 size 3590619888

train_results.json

@@ -1,8 +1,8 @@
 {
-    "epoch": 3.0,
-    "train_loss": 0.5981406688690185,
-    "train_runtime": 346.5059,
-    "train_samples": 1885,
-    "train_samples_per_second": 16.32,
-    "train_steps_per_second": 0.13
+    "epoch": 1.0,
+    "train_loss": 0.2519006322069866,
+    "train_runtime": 787.0698,
+    "train_samples": 10477,
+    "train_samples_per_second": 13.311,
+    "train_steps_per_second": 0.208
 }

trainer_state.json

@@ -1,21 +1,21 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 3.0,
+  "epoch": 1.0,
   "eval_steps": 100,
-  "global_step": 45,
+  "global_step": 164,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 0.07,
-      "grad_norm": 14.650781993147133,
-      "learning_rate": 1e-07,
-      "logits/chosen": -1.8544178009033203,
-      "logits/rejected": -1.5131595134735107,
-      "logps/chosen": -382.39324951171875,
-      "logps/rejected": -74.34650421142578,
+      "epoch": 0.01,
+      "grad_norm": 6.5958876428735564,
+      "learning_rate": 2.941176470588235e-08,
+      "logits/chosen": -1.6130714416503906,
+      "logits/rejected": -1.7848026752471924,
+      "logps/chosen": -143.55209350585938,
+      "logps/rejected": -137.43441772460938,
       "loss": 0.6931,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
@@ -24,79 +24,275 @@
       "step": 1
     },
     {
-      "epoch": 0.67,
-      "grad_norm": 13.355694493789834,
-      "learning_rate": 4.809698831278217e-07,
-      "logits/chosen": -1.7981030941009521,
-      "logits/rejected": -1.8637254238128662,
-      "logps/chosen": -297.70697021484375,
-      "logps/rejected": -91.71724700927734,
-      "loss": 0.6887,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": 0.0073351990431547165,
-      "rewards/margins": 0.007683979347348213,
-      "rewards/rejected": -0.000348779110936448,
+      "epoch": 0.06,
+      "grad_norm": 5.967532383605112,
+      "learning_rate": 2.941176470588235e-07,
+      "logits/chosen": -1.8283494710922241,
+      "logits/rejected": -1.7852643728256226,
+      "logps/chosen": -158.81536865234375,
+      "logps/rejected": -151.6327362060547,
+      "loss": 0.693,
+      "rewards/accuracies": 0.4722222089767456,
+      "rewards/chosen": 8.654648991068825e-05,
+      "rewards/margins": 0.0005829257424920797,
+      "rewards/rejected": -0.0004963793326169252,
       "step": 10
     },
     {
-      "epoch": 1.33,
-      "grad_norm": 10.957160162463524,
-      "learning_rate": 3.4567085809127245e-07,
-      "logits/chosen": -1.7605440616607666,
-      "logits/rejected": -1.6980727910995483,
-      "logps/chosen": -317.1847229003906,
-      "logps/rejected": -89.17781066894531,
-      "loss": 0.6424,
-      "rewards/accuracies": 0.9937499761581421,
-      "rewards/chosen": 0.10211040079593658,
-      "rewards/margins": 0.10564006865024567,
-      "rewards/rejected": -0.003529661800712347,
+      "epoch": 0.12,
+      "grad_norm": 5.606818404653461,
+      "learning_rate": 4.994863481875841e-07,
+      "logits/chosen": -1.8151414394378662,
+      "logits/rejected": -1.7734615802764893,
+      "logps/chosen": -151.97584533691406,
+      "logps/rejected": -164.20437622070312,
+      "loss": 0.6923,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": 0.003920617047697306,
+      "rewards/margins": 0.0024364024866372347,
+      "rewards/rejected": 0.001484214561060071,
       "step": 20
     },
     {
-      "epoch": 2.0,
-      "grad_norm": 9.631401952273302,
-      "learning_rate": 1.5432914190872756e-07,
-      "logits/chosen": -1.7424694299697876,
-      "logits/rejected": -1.690342903137207,
-      "logps/chosen": -304.099609375,
-      "logps/rejected": -91.2957763671875,
-      "loss": 0.5679,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": 0.263149231672287,
-      "rewards/margins": 0.2803087830543518,
-      "rewards/rejected": -0.017159538343548775,
+      "epoch": 0.18,
+      "grad_norm": 6.452038531330129,
+      "learning_rate": 4.904133592102591e-07,
+      "logits/chosen": -1.8305763006210327,
+      "logits/rejected": -1.7172702550888062,
+      "logps/chosen": -154.3677520751953,
+      "logps/rejected": -148.50753784179688,
+      "loss": 0.6882,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": 0.016027290374040604,
+      "rewards/margins": 0.00950100552290678,
+      "rewards/rejected": 0.006526285316795111,
       "step": 30
     },
     {
-      "epoch": 2.67,
-      "grad_norm": 9.39281581673066,
-      "learning_rate": 1.9030116872178314e-08,
-      "logits/chosen": -1.7320054769515991,
-      "logits/rejected": -1.6545593738555908,
-      "logps/chosen": -303.2212829589844,
-      "logps/rejected": -99.20294189453125,
-      "loss": 0.5282,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": 0.36085695028305054,
-      "rewards/margins": 0.40523195266723633,
-      "rewards/rejected": -0.04437502473592758,
+      "epoch": 0.24,
+      "grad_norm": 6.2953570308846825,
+      "learning_rate": 4.704015606870022e-07,
+      "logits/chosen": -1.7697455883026123,
+      "logits/rejected": -1.7966588735580444,
+      "logps/chosen": -143.58848571777344,
+      "logps/rejected": -166.49522399902344,
+      "loss": 0.6829,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.03490619733929634,
+      "rewards/margins": 0.02003355883061886,
+      "rewards/rejected": 0.014872634783387184,
       "step": 40
     },
     {
-      "epoch": 3.0,
-      "step": 45,
+      "epoch": 0.3,
+      "grad_norm": 6.274119591898531,
+      "learning_rate": 4.4036148959228356e-07,
+      "logits/chosen": -1.7394487857818604,
+      "logits/rejected": -1.804693579673767,
+      "logps/chosen": -159.61492919921875,
+      "logps/rejected": -136.1581268310547,
+      "loss": 0.6763,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": 0.06023404002189636,
+      "rewards/margins": 0.042321957647800446,
+      "rewards/rejected": 0.017912080511450768,
+      "step": 50
+    },
+    {
+      "epoch": 0.37,
+      "grad_norm": 6.180992532830828,
+      "learning_rate": 4.016599693735638e-07,
+      "logits/chosen": -1.6605278253555298,
+      "logits/rejected": -1.724905252456665,
+      "logps/chosen": -146.7899932861328,
+      "logps/rejected": -148.02505493164062,
+      "loss": 0.6733,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": 0.0686994269490242,
+      "rewards/margins": 0.04312276840209961,
+      "rewards/rejected": 0.02557666040956974,
+      "step": 60
+    },
+    {
+      "epoch": 0.43,
+      "grad_norm": 5.590599679916071,
+      "learning_rate": 3.5605791947475926e-07,
+      "logits/chosen": -1.7533237934112549,
+      "logits/rejected": -1.702845811843872,
+      "logps/chosen": -146.6136474609375,
+      "logps/rejected": -140.97921752929688,
+      "loss": 0.6631,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.07920090854167938,
+      "rewards/margins": 0.053236376494169235,
+      "rewards/rejected": 0.0259645227342844,
+      "step": 70
+    },
+    {
+      "epoch": 0.49,
+      "grad_norm": 5.096416269116106,
+      "learning_rate": 3.056302334890786e-07,
+      "logits/chosen": -1.616193413734436,
+      "logits/rejected": -1.6094154119491577,
+      "logps/chosen": -142.79188537597656,
+      "logps/rejected": -140.85447692871094,
+      "loss": 0.6609,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 0.09861920028924942,
+      "rewards/margins": 0.0706188827753067,
+      "rewards/rejected": 0.028000324964523315,
+      "step": 80
+    },
+    {
+      "epoch": 0.55,
+      "grad_norm": 5.517912420297569,
+      "learning_rate": 2.526713714858433e-07,
+      "logits/chosen": -1.608278512954712,
+      "logits/rejected": -1.5585658550262451,
+      "logps/chosen": -132.39981079101562,
+      "logps/rejected": -143.10488891601562,
+      "loss": 0.6557,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": 0.1142318844795227,
+      "rewards/margins": 0.07896542549133301,
+      "rewards/rejected": 0.0352664515376091,
+      "step": 90
+    },
+    {
+      "epoch": 0.61,
+      "grad_norm": 5.179137970855667,
+      "learning_rate": 1.9959096206109175e-07,
+      "logits/chosen": -1.5899827480316162,
+      "logits/rejected": -1.5742290019989014,
+      "logps/chosen": -136.0356903076172,
+      "logps/rejected": -162.7815704345703,
+      "loss": 0.6508,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": 0.11762702465057373,
+      "rewards/margins": 0.08622404932975769,
+      "rewards/rejected": 0.03140297532081604,
+      "step": 100
+    },
+    {
+      "epoch": 0.61,
+      "eval_logits/chosen": -1.6967989206314087,
+      "eval_logits/rejected": -1.6722551584243774,
+      "eval_logps/chosen": -158.87005615234375,
+      "eval_logps/rejected": -170.24278259277344,
+      "eval_loss": 0.6690559983253479,
+      "eval_rewards/accuracies": 0.6940954923629761,
+      "eval_rewards/chosen": 0.07056128978729248,
+      "eval_rewards/margins": 0.050339534878730774,
+      "eval_rewards/rejected": 0.020221758633852005,
+      "eval_runtime": 1977.6877,
+      "eval_samples_per_second": 9.659,
+      "eval_steps_per_second": 0.302,
+      "step": 100
+    },
+    {
+      "epoch": 0.67,
+      "grad_norm": 5.410829812028072,
+      "learning_rate": 1.4880416421940154e-07,
+      "logits/chosen": -1.6502714157104492,
+      "logits/rejected": -1.6523603200912476,
+      "logps/chosen": -134.38687133789062,
+      "logps/rejected": -157.00936889648438,
+      "loss": 0.6512,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": 0.11995081603527069,
+      "rewards/margins": 0.09394902735948563,
+      "rewards/rejected": 0.026001790538430214,
+      "step": 110
+    },
+    {
+      "epoch": 0.73,
+      "grad_norm": 5.845780336717107,
+      "learning_rate": 1.0262177762208507e-07,
+      "logits/chosen": -1.565212607383728,
+      "logits/rejected": -1.6423566341400146,
+      "logps/chosen": -143.96304321289062,
+      "logps/rejected": -149.28546142578125,
+      "loss": 0.6496,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": 0.13488885760307312,
+      "rewards/margins": 0.10831846296787262,
+      "rewards/rejected": 0.026570383459329605,
+      "step": 120
+    },
+    {
+      "epoch": 0.79,
+      "grad_norm": 5.76403048084688,
+      "learning_rate": 6.31451011862412e-08,
+      "logits/chosen": -1.6332323551177979,
+      "logits/rejected": -1.6044152975082397,
+      "logps/chosen": -137.62985229492188,
+      "logps/rejected": -159.90980529785156,
+      "loss": 0.6439,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": 0.13013367354869843,
+      "rewards/margins": 0.10071909427642822,
+      "rewards/rejected": 0.02941458486020565,
+      "step": 130
+    },
+    {
+      "epoch": 0.85,
+      "grad_norm": 5.119446644831888,
+      "learning_rate": 3.217032396915265e-08,
+      "logits/chosen": -1.569746971130371,
+      "logits/rejected": -1.6146259307861328,
+      "logps/chosen": -130.83258056640625,
+      "logps/rejected": -160.59701538085938,
+      "loss": 0.6439,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 0.1322535276412964,
+      "rewards/margins": 0.10249896347522736,
+      "rewards/rejected": 0.029754554852843285,
+      "step": 140
+    },
+    {
+      "epoch": 0.91,
+      "grad_norm": 5.590191167835734,
+      "learning_rate": 1.1106798553464802e-08,
+      "logits/chosen": -1.6109774112701416,
+      "logits/rejected": -1.607143759727478,
+      "logps/chosen": -145.5422821044922,
+      "logps/rejected": -155.8082733154297,
+      "loss": 0.6426,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": 0.14719954133033752,
+      "rewards/margins": 0.11081697046756744,
+      "rewards/rejected": 0.03638254478573799,
+      "step": 150
+    },
+    {
+      "epoch": 0.98,
+      "grad_norm": 5.417981503927173,
+      "learning_rate": 9.129154946982687e-10,
+      "logits/chosen": -1.5755327939987183,
+      "logits/rejected": -1.6533405780792236,
+      "logps/chosen": -144.75936889648438,
+      "logps/rejected": -150.3732452392578,
+      "loss": 0.6439,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": 0.1261114478111267,
+      "rewards/margins": 0.10229575634002686,
+      "rewards/rejected": 0.023815687745809555,
+      "step": 160
+    },
+    {
+      "epoch": 1.0,
+      "step": 164,
       "total_flos": 0.0,
-      "train_loss": 0.5981406688690185,
-      "train_runtime": 346.5059,
-      "train_samples_per_second": 16.32,
-      "train_steps_per_second": 0.13
+      "train_loss": 0.2519006322069866,
+      "train_runtime": 787.0698,
+      "train_samples_per_second": 13.311,
+      "train_steps_per_second": 0.208
     }
   ],
   "logging_steps": 10,
-  "max_steps": 45,
+  "max_steps": 164,
   "num_input_tokens_seen": 0,
-  "num_train_epochs": 3,
+  "num_train_epochs": 1,
   "save_steps": 100,
   "total_flos": 0.0,
   "train_batch_size": 8,

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4aa92a8826c6d09e2cf1a597f1a350d4b560367e1b01a843b0bed4d983eaabf7
+oid sha256:28f15ca60ba3b07f2027c55b34635f9cf9250401baa46a13461b5c1fbbfdd3f5
 size 6264