Hugging Face's logo

DUAL-GPO
/
phi-2-gpo-renew2-b0.001-0.05ultrafeedback-rank128-i1

PEFT
TensorBoard
Safetensors
phi
alignment-handbook
Generated from Trainer
trl
dpo
Model card Files Metrics Community
BraylonDash commited on
Commit
41b8675
·
verified ·
1 Parent(s): 19a28a4

Model save

Browse files
Files changed (6) hide show
  1. README.md +60 -0
  2. adapter_model.safetensors +1 -1
  3. all_results.json +8 -0
  4. runs/May07_00-24-01_gpu4-119-5/events.out.tfevents.1715005526.gpu4-119-5.2623169.0 +2 -2
  5. train_results.json +8 -0
  6. trainer_state.json +310 -0
README.md ADDED
@@ -0,0 +1,60 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ license: mit
3
+ library_name: peft
4
+ tags:
5
+ - trl
6
+ - dpo
7
+ - generated_from_trainer
8
+ base_model: microsoft/phi-2
9
+ model-index:
10
+ - name: phi-2-gpo-renew2-b0.001-0.05ultrafeedback-rank128-i1
11
+ results: []
12
+ ---
13
+
14
+ <!-- This model card has been generated automatically according to the information the Trainer had access to. You
15
+ should probably proofread and complete it, then remove this comment. -->
16
+
17
+ # phi-2-gpo-renew2-b0.001-0.05ultrafeedback-rank128-i1
18
+
19
+ This model is a fine-tuned version of [microsoft/phi-2](https://huggingface.co/microsoft/phi-2) on the None dataset.
20
+
21
+ ## Model description
22
+
23
+ More information needed
24
+
25
+ ## Intended uses & limitations
26
+
27
+ More information needed
28
+
29
+ ## Training and evaluation data
30
+
31
+ More information needed
32
+
33
+ ## Training procedure
34
+
35
+ ### Training hyperparameters
36
+
37
+ The following hyperparameters were used during training:
38
+ - learning_rate: 5e-06
39
+ - train_batch_size: 4
40
+ - eval_batch_size: 4
41
+ - seed: 42
42
+ - distributed_type: multi-GPU
43
+ - gradient_accumulation_steps: 4
44
+ - total_train_batch_size: 16
45
+ - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
46
+ - lr_scheduler_type: cosine
47
+ - lr_scheduler_warmup_ratio: 0.1
48
+ - num_epochs: 1
49
+
50
+ ### Training results
51
+
52
+
53
+
54
+ ### Framework versions
55
+
56
+ - PEFT 0.7.1
57
+ - Transformers 4.36.2
58
+ - Pytorch 2.1.2
59
+ - Datasets 2.14.6
60
+ - Tokenizers 0.15.2
adapter_model.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:b416beb8331c40f25d0d4ba2da2bdffac3d67e5e0e8af1c0399a28384e5bdef4
3
  size 41951728
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:43fe8caec1898ff4ab65decc05fa0077e80b39029af651bd534a7e5446be0cd6
3
  size 41951728
all_results.json ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "epoch": 1.0,
3
+ "train_loss": 0.054820564112270063,
4
+ "train_runtime": 1654.1929,
5
+ "train_samples": 3056,
6
+ "train_samples_per_second": 1.847,
7
+ "train_steps_per_second": 0.115
8
+ }
runs/May07_00-24-01_gpu4-119-5/events.out.tfevents.1715005526.gpu4-119-5.2623169.0 CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:41cccdfab7f5f0fc2e3a5035f0dcdf6798f487cd515da312318bbb0554ddcdfd
3
- size 11632
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e3724110b086322326c66be0c4c556d0aeed12b7c6667ec797513c1a17f59254
3
+ size 17670
train_results.json ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "epoch": 1.0,
3
+ "train_loss": 0.054820564112270063,
4
+ "train_runtime": 1654.1929,
5
+ "train_samples": 3056,
6
+ "train_samples_per_second": 1.847,
7
+ "train_steps_per_second": 0.115
8
+ }
trainer_state.json ADDED
@@ -0,0 +1,310 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_metric": null,
3
+ "best_model_checkpoint": null,
4
+ "epoch": 1.0,
5
+ "eval_steps": 500,
6
+ "global_step": 191,
7
+ "is_hyper_param_search": false,
8
+ "is_local_process_zero": true,
9
+ "is_world_process_zero": true,
10
+ "log_history": [
11
+ {
12
+ "epoch": 0.01,
13
+ "learning_rate": 2.5000000000000004e-07,
14
+ "logits/chosen": -0.11423701792955399,
15
+ "logits/rejected": -0.28093862533569336,
16
+ "logps/chosen": -2311.106689453125,
17
+ "logps/rejected": -2321.17236328125,
18
+ "loss": 0.0345,
19
+ "rewards/accuracies": 0.0,
20
+ "rewards/chosen": 0.0,
21
+ "rewards/margins": 0.0,
22
+ "rewards/rejected": 0.0,
23
+ "step": 1
24
+ },
25
+ {
26
+ "epoch": 0.05,
27
+ "learning_rate": 2.5e-06,
28
+ "logits/chosen": -0.20504091680049896,
29
+ "logits/rejected": -0.19551396369934082,
30
+ "logps/chosen": -2097.760009765625,
31
+ "logps/rejected": -1615.1165771484375,
32
+ "loss": 0.0635,
33
+ "rewards/accuracies": 0.4513888955116272,
34
+ "rewards/chosen": 0.0004200673720333725,
35
+ "rewards/margins": 0.00015599567268509418,
36
+ "rewards/rejected": 0.00026407171390019357,
37
+ "step": 10
38
+ },
39
+ {
40
+ "epoch": 0.1,
41
+ "learning_rate": 5e-06,
42
+ "logits/chosen": -0.19404059648513794,
43
+ "logits/rejected": -0.21347244083881378,
44
+ "logps/chosen": -2510.72412109375,
45
+ "logps/rejected": -2259.84130859375,
46
+ "loss": 0.0536,
47
+ "rewards/accuracies": 0.4937500059604645,
48
+ "rewards/chosen": 0.008191073313355446,
49
+ "rewards/margins": 0.0010476394090801477,
50
+ "rewards/rejected": 0.007143433205783367,
51
+ "step": 20
52
+ },
53
+ {
54
+ "epoch": 0.16,
55
+ "learning_rate": 4.957927758986888e-06,
56
+ "logits/chosen": -0.20243415236473083,
57
+ "logits/rejected": -0.2350526601076126,
58
+ "logps/chosen": -2171.201416015625,
59
+ "logps/rejected": -1893.294189453125,
60
+ "loss": 0.0479,
61
+ "rewards/accuracies": 0.4937500059604645,
62
+ "rewards/chosen": 0.04045766964554787,
63
+ "rewards/margins": 0.005300281103700399,
64
+ "rewards/rejected": 0.03515738993883133,
65
+ "step": 30
66
+ },
67
+ {
68
+ "epoch": 0.21,
69
+ "learning_rate": 4.833127094718643e-06,
70
+ "logits/chosen": -0.23783007264137268,
71
+ "logits/rejected": -0.23550598323345184,
72
+ "logps/chosen": -1967.8616943359375,
73
+ "logps/rejected": -1635.60595703125,
74
+ "loss": 0.0585,
75
+ "rewards/accuracies": 0.5062500238418579,
76
+ "rewards/chosen": 0.05153650790452957,
77
+ "rewards/margins": 0.009783166460692883,
78
+ "rewards/rejected": 0.04175334423780441,
79
+ "step": 40
80
+ },
81
+ {
82
+ "epoch": 0.26,
83
+ "learning_rate": 4.629798522095818e-06,
84
+ "logits/chosen": -0.18633902072906494,
85
+ "logits/rejected": -0.1957712322473526,
86
+ "logps/chosen": -1903.855712890625,
87
+ "logps/rejected": -1633.9322509765625,
88
+ "loss": 0.0553,
89
+ "rewards/accuracies": 0.5249999761581421,
90
+ "rewards/chosen": 0.0319008007645607,
91
+ "rewards/margins": 0.005539845675230026,
92
+ "rewards/rejected": 0.026360953226685524,
93
+ "step": 50
94
+ },
95
+ {
96
+ "epoch": 0.31,
97
+ "learning_rate": 4.354785632088204e-06,
98
+ "logits/chosen": -0.24222198128700256,
99
+ "logits/rejected": -0.2740092873573303,
100
+ "logps/chosen": -2073.35693359375,
101
+ "logps/rejected": -1743.849365234375,
102
+ "loss": 0.0371,
103
+ "rewards/accuracies": 0.53125,
104
+ "rewards/chosen": 0.04043535515666008,
105
+ "rewards/margins": 0.00865642074495554,
106
+ "rewards/rejected": 0.031778931617736816,
107
+ "step": 60
108
+ },
109
+ {
110
+ "epoch": 0.37,
111
+ "learning_rate": 4.017344751567892e-06,
112
+ "logits/chosen": -0.20423555374145508,
113
+ "logits/rejected": -0.20531468093395233,
114
+ "logps/chosen": -1964.8072509765625,
115
+ "logps/rejected": -1502.9725341796875,
116
+ "loss": 0.0698,
117
+ "rewards/accuracies": 0.6187499761581421,
118
+ "rewards/chosen": 0.058043450117111206,
119
+ "rewards/margins": 0.015346303582191467,
120
+ "rewards/rejected": 0.04269714653491974,
121
+ "step": 70
122
+ },
123
+ {
124
+ "epoch": 0.42,
125
+ "learning_rate": 3.628833395777224e-06,
126
+ "logits/chosen": -0.19541537761688232,
127
+ "logits/rejected": -0.2006763517856598,
128
+ "logps/chosen": -2096.223388671875,
129
+ "logps/rejected": -1828.358154296875,
130
+ "loss": 0.0452,
131
+ "rewards/accuracies": 0.512499988079071,
132
+ "rewards/chosen": 0.07907889783382416,
133
+ "rewards/margins": 0.013175728730857372,
134
+ "rewards/rejected": 0.06590317189693451,
135
+ "step": 80
136
+ },
137
+ {
138
+ "epoch": 0.47,
139
+ "learning_rate": 3.2023279994339242e-06,
140
+ "logits/chosen": -0.23748663067817688,
141
+ "logits/rejected": -0.25478672981262207,
142
+ "logps/chosen": -2252.242919921875,
143
+ "logps/rejected": -2046.766357421875,
144
+ "loss": 0.0523,
145
+ "rewards/accuracies": 0.48750001192092896,
146
+ "rewards/chosen": 0.0834890678524971,
147
+ "rewards/margins": 0.011992331594228745,
148
+ "rewards/rejected": 0.07149673998355865,
149
+ "step": 90
150
+ },
151
+ {
152
+ "epoch": 0.52,
153
+ "learning_rate": 2.752183792800671e-06,
154
+ "logits/chosen": -0.18519023060798645,
155
+ "logits/rejected": -0.19959956407546997,
156
+ "logps/chosen": -2170.12353515625,
157
+ "logps/rejected": -1764.599365234375,
158
+ "loss": 0.0609,
159
+ "rewards/accuracies": 0.5562499761581421,
160
+ "rewards/chosen": 0.06967119872570038,
161
+ "rewards/margins": 0.01504653412848711,
162
+ "rewards/rejected": 0.05462465435266495,
163
+ "step": 100
164
+ },
165
+ {
166
+ "epoch": 0.58,
167
+ "learning_rate": 2.2935516363191695e-06,
168
+ "logits/chosen": -0.2315918654203415,
169
+ "logits/rejected": -0.22946839034557343,
170
+ "logps/chosen": -2024.966064453125,
171
+ "logps/rejected": -1654.7867431640625,
172
+ "loss": 0.0416,
173
+ "rewards/accuracies": 0.53125,
174
+ "rewards/chosen": 0.07086780667304993,
175
+ "rewards/margins": 0.015912258997559547,
176
+ "rewards/rejected": 0.05495554953813553,
177
+ "step": 110
178
+ },
179
+ {
180
+ "epoch": 0.63,
181
+ "learning_rate": 1.8418680760885028e-06,
182
+ "logits/chosen": -0.2193867713212967,
183
+ "logits/rejected": -0.23512962460517883,
184
+ "logps/chosen": -2064.80419921875,
185
+ "logps/rejected": -1861.2181396484375,
186
+ "loss": 0.0517,
187
+ "rewards/accuracies": 0.4937500059604645,
188
+ "rewards/chosen": 0.068758025765419,
189
+ "rewards/margins": 0.00728983711451292,
190
+ "rewards/rejected": 0.06146818399429321,
191
+ "step": 120
192
+ },
193
+ {
194
+ "epoch": 0.68,
195
+ "learning_rate": 1.4123357837948177e-06,
196
+ "logits/chosen": -0.18262717127799988,
197
+ "logits/rejected": -0.2169136106967926,
198
+ "logps/chosen": -2098.0537109375,
199
+ "logps/rejected": -1826.62890625,
200
+ "loss": 0.0561,
201
+ "rewards/accuracies": 0.512499988079071,
202
+ "rewards/chosen": 0.0704958364367485,
203
+ "rewards/margins": 0.009869338013231754,
204
+ "rewards/rejected": 0.060626495629549026,
205
+ "step": 130
206
+ },
207
+ {
208
+ "epoch": 0.73,
209
+ "learning_rate": 1.0194118683375502e-06,
210
+ "logits/chosen": -0.21682734787464142,
211
+ "logits/rejected": -0.2506701350212097,
212
+ "logps/chosen": -2177.736083984375,
213
+ "logps/rejected": -1810.03125,
214
+ "loss": 0.0554,
215
+ "rewards/accuracies": 0.5249999761581421,
216
+ "rewards/chosen": 0.06924765557050705,
217
+ "rewards/margins": 0.014119689352810383,
218
+ "rewards/rejected": 0.05512797087430954,
219
+ "step": 140
220
+ },
221
+ {
222
+ "epoch": 0.79,
223
+ "learning_rate": 6.763212814534484e-07,
224
+ "logits/chosen": -0.206621453166008,
225
+ "logits/rejected": -0.20318496227264404,
226
+ "logps/chosen": -2247.090087890625,
227
+ "logps/rejected": -1934.0435791015625,
228
+ "loss": 0.0511,
229
+ "rewards/accuracies": 0.53125,
230
+ "rewards/chosen": 0.06437276303768158,
231
+ "rewards/margins": 0.010379484854638577,
232
+ "rewards/rejected": 0.05399327352643013,
233
+ "step": 150
234
+ },
235
+ {
236
+ "epoch": 0.84,
237
+ "learning_rate": 3.94611695031086e-07,
238
+ "logits/chosen": -0.22235846519470215,
239
+ "logits/rejected": -0.21236348152160645,
240
+ "logps/chosen": -2159.637939453125,
241
+ "logps/rejected": -1753.1939697265625,
242
+ "loss": 0.0597,
243
+ "rewards/accuracies": 0.5687500238418579,
244
+ "rewards/chosen": 0.05504429340362549,
245
+ "rewards/margins": 0.010210799984633923,
246
+ "rewards/rejected": 0.04483348876237869,
247
+ "step": 160
248
+ },
249
+ {
250
+ "epoch": 0.89,
251
+ "learning_rate": 1.837648319629956e-07,
252
+ "logits/chosen": -0.209184929728508,
253
+ "logits/rejected": -0.24073660373687744,
254
+ "logps/chosen": -2140.547119140625,
255
+ "logps/rejected": -1967.44140625,
256
+ "loss": 0.067,
257
+ "rewards/accuracies": 0.5562499761581421,
258
+ "rewards/chosen": 0.05722243711352348,
259
+ "rewards/margins": 0.008445021696388721,
260
+ "rewards/rejected": 0.04877741262316704,
261
+ "step": 170
262
+ },
263
+ {
264
+ "epoch": 0.94,
265
+ "learning_rate": 5.087733228106517e-08,
266
+ "logits/chosen": -0.19862544536590576,
267
+ "logits/rejected": -0.20505478978157043,
268
+ "logps/chosen": -2066.720703125,
269
+ "logps/rejected": -1937.8511962890625,
270
+ "loss": 0.0625,
271
+ "rewards/accuracies": 0.44999998807907104,
272
+ "rewards/chosen": 0.052448295056819916,
273
+ "rewards/margins": 0.0056441002525389194,
274
+ "rewards/rejected": 0.04680419713258743,
275
+ "step": 180
276
+ },
277
+ {
278
+ "epoch": 0.99,
279
+ "learning_rate": 4.2189591669322684e-10,
280
+ "logits/chosen": -0.21386022865772247,
281
+ "logits/rejected": -0.2078590840101242,
282
+ "logps/chosen": -1902.8060302734375,
283
+ "logps/rejected": -1850.3919677734375,
284
+ "loss": 0.0497,
285
+ "rewards/accuracies": 0.518750011920929,
286
+ "rewards/chosen": 0.0482863187789917,
287
+ "rewards/margins": 0.003552659647539258,
288
+ "rewards/rejected": 0.04473365843296051,
289
+ "step": 190
290
+ },
291
+ {
292
+ "epoch": 1.0,
293
+ "step": 191,
294
+ "total_flos": 0.0,
295
+ "train_loss": 0.054820564112270063,
296
+ "train_runtime": 1654.1929,
297
+ "train_samples_per_second": 1.847,
298
+ "train_steps_per_second": 0.115
299
+ }
300
+ ],
301
+ "logging_steps": 10,
302
+ "max_steps": 191,
303
+ "num_input_tokens_seen": 0,
304
+ "num_train_epochs": 1,
305
+ "save_steps": 100,
306
+ "total_flos": 0.0,
307
+ "train_batch_size": 4,
308
+ "trial_name": null,
309
+ "trial_params": null
310
+ }