Hugging Face's logo

royallab
/
LimaRP-ShareGPT-13b-qloras

PEFT
Not-For-All-Audiences
Model card Files Community
2
kingbri commited on
Commit
c2f1218
·
1 Parent(s): 85aac7e

Add multi-prompt onechar

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. multi-prompt/onechar/README.md +21 -0
  2. multi-prompt/onechar/adapter_config.json +26 -0
  3. multi-prompt/onechar/adapter_model.bin +3 -0
  4. multi-prompt/onechar/added_tokens.json +3 -0
  5. multi-prompt/onechar/checkpoint-100/README.md +21 -0
  6. multi-prompt/onechar/checkpoint-100/adapter_config.json +26 -0
  7. multi-prompt/onechar/checkpoint-100/adapter_model.bin +3 -0
  8. multi-prompt/onechar/checkpoint-100/adapter_model/README.md +21 -0
  9. multi-prompt/onechar/checkpoint-100/adapter_model/adapter_config.json +26 -0
  10. multi-prompt/onechar/checkpoint-100/adapter_model/adapter_model.bin +3 -0
  11. multi-prompt/onechar/checkpoint-100/optimizer.pt +3 -0
  12. multi-prompt/onechar/checkpoint-100/rng_state_0.pth +3 -0
  13. multi-prompt/onechar/checkpoint-100/rng_state_1.pth +3 -0
  14. multi-prompt/onechar/checkpoint-100/rng_state_2.pth +3 -0
  15. multi-prompt/onechar/checkpoint-100/rng_state_3.pth +3 -0
  16. multi-prompt/onechar/checkpoint-100/rng_state_4.pth +3 -0
  17. multi-prompt/onechar/checkpoint-100/rng_state_5.pth +3 -0
  18. multi-prompt/onechar/checkpoint-100/rng_state_6.pth +3 -0
  19. multi-prompt/onechar/checkpoint-100/rng_state_7.pth +3 -0
  20. multi-prompt/onechar/checkpoint-100/scheduler.pt +3 -0
  21. multi-prompt/onechar/checkpoint-100/trainer_state.json +659 -0
  22. multi-prompt/onechar/checkpoint-100/training_args.bin +3 -0
  23. multi-prompt/onechar/checkpoint-150/README.md +21 -0
  24. multi-prompt/onechar/checkpoint-150/adapter_config.json +26 -0
  25. multi-prompt/onechar/checkpoint-150/adapter_model.bin +3 -0
  26. multi-prompt/onechar/checkpoint-150/adapter_model/README.md +21 -0
  27. multi-prompt/onechar/checkpoint-150/adapter_model/adapter_config.json +26 -0
  28. multi-prompt/onechar/checkpoint-150/adapter_model/adapter_model.bin +3 -0
  29. multi-prompt/onechar/checkpoint-150/optimizer.pt +3 -0
  30. multi-prompt/onechar/checkpoint-150/rng_state_0.pth +3 -0
  31. multi-prompt/onechar/checkpoint-150/rng_state_1.pth +3 -0
  32. multi-prompt/onechar/checkpoint-150/rng_state_2.pth +3 -0
  33. multi-prompt/onechar/checkpoint-150/rng_state_3.pth +3 -0
  34. multi-prompt/onechar/checkpoint-150/rng_state_4.pth +3 -0
  35. multi-prompt/onechar/checkpoint-150/rng_state_5.pth +3 -0
  36. multi-prompt/onechar/checkpoint-150/rng_state_6.pth +3 -0
  37. multi-prompt/onechar/checkpoint-150/rng_state_7.pth +3 -0
  38. multi-prompt/onechar/checkpoint-150/scheduler.pt +3 -0
  39. multi-prompt/onechar/checkpoint-150/trainer_state.json +975 -0
  40. multi-prompt/onechar/checkpoint-150/training_args.bin +3 -0
  41. multi-prompt/onechar/checkpoint-200/README.md +21 -0
  42. multi-prompt/onechar/checkpoint-200/adapter_config.json +26 -0
  43. multi-prompt/onechar/checkpoint-200/adapter_model.bin +3 -0
  44. multi-prompt/onechar/checkpoint-200/adapter_model/README.md +21 -0
  45. multi-prompt/onechar/checkpoint-200/adapter_model/adapter_config.json +26 -0
  46. multi-prompt/onechar/checkpoint-200/adapter_model/adapter_model.bin +3 -0
  47. multi-prompt/onechar/checkpoint-200/optimizer.pt +3 -0
  48. multi-prompt/onechar/checkpoint-200/rng_state_0.pth +3 -0
  49. multi-prompt/onechar/checkpoint-200/rng_state_1.pth +3 -0
  50. multi-prompt/onechar/checkpoint-200/rng_state_2.pth +3 -0
multi-prompt/onechar/README.md ADDED
@@ -0,0 +1,21 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ library_name: peft
3
+ ---
4
+ ## Training procedure
5
+
6
+
7
+ The following `bitsandbytes` quantization config was used during training:
8
+ - quant_method: bitsandbytes
9
+ - load_in_8bit: False
10
+ - load_in_4bit: True
11
+ - llm_int8_threshold: 6.0
12
+ - llm_int8_skip_modules: None
13
+ - llm_int8_enable_fp32_cpu_offload: False
14
+ - llm_int8_has_fp16_weight: False
15
+ - bnb_4bit_quant_type: nf4
16
+ - bnb_4bit_use_double_quant: True
17
+ - bnb_4bit_compute_dtype: bfloat16
18
+ ### Framework versions
19
+
20
+
21
+ - PEFT 0.6.0.dev0
multi-prompt/onechar/adapter_config.json ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "auto_mapping": null,
3
+ "base_model_name_or_path": "meta-llama/Llama-2-13b-hf",
4
+ "bias": "none",
5
+ "fan_in_fan_out": null,
6
+ "inference_mode": true,
7
+ "init_lora_weights": true,
8
+ "layers_pattern": null,
9
+ "layers_to_transform": null,
10
+ "lora_alpha": 16,
11
+ "lora_dropout": 0.01,
12
+ "modules_to_save": null,
13
+ "peft_type": "LORA",
14
+ "r": 8,
15
+ "revision": null,
16
+ "target_modules": [
17
+ "down_proj",
18
+ "up_proj",
19
+ "k_proj",
20
+ "q_proj",
21
+ "o_proj",
22
+ "v_proj",
23
+ "gate_proj"
24
+ ],
25
+ "task_type": "CAUSAL_LM"
26
+ }
multi-prompt/onechar/adapter_model.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f266c8f28aeb4146c465ef7004873002f6e6c034944a4db8f17302ce67c6a862
3
+ size 125374989
multi-prompt/onechar/added_tokens.json ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ {
2
+ "[PAD]": 32000
3
+ }
multi-prompt/onechar/checkpoint-100/README.md ADDED
@@ -0,0 +1,21 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ library_name: peft
3
+ ---
4
+ ## Training procedure
5
+
6
+
7
+ The following `bitsandbytes` quantization config was used during training:
8
+ - quant_method: bitsandbytes
9
+ - load_in_8bit: False
10
+ - load_in_4bit: True
11
+ - llm_int8_threshold: 6.0
12
+ - llm_int8_skip_modules: None
13
+ - llm_int8_enable_fp32_cpu_offload: False
14
+ - llm_int8_has_fp16_weight: False
15
+ - bnb_4bit_quant_type: nf4
16
+ - bnb_4bit_use_double_quant: True
17
+ - bnb_4bit_compute_dtype: bfloat16
18
+ ### Framework versions
19
+
20
+
21
+ - PEFT 0.6.0.dev0
multi-prompt/onechar/checkpoint-100/adapter_config.json ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "auto_mapping": null,
3
+ "base_model_name_or_path": "meta-llama/Llama-2-13b-hf",
4
+ "bias": "none",
5
+ "fan_in_fan_out": null,
6
+ "inference_mode": true,
7
+ "init_lora_weights": true,
8
+ "layers_pattern": null,
9
+ "layers_to_transform": null,
10
+ "lora_alpha": 16,
11
+ "lora_dropout": 0.01,
12
+ "modules_to_save": null,
13
+ "peft_type": "LORA",
14
+ "r": 8,
15
+ "revision": null,
16
+ "target_modules": [
17
+ "down_proj",
18
+ "up_proj",
19
+ "k_proj",
20
+ "q_proj",
21
+ "o_proj",
22
+ "v_proj",
23
+ "gate_proj"
24
+ ],
25
+ "task_type": "CAUSAL_LM"
26
+ }
multi-prompt/onechar/checkpoint-100/adapter_model.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d1af7c00d9a0e6b2273252442fb5b28c4fda27d30155fc3a45b243f10710e64e
3
+ size 125374989
multi-prompt/onechar/checkpoint-100/adapter_model/README.md ADDED
@@ -0,0 +1,21 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ library_name: peft
3
+ ---
4
+ ## Training procedure
5
+
6
+
7
+ The following `bitsandbytes` quantization config was used during training:
8
+ - quant_method: bitsandbytes
9
+ - load_in_8bit: False
10
+ - load_in_4bit: True
11
+ - llm_int8_threshold: 6.0
12
+ - llm_int8_skip_modules: None
13
+ - llm_int8_enable_fp32_cpu_offload: False
14
+ - llm_int8_has_fp16_weight: False
15
+ - bnb_4bit_quant_type: nf4
16
+ - bnb_4bit_use_double_quant: True
17
+ - bnb_4bit_compute_dtype: bfloat16
18
+ ### Framework versions
19
+
20
+
21
+ - PEFT 0.6.0.dev0
multi-prompt/onechar/checkpoint-100/adapter_model/adapter_config.json ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "auto_mapping": null,
3
+ "base_model_name_or_path": "meta-llama/Llama-2-13b-hf",
4
+ "bias": "none",
5
+ "fan_in_fan_out": null,
6
+ "inference_mode": true,
7
+ "init_lora_weights": true,
8
+ "layers_pattern": null,
9
+ "layers_to_transform": null,
10
+ "lora_alpha": 16,
11
+ "lora_dropout": 0.01,
12
+ "modules_to_save": null,
13
+ "peft_type": "LORA",
14
+ "r": 8,
15
+ "revision": null,
16
+ "target_modules": [
17
+ "down_proj",
18
+ "up_proj",
19
+ "k_proj",
20
+ "q_proj",
21
+ "o_proj",
22
+ "v_proj",
23
+ "gate_proj"
24
+ ],
25
+ "task_type": "CAUSAL_LM"
26
+ }
multi-prompt/onechar/checkpoint-100/adapter_model/adapter_model.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d1af7c00d9a0e6b2273252442fb5b28c4fda27d30155fc3a45b243f10710e64e
3
+ size 125374989
multi-prompt/onechar/checkpoint-100/optimizer.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:53efac46328a269479ab1d4b76210aa6e5cbeb00730ffd8b0465e62803e2d1e8
3
+ size 250681597
multi-prompt/onechar/checkpoint-100/rng_state_0.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ffac017074dc1743416b93d99259123de8bccd7eef35b7714d90c3ff554892e6
3
+ size 21687
multi-prompt/onechar/checkpoint-100/rng_state_1.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:927e0cc9e535e309ce8f950c18cbb24e9ff4ce11f775e216fea35f08db6749bb
3
+ size 21687
multi-prompt/onechar/checkpoint-100/rng_state_2.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3b04f5328a9e61e03750a85f07aeb8175eda82723ff1b161d1d92242b3f11b88
3
+ size 21687
multi-prompt/onechar/checkpoint-100/rng_state_3.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2196d11b98dce47062253ced048888cfcc6e240b6b982ca31ed9a3c176c65f0c
3
+ size 21687
multi-prompt/onechar/checkpoint-100/rng_state_4.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:692cbdc0c723117d42d247d2bbc9b53db924c46288b3071feb3d078e1da9f0bb
3
+ size 21687
multi-prompt/onechar/checkpoint-100/rng_state_5.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4c2484a26724d353031205b8a8cf5c4a591784b893dff3bebf515dc4c611a2de
3
+ size 21687
multi-prompt/onechar/checkpoint-100/rng_state_6.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:892311743a14ba0c3ac8aade82c0dbe84b33bc30737ef2d5af1c0d17ebec0769
3
+ size 21687
multi-prompt/onechar/checkpoint-100/rng_state_7.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:59d4826d969f40c21fb5971719a30b577c16e685840e162d55bf0ac7dac2850e
3
+ size 21687
multi-prompt/onechar/checkpoint-100/scheduler.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5c741f9566ddbc689206be27d7fd96297c367c06c1ae6caa0c1917264131a0d1
3
+ size 627
multi-prompt/onechar/checkpoint-100/trainer_state.json ADDED
@@ -0,0 +1,659 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_metric": null,
3
+ "best_model_checkpoint": null,
4
+ "epoch": 0.9615384615384616,
5
+ "eval_steps": 20,
6
+ "global_step": 100,
7
+ "is_hyper_param_search": false,
8
+ "is_local_process_zero": true,
9
+ "is_world_process_zero": true,
10
+ "log_history": [
11
+ {
12
+ "epoch": 0.01,
13
+ "learning_rate": 6.5e-06,
14
+ "loss": 1.977,
15
+ "step": 1
16
+ },
17
+ {
18
+ "epoch": 0.02,
19
+ "learning_rate": 1.3e-05,
20
+ "loss": 2.1076,
21
+ "step": 2
22
+ },
23
+ {
24
+ "epoch": 0.03,
25
+ "learning_rate": 1.9499999999999996e-05,
26
+ "loss": 1.9238,
27
+ "step": 3
28
+ },
29
+ {
30
+ "epoch": 0.04,
31
+ "learning_rate": 2.6e-05,
32
+ "loss": 2.2224,
33
+ "step": 4
34
+ },
35
+ {
36
+ "epoch": 0.05,
37
+ "learning_rate": 3.25e-05,
38
+ "loss": 2.198,
39
+ "step": 5
40
+ },
41
+ {
42
+ "epoch": 0.06,
43
+ "learning_rate": 3.899999999999999e-05,
44
+ "loss": 2.1477,
45
+ "step": 6
46
+ },
47
+ {
48
+ "epoch": 0.07,
49
+ "learning_rate": 4.5499999999999995e-05,
50
+ "loss": 2.0655,
51
+ "step": 7
52
+ },
53
+ {
54
+ "epoch": 0.08,
55
+ "learning_rate": 5.2e-05,
56
+ "loss": 2.1448,
57
+ "step": 8
58
+ },
59
+ {
60
+ "epoch": 0.09,
61
+ "learning_rate": 5.85e-05,
62
+ "loss": 2.1936,
63
+ "step": 9
64
+ },
65
+ {
66
+ "epoch": 0.1,
67
+ "learning_rate": 6.5e-05,
68
+ "loss": 2.201,
69
+ "step": 10
70
+ },
71
+ {
72
+ "epoch": 0.11,
73
+ "learning_rate": 6.499590914940094e-05,
74
+ "loss": 2.169,
75
+ "step": 11
76
+ },
77
+ {
78
+ "epoch": 0.12,
79
+ "learning_rate": 6.49836376274535e-05,
80
+ "loss": 2.0327,
81
+ "step": 12
82
+ },
83
+ {
84
+ "epoch": 0.12,
85
+ "learning_rate": 6.496318852344776e-05,
86
+ "loss": 2.2921,
87
+ "step": 13
88
+ },
89
+ {
90
+ "epoch": 0.13,
91
+ "learning_rate": 6.493456698533623e-05,
92
+ "loss": 2.0172,
93
+ "step": 14
94
+ },
95
+ {
96
+ "epoch": 0.14,
97
+ "learning_rate": 6.489778021843813e-05,
98
+ "loss": 2.232,
99
+ "step": 15
100
+ },
101
+ {
102
+ "epoch": 0.15,
103
+ "learning_rate": 6.485283748362525e-05,
104
+ "loss": 1.9163,
105
+ "step": 16
106
+ },
107
+ {
108
+ "epoch": 0.16,
109
+ "learning_rate": 6.479975009499076e-05,
110
+ "loss": 2.0446,
111
+ "step": 17
112
+ },
113
+ {
114
+ "epoch": 0.17,
115
+ "learning_rate": 6.473853141700085e-05,
116
+ "loss": 2.005,
117
+ "step": 18
118
+ },
119
+ {
120
+ "epoch": 0.18,
121
+ "learning_rate": 6.466919686113032e-05,
122
+ "loss": 2.1004,
123
+ "step": 19
124
+ },
125
+ {
126
+ "epoch": 0.19,
127
+ "learning_rate": 6.45917638819828e-05,
128
+ "loss": 2.308,
129
+ "step": 20
130
+ },
131
+ {
132
+ "epoch": 0.19,
133
+ "eval_loss": 2.1830556392669678,
134
+ "eval_runtime": 2.6888,
135
+ "eval_samples_per_second": 4.091,
136
+ "eval_steps_per_second": 0.744,
137
+ "step": 20
138
+ },
139
+ {
140
+ "epoch": 0.2,
141
+ "learning_rate": 6.450625197289675e-05,
142
+ "loss": 2.119,
143
+ "step": 21
144
+ },
145
+ {
146
+ "epoch": 0.21,
147
+ "learning_rate": 6.441268266103796e-05,
148
+ "loss": 2.1047,
149
+ "step": 22
150
+ },
151
+ {
152
+ "epoch": 0.22,
153
+ "learning_rate": 6.43110795019803e-05,
154
+ "loss": 2.0454,
155
+ "step": 23
156
+ },
157
+ {
158
+ "epoch": 0.23,
159
+ "learning_rate": 6.420146807377572e-05,
160
+ "loss": 2.0439,
161
+ "step": 24
162
+ },
163
+ {
164
+ "epoch": 0.24,
165
+ "learning_rate": 6.408387597051511e-05,
166
+ "loss": 2.0833,
167
+ "step": 25
168
+ },
169
+ {
170
+ "epoch": 0.25,
171
+ "learning_rate": 6.395833279538157e-05,
172
+ "loss": 1.9844,
173
+ "step": 26
174
+ },
175
+ {
176
+ "epoch": 0.26,
177
+ "learning_rate": 6.382487015319812e-05,
178
+ "loss": 2.1317,
179
+ "step": 27
180
+ },
181
+ {
182
+ "epoch": 0.27,
183
+ "learning_rate": 6.368352164247117e-05,
184
+ "loss": 2.2273,
185
+ "step": 28
186
+ },
187
+ {
188
+ "epoch": 0.28,
189
+ "learning_rate": 6.35343228469324e-05,
190
+ "loss": 2.1833,
191
+ "step": 29
192
+ },
193
+ {
194
+ "epoch": 0.29,
195
+ "learning_rate": 6.337731132658071e-05,
196
+ "loss": 2.0888,
197
+ "step": 30
198
+ },
199
+ {
200
+ "epoch": 0.3,
201
+ "learning_rate": 6.321252660822672e-05,
202
+ "loss": 2.0598,
203
+ "step": 31
204
+ },
205
+ {
206
+ "epoch": 0.31,
207
+ "learning_rate": 6.304001017554202e-05,
208
+ "loss": 2.0894,
209
+ "step": 32
210
+ },
211
+ {
212
+ "epoch": 0.32,
213
+ "learning_rate": 6.285980545861596e-05,
214
+ "loss": 2.0645,
215
+ "step": 33
216
+ },
217
+ {
218
+ "epoch": 0.33,
219
+ "learning_rate": 6.267195782302235e-05,
220
+ "loss": 2.123,
221
+ "step": 34
222
+ },
223
+ {
224
+ "epoch": 0.34,
225
+ "learning_rate": 6.247651455839889e-05,
226
+ "loss": 2.1683,
227
+ "step": 35
228
+ },
229
+ {
230
+ "epoch": 0.35,
231
+ "learning_rate": 6.227352486654226e-05,
232
+ "loss": 2.2418,
233
+ "step": 36
234
+ },
235
+ {
236
+ "epoch": 0.36,
237
+ "learning_rate": 6.206303984902185e-05,
238
+ "loss": 2.0361,
239
+ "step": 37
240
+ },
241
+ {
242
+ "epoch": 0.37,
243
+ "learning_rate": 6.184511249431519e-05,
244
+ "loss": 1.9246,
245
+ "step": 38
246
+ },
247
+ {
248
+ "epoch": 0.38,
249
+ "learning_rate": 6.16197976644684e-05,
250
+ "loss": 1.9607,
251
+ "step": 39
252
+ },
253
+ {
254
+ "epoch": 0.38,
255
+ "learning_rate": 6.138715208128501e-05,
256
+ "loss": 2.1105,
257
+ "step": 40
258
+ },
259
+ {
260
+ "epoch": 0.38,
261
+ "eval_loss": 2.1629483699798584,
262
+ "eval_runtime": 2.6976,
263
+ "eval_samples_per_second": 4.078,
264
+ "eval_steps_per_second": 0.741,
265
+ "step": 40
266
+ },
267
+ {
268
+ "epoch": 0.39,
269
+ "learning_rate": 6.114723431204641e-05,
270
+ "loss": 2.1141,
271
+ "step": 41
272
+ },
273
+ {
274
+ "epoch": 0.4,
275
+ "learning_rate": 6.090010475476801e-05,
276
+ "loss": 2.1362,
277
+ "step": 42
278
+ },
279
+ {
280
+ "epoch": 0.41,
281
+ "learning_rate": 6.0645825622994254e-05,
282
+ "loss": 1.9861,
283
+ "step": 43
284
+ },
285
+ {
286
+ "epoch": 0.42,
287
+ "learning_rate": 6.038446093013675e-05,
288
+ "loss": 2.0427,
289
+ "step": 44
290
+ },
291
+ {
292
+ "epoch": 0.43,
293
+ "learning_rate": 6.011607647335922e-05,
294
+ "loss": 2.0936,
295
+ "step": 45
296
+ },
297
+ {
298
+ "epoch": 0.44,
299
+ "learning_rate": 5.9840739817013386e-05,
300
+ "loss": 2.1971,
301
+ "step": 46
302
+ },
303
+ {
304
+ "epoch": 0.45,
305
+ "learning_rate": 5.955852027563006e-05,
306
+ "loss": 2.033,
307
+ "step": 47
308
+ },
309
+ {
310
+ "epoch": 0.46,
311
+ "learning_rate": 5.9269488896469564e-05,
312
+ "loss": 2.1994,
313
+ "step": 48
314
+ },
315
+ {
316
+ "epoch": 0.47,
317
+ "learning_rate": 5.897371844163591e-05,
318
+ "loss": 2.0339,
319
+ "step": 49
320
+ },
321
+ {
322
+ "epoch": 0.48,
323
+ "learning_rate": 5.86712833697594e-05,
324
+ "loss": 2.0523,
325
+ "step": 50
326
+ },
327
+ {
328
+ "epoch": 0.49,
329
+ "learning_rate": 5.8362259817252044e-05,
330
+ "loss": 2.0054,
331
+ "step": 51
332
+ },
333
+ {
334
+ "epoch": 0.5,
335
+ "learning_rate": 5.8046725579140586e-05,
336
+ "loss": 2.0617,
337
+ "step": 52
338
+ },
339
+ {
340
+ "epoch": 0.51,
341
+ "learning_rate": 5.772476008948209e-05,
342
+ "loss": 2.0812,
343
+ "step": 53
344
+ },
345
+ {
346
+ "epoch": 0.52,
347
+ "learning_rate": 5.739644440136678e-05,
348
+ "loss": 1.993,
349
+ "step": 54
350
+ },
351
+ {
352
+ "epoch": 0.53,
353
+ "learning_rate": 5.706186116651339e-05,
354
+ "loss": 2.0843,
355
+ "step": 55
356
+ },
357
+ {
358
+ "epoch": 0.54,
359
+ "learning_rate": 5.6721094614462025e-05,
360
+ "loss": 2.2769,
361
+ "step": 56
362
+ },
363
+ {
364
+ "epoch": 0.55,
365
+ "learning_rate": 5.637423053136982e-05,
366
+ "loss": 2.0877,
367
+ "step": 57
368
+ },
369
+ {
370
+ "epoch": 0.56,
371
+ "learning_rate": 5.6021356238414775e-05,
372
+ "loss": 1.995,
373
+ "step": 58
374
+ },
375
+ {
376
+ "epoch": 0.57,
377
+ "learning_rate": 5.5662560569813045e-05,
378
+ "loss": 2.0595,
379
+ "step": 59
380
+ },
381
+ {
382
+ "epoch": 0.58,
383
+ "learning_rate": 5.5297933850455436e-05,
384
+ "loss": 2.0496,
385
+ "step": 60
386
+ },
387
+ {
388
+ "epoch": 0.58,
389
+ "eval_loss": 2.146815299987793,
390
+ "eval_runtime": 2.6998,
391
+ "eval_samples_per_second": 4.074,
392
+ "eval_steps_per_second": 0.741,
393
+ "step": 60
394
+ },
395
+ {
396
+ "epoch": 0.59,
397
+ "learning_rate": 5.492756787316864e-05,
398
+ "loss": 2.0805,
399
+ "step": 61
400
+ },
401
+ {
402
+ "epoch": 0.6,
403
+ "learning_rate": 5.455155587560679e-05,
404
+ "loss": 2.055,
405
+ "step": 62
406
+ },
407
+ {
408
+ "epoch": 0.61,
409
+ "learning_rate": 5.416999251677947e-05,
410
+ "loss": 2.1191,
411
+ "step": 63
412
+ },
413
+ {
414
+ "epoch": 0.62,
415
+ "learning_rate": 5.3782973853221766e-05,
416
+ "loss": 2.0373,
417
+ "step": 64
418
+ },
419
+ {
420
+ "epoch": 0.62,
421
+ "learning_rate": 5.339059731481252e-05,
422
+ "loss": 1.9465,
423
+ "step": 65
424
+ },
425
+ {
426
+ "epoch": 0.63,
427
+ "learning_rate": 5.2992961680246975e-05,
428
+ "loss": 2.0444,
429
+ "step": 66
430
+ },
431
+ {
432
+ "epoch": 0.64,
433
+ "learning_rate": 5.259016705216966e-05,
434
+ "loss": 2.1411,
435
+ "step": 67
436
+ },
437
+ {
438
+ "epoch": 0.65,
439
+ "learning_rate": 5.218231483197416e-05,
440
+ "loss": 2.0416,
441
+ "step": 68
442
+ },
443
+ {
444
+ "epoch": 0.66,
445
+ "learning_rate": 5.176950769427582e-05,
446
+ "loss": 1.9937,
447
+ "step": 69
448
+ },
449
+ {
450
+ "epoch": 0.67,
451
+ "learning_rate": 5.135184956106394e-05,
452
+ "loss": 2.0153,
453
+ "step": 70
454
+ },
455
+ {
456
+ "epoch": 0.68,
457
+ "learning_rate": 5.092944557554004e-05,
458
+ "loss": 1.9772,
459
+ "step": 71
460
+ },
461
+ {
462
+ "epoch": 0.69,
463
+ "learning_rate": 5.0502402075648585e-05,
464
+ "loss": 2.0616,
465
+ "step": 72
466
+ },
467
+ {
468
+ "epoch": 0.7,
469
+ "learning_rate": 5.007082656730692e-05,
470
+ "loss": 2.0015,
471
+ "step": 73
472
+ },
473
+ {
474
+ "epoch": 0.71,
475
+ "learning_rate": 4.963482769734132e-05,
476
+ "loss": 1.951,
477
+ "step": 74
478
+ },
479
+ {
480
+ "epoch": 0.72,
481
+ "learning_rate": 4.9194515226135705e-05,
482
+ "loss": 2.0443,
483
+ "step": 75
484
+ },
485
+ {
486
+ "epoch": 0.73,
487
+ "learning_rate": 4.874999999999999e-05,
488
+ "loss": 2.0238,
489
+ "step": 76
490
+ },
491
+ {
492
+ "epoch": 0.74,
493
+ "learning_rate": 4.830139392326523e-05,
494
+ "loss": 1.9664,
495
+ "step": 77
496
+ },
497
+ {
498
+ "epoch": 0.75,
499
+ "learning_rate": 4.784880993011219e-05,
500
+ "loss": 2.0828,
501
+ "step": 78
502
+ },
503
+ {
504
+ "epoch": 0.76,
505
+ "learning_rate": 4.739236195614084e-05,
506
+ "loss": 2.0008,
507
+ "step": 79
508
+ },
509
+ {
510
+ "epoch": 0.77,
511
+ "learning_rate": 4.6932164909687655e-05,
512
+ "loss": 2.0787,
513
+ "step": 80
514
+ },
515
+ {
516
+ "epoch": 0.77,
517
+ "eval_loss": 2.13932204246521,
518
+ "eval_runtime": 2.7011,
519
+ "eval_samples_per_second": 4.072,
520
+ "eval_steps_per_second": 0.74,
521
+ "step": 80
522
+ },
523
+ {
524
+ "epoch": 0.78,
525
+ "learning_rate": 4.646833464289808e-05,
526
+ "loss": 2.0961,
527
+ "step": 81
528
+ },
529
+ {
530
+ "epoch": 0.79,
531
+ "learning_rate": 4.600098792256131e-05,
532
+ "loss": 1.986,
533
+ "step": 82
534
+ },
535
+ {
536
+ "epoch": 0.8,
537
+ "learning_rate": 4.5530242400714944e-05,
538
+ "loss": 2.1558,
539
+ "step": 83
540
+ },
541
+ {
542
+ "epoch": 0.81,
543
+ "learning_rate": 4.505621658502667e-05,
544
+ "loss": 1.9636,
545
+ "step": 84
546
+ },
547
+ {
548
+ "epoch": 0.82,
549
+ "learning_rate": 4.457902980896064e-05,
550
+ "loss": 2.0659,
551
+ "step": 85
552
+ },
553
+ {
554
+ "epoch": 0.83,
555
+ "learning_rate": 4.409880220173583e-05,
556
+ "loss": 2.0239,
557
+ "step": 86
558
+ },
559
+ {
560
+ "epoch": 0.84,
561
+ "learning_rate": 4.3615654658084234e-05,
562
+ "loss": 2.0932,
563
+ "step": 87
564
+ },
565
+ {
566
+ "epoch": 0.85,
567
+ "learning_rate": 4.3129708807816206e-05,
568
+ "loss": 1.8754,
569
+ "step": 88
570
+ },
571
+ {
572
+ "epoch": 0.86,
573
+ "learning_rate": 4.264108698520082e-05,
574
+ "loss": 2.0868,
575
+ "step": 89
576
+ },
577
+ {
578
+ "epoch": 0.87,
579
+ "learning_rate": 4.2149912198168935e-05,
580
+ "loss": 2.0433,
581
+ "step": 90
582
+ },
583
+ {
584
+ "epoch": 0.88,
585
+ "learning_rate": 4.1656308097346466e-05,
586
+ "loss": 2.1271,
587
+ "step": 91
588
+ },
589
+ {
590
+ "epoch": 0.88,
591
+ "learning_rate": 4.116039894492613e-05,
592
+ "loss": 2.1713,
593
+ "step": 92
594
+ },
595
+ {
596
+ "epoch": 0.89,
597
+ "learning_rate": 4.066230958338507e-05,
598
+ "loss": 2.021,
599
+ "step": 93
600
+ },
601
+ {
602
+ "epoch": 0.9,
603
+ "learning_rate": 4.0162165404056384e-05,
604
+ "loss": 2.021,
605
+ "step": 94
606
+ },
607
+ {
608
+ "epoch": 0.91,
609
+ "learning_rate": 3.9660092315562566e-05,
610
+ "loss": 2.0075,
611
+ "step": 95
612
+ },
613
+ {
614
+ "epoch": 0.92,
615
+ "learning_rate": 3.91562167121187e-05,
616
+ "loss": 2.0444,
617
+ "step": 96
618
+ },
619
+ {
620
+ "epoch": 0.93,
621
+ "learning_rate": 3.8650665441713327e-05,
622
+ "loss": 1.9967,
623
+ "step": 97
624
+ },
625
+ {
626
+ "epoch": 0.94,
627
+ "learning_rate": 3.814356577417524e-05,
628
+ "loss": 2.2019,
629
+ "step": 98
630
+ },
631
+ {
632
+ "epoch": 0.95,
633
+ "learning_rate": 3.763504536913387e-05,
634
+ "loss": 1.936,
635
+ "step": 99
636
+ },
637
+ {
638
+ "epoch": 0.96,
639
+ "learning_rate": 3.712523224388177e-05,
640
+ "loss": 1.9272,
641
+ "step": 100
642
+ },
643
+ {
644
+ "epoch": 0.96,
645
+ "eval_loss": 2.134587287902832,
646
+ "eval_runtime": 2.7065,
647
+ "eval_samples_per_second": 4.064,
648
+ "eval_steps_per_second": 0.739,
649
+ "step": 100
650
+ }
651
+ ],
652
+ "logging_steps": 1,
653
+ "max_steps": 208,
654
+ "num_train_epochs": 2,
655
+ "save_steps": 50,
656
+ "total_flos": 8.151676451238707e+16,
657
+ "trial_name": null,
658
+ "trial_params": null
659
+ }
multi-prompt/onechar/checkpoint-100/training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:deca3b521b13630aa127a677d66c86cecd19f6ba8eb8e593ac9a0d31193246e9
3
+ size 4475
multi-prompt/onechar/checkpoint-150/README.md ADDED
@@ -0,0 +1,21 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ library_name: peft
3
+ ---
4
+ ## Training procedure
5
+
6
+
7
+ The following `bitsandbytes` quantization config was used during training:
8
+ - quant_method: bitsandbytes
9
+ - load_in_8bit: False
10
+ - load_in_4bit: True
11
+ - llm_int8_threshold: 6.0
12
+ - llm_int8_skip_modules: None
13
+ - llm_int8_enable_fp32_cpu_offload: False
14
+ - llm_int8_has_fp16_weight: False
15
+ - bnb_4bit_quant_type: nf4
16
+ - bnb_4bit_use_double_quant: True
17
+ - bnb_4bit_compute_dtype: bfloat16
18
+ ### Framework versions
19
+
20
+
21
+ - PEFT 0.6.0.dev0
multi-prompt/onechar/checkpoint-150/adapter_config.json ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "auto_mapping": null,
3
+ "base_model_name_or_path": "meta-llama/Llama-2-13b-hf",
4
+ "bias": "none",
5
+ "fan_in_fan_out": null,
6
+ "inference_mode": true,
7
+ "init_lora_weights": true,
8
+ "layers_pattern": null,
9
+ "layers_to_transform": null,
10
+ "lora_alpha": 16,
11
+ "lora_dropout": 0.01,
12
+ "modules_to_save": null,
13
+ "peft_type": "LORA",
14
+ "r": 8,
15
+ "revision": null,
16
+ "target_modules": [
17
+ "down_proj",
18
+ "up_proj",
19
+ "k_proj",
20
+ "q_proj",
21
+ "o_proj",
22
+ "v_proj",
23
+ "gate_proj"
24
+ ],
25
+ "task_type": "CAUSAL_LM"
26
+ }
multi-prompt/onechar/checkpoint-150/adapter_model.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a88ca139e2a7d5321493c43410f7c8874dcf5dd158c7f2267189238cba5b2e8e
3
+ size 125374989
multi-prompt/onechar/checkpoint-150/adapter_model/README.md ADDED
@@ -0,0 +1,21 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ library_name: peft
3
+ ---
4
+ ## Training procedure
5
+
6
+
7
+ The following `bitsandbytes` quantization config was used during training:
8
+ - quant_method: bitsandbytes
9
+ - load_in_8bit: False
10
+ - load_in_4bit: True
11
+ - llm_int8_threshold: 6.0
12
+ - llm_int8_skip_modules: None
13
+ - llm_int8_enable_fp32_cpu_offload: False
14
+ - llm_int8_has_fp16_weight: False
15
+ - bnb_4bit_quant_type: nf4
16
+ - bnb_4bit_use_double_quant: True
17
+ - bnb_4bit_compute_dtype: bfloat16
18
+ ### Framework versions
19
+
20
+
21
+ - PEFT 0.6.0.dev0
multi-prompt/onechar/checkpoint-150/adapter_model/adapter_config.json ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "auto_mapping": null,
3
+ "base_model_name_or_path": "meta-llama/Llama-2-13b-hf",
4
+ "bias": "none",
5
+ "fan_in_fan_out": null,
6
+ "inference_mode": true,
7
+ "init_lora_weights": true,
8
+ "layers_pattern": null,
9
+ "layers_to_transform": null,
10
+ "lora_alpha": 16,
11
+ "lora_dropout": 0.01,
12
+ "modules_to_save": null,
13
+ "peft_type": "LORA",
14
+ "r": 8,
15
+ "revision": null,
16
+ "target_modules": [
17
+ "down_proj",
18
+ "up_proj",
19
+ "k_proj",
20
+ "q_proj",
21
+ "o_proj",
22
+ "v_proj",
23
+ "gate_proj"
24
+ ],
25
+ "task_type": "CAUSAL_LM"
26
+ }
multi-prompt/onechar/checkpoint-150/adapter_model/adapter_model.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a88ca139e2a7d5321493c43410f7c8874dcf5dd158c7f2267189238cba5b2e8e
3
+ size 125374989
multi-prompt/onechar/checkpoint-150/optimizer.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:36f6f59bf42fda7cef4a3db7d2ba9063716670a01f31f410451c3877b93b91fd
3
+ size 250681597
multi-prompt/onechar/checkpoint-150/rng_state_0.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ebe4080394353d7133f04b4673e50961f3043110ec8d1128ee2e3e296b061c86
3
+ size 21687
multi-prompt/onechar/checkpoint-150/rng_state_1.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3a4d9fd37760216bbb28070ef4d2e2e0be700bbf1ec89831778ebc6a298e5622
3
+ size 21687
multi-prompt/onechar/checkpoint-150/rng_state_2.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:302d299783dd83cc63183bbf69795215510d85b9c0b5f56b75b7989f03437344
3
+ size 21687
multi-prompt/onechar/checkpoint-150/rng_state_3.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ef7613df4d25b459724ab8e45996bbe862cf7f498a97f5f0bedd0a20e60b04a3
3
+ size 21687
multi-prompt/onechar/checkpoint-150/rng_state_4.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a09d5660be18b2300f0c6030ffd427dd486af8a98254c4f6aba3a481a043d572
3
+ size 21687
multi-prompt/onechar/checkpoint-150/rng_state_5.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:784e8a97c961f412059dba03c53f1a4b5b3e4c4d39bedf8eae0b5e0d4f41e6c3
3
+ size 21687
multi-prompt/onechar/checkpoint-150/rng_state_6.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a75028de3fd9a7ddf9a281872bb8f021c1d2901266449ea9e5c74d790d29f6db
3
+ size 21687
multi-prompt/onechar/checkpoint-150/rng_state_7.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:440b7938477873ff8f28f52682c20f692e5381cdbfe768b1f432c98615010804
3
+ size 21687
multi-prompt/onechar/checkpoint-150/scheduler.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3b570b3ad0cad17fc1d14736697e293a3b6ed0588a61240c41e6ab9ccc80f4ee
3
+ size 627
multi-prompt/onechar/checkpoint-150/trainer_state.json ADDED
@@ -0,0 +1,975 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_metric": null,
3
+ "best_model_checkpoint": null,
4
+ "epoch": 1.4423076923076923,
5
+ "eval_steps": 20,
6
+ "global_step": 150,
7
+ "is_hyper_param_search": false,
8
+ "is_local_process_zero": true,
9
+ "is_world_process_zero": true,
10
+ "log_history": [
11
+ {
12
+ "epoch": 0.01,
13
+ "learning_rate": 6.5e-06,
14
+ "loss": 1.977,
15
+ "step": 1
16
+ },
17
+ {
18
+ "epoch": 0.02,
19
+ "learning_rate": 1.3e-05,
20
+ "loss": 2.1076,
21
+ "step": 2
22
+ },
23
+ {
24
+ "epoch": 0.03,
25
+ "learning_rate": 1.9499999999999996e-05,
26
+ "loss": 1.9238,
27
+ "step": 3
28
+ },
29
+ {
30
+ "epoch": 0.04,
31
+ "learning_rate": 2.6e-05,
32
+ "loss": 2.2224,
33
+ "step": 4
34
+ },
35
+ {
36
+ "epoch": 0.05,
37
+ "learning_rate": 3.25e-05,
38
+ "loss": 2.198,
39
+ "step": 5
40
+ },
41
+ {
42
+ "epoch": 0.06,
43
+ "learning_rate": 3.899999999999999e-05,
44
+ "loss": 2.1477,
45
+ "step": 6
46
+ },
47
+ {
48
+ "epoch": 0.07,
49
+ "learning_rate": 4.5499999999999995e-05,
50
+ "loss": 2.0655,
51
+ "step": 7
52
+ },
53
+ {
54
+ "epoch": 0.08,
55
+ "learning_rate": 5.2e-05,
56
+ "loss": 2.1448,
57
+ "step": 8
58
+ },
59
+ {
60
+ "epoch": 0.09,
61
+ "learning_rate": 5.85e-05,
62
+ "loss": 2.1936,
63
+ "step": 9
64
+ },
65
+ {
66
+ "epoch": 0.1,
67
+ "learning_rate": 6.5e-05,
68
+ "loss": 2.201,
69
+ "step": 10
70
+ },
71
+ {
72
+ "epoch": 0.11,
73
+ "learning_rate": 6.499590914940094e-05,
74
+ "loss": 2.169,
75
+ "step": 11
76
+ },
77
+ {
78
+ "epoch": 0.12,
79
+ "learning_rate": 6.49836376274535e-05,
80
+ "loss": 2.0327,
81
+ "step": 12
82
+ },
83
+ {
84
+ "epoch": 0.12,
85
+ "learning_rate": 6.496318852344776e-05,
86
+ "loss": 2.2921,
87
+ "step": 13
88
+ },
89
+ {
90
+ "epoch": 0.13,
91
+ "learning_rate": 6.493456698533623e-05,
92
+ "loss": 2.0172,
93
+ "step": 14
94
+ },
95
+ {
96
+ "epoch": 0.14,
97
+ "learning_rate": 6.489778021843813e-05,
98
+ "loss": 2.232,
99
+ "step": 15
100
+ },
101
+ {
102
+ "epoch": 0.15,
103
+ "learning_rate": 6.485283748362525e-05,
104
+ "loss": 1.9163,
105
+ "step": 16
106
+ },
107
+ {
108
+ "epoch": 0.16,
109
+ "learning_rate": 6.479975009499076e-05,
110
+ "loss": 2.0446,
111
+ "step": 17
112
+ },
113
+ {
114
+ "epoch": 0.17,
115
+ "learning_rate": 6.473853141700085e-05,
116
+ "loss": 2.005,
117
+ "step": 18
118
+ },
119
+ {
120
+ "epoch": 0.18,
121
+ "learning_rate": 6.466919686113032e-05,
122
+ "loss": 2.1004,
123
+ "step": 19
124
+ },
125
+ {
126
+ "epoch": 0.19,
127
+ "learning_rate": 6.45917638819828e-05,
128
+ "loss": 2.308,
129
+ "step": 20
130
+ },
131
+ {
132
+ "epoch": 0.19,
133
+ "eval_loss": 2.1830556392669678,
134
+ "eval_runtime": 2.6888,
135
+ "eval_samples_per_second": 4.091,
136
+ "eval_steps_per_second": 0.744,
137
+ "step": 20
138
+ },
139
+ {
140
+ "epoch": 0.2,
141
+ "learning_rate": 6.450625197289675e-05,
142
+ "loss": 2.119,
143
+ "step": 21
144
+ },
145
+ {
146
+ "epoch": 0.21,
147
+ "learning_rate": 6.441268266103796e-05,
148
+ "loss": 2.1047,
149
+ "step": 22
150
+ },
151
+ {
152
+ "epoch": 0.22,
153
+ "learning_rate": 6.43110795019803e-05,
154
+ "loss": 2.0454,
155
+ "step": 23
156
+ },
157
+ {
158
+ "epoch": 0.23,
159
+ "learning_rate": 6.420146807377572e-05,
160
+ "loss": 2.0439,
161
+ "step": 24
162
+ },
163
+ {
164
+ "epoch": 0.24,
165
+ "learning_rate": 6.408387597051511e-05,
166
+ "loss": 2.0833,
167
+ "step": 25
168
+ },
169
+ {
170
+ "epoch": 0.25,
171
+ "learning_rate": 6.395833279538157e-05,
172
+ "loss": 1.9844,
173
+ "step": 26
174
+ },
175
+ {
176
+ "epoch": 0.26,
177
+ "learning_rate": 6.382487015319812e-05,
178
+ "loss": 2.1317,
179
+ "step": 27
180
+ },
181
+ {
182
+ "epoch": 0.27,
183
+ "learning_rate": 6.368352164247117e-05,
184
+ "loss": 2.2273,
185
+ "step": 28
186
+ },
187
+ {
188
+ "epoch": 0.28,
189
+ "learning_rate": 6.35343228469324e-05,
190
+ "loss": 2.1833,
191
+ "step": 29
192
+ },
193
+ {
194
+ "epoch": 0.29,
195
+ "learning_rate": 6.337731132658071e-05,
196
+ "loss": 2.0888,
197
+ "step": 30
198
+ },
199
+ {
200
+ "epoch": 0.3,
201
+ "learning_rate": 6.321252660822672e-05,
202
+ "loss": 2.0598,
203
+ "step": 31
204
+ },
205
+ {
206
+ "epoch": 0.31,
207
+ "learning_rate": 6.304001017554202e-05,
208
+ "loss": 2.0894,
209
+ "step": 32
210
+ },
211
+ {
212
+ "epoch": 0.32,
213
+ "learning_rate": 6.285980545861596e-05,
214
+ "loss": 2.0645,
215
+ "step": 33
216
+ },
217
+ {
218
+ "epoch": 0.33,
219
+ "learning_rate": 6.267195782302235e-05,
220
+ "loss": 2.123,
221
+ "step": 34
222
+ },
223
+ {
224
+ "epoch": 0.34,
225
+ "learning_rate": 6.247651455839889e-05,
226
+ "loss": 2.1683,
227
+ "step": 35
228
+ },
229
+ {
230
+ "epoch": 0.35,
231
+ "learning_rate": 6.227352486654226e-05,
232
+ "loss": 2.2418,
233
+ "step": 36
234
+ },
235
+ {
236
+ "epoch": 0.36,
237
+ "learning_rate": 6.206303984902185e-05,
238
+ "loss": 2.0361,
239
+ "step": 37
240
+ },
241
+ {
242
+ "epoch": 0.37,
243
+ "learning_rate": 6.184511249431519e-05,
244
+ "loss": 1.9246,
245
+ "step": 38
246
+ },
247
+ {
248
+ "epoch": 0.38,
249
+ "learning_rate": 6.16197976644684e-05,
250
+ "loss": 1.9607,
251
+ "step": 39
252
+ },
253
+ {
254
+ "epoch": 0.38,
255
+ "learning_rate": 6.138715208128501e-05,
256
+ "loss": 2.1105,
257
+ "step": 40
258
+ },
259
+ {
260
+ "epoch": 0.38,
261
+ "eval_loss": 2.1629483699798584,
262
+ "eval_runtime": 2.6976,
263
+ "eval_samples_per_second": 4.078,
264
+ "eval_steps_per_second": 0.741,
265
+ "step": 40
266
+ },
267
+ {
268
+ "epoch": 0.39,
269
+ "learning_rate": 6.114723431204641e-05,
270
+ "loss": 2.1141,
271
+ "step": 41
272
+ },
273
+ {
274
+ "epoch": 0.4,
275
+ "learning_rate": 6.090010475476801e-05,
276
+ "loss": 2.1362,
277
+ "step": 42
278
+ },
279
+ {
280
+ "epoch": 0.41,
281
+ "learning_rate": 6.0645825622994254e-05,
282
+ "loss": 1.9861,
283
+ "step": 43
284
+ },
285
+ {
286
+ "epoch": 0.42,
287
+ "learning_rate": 6.038446093013675e-05,
288
+ "loss": 2.0427,
289
+ "step": 44
290
+ },
291
+ {
292
+ "epoch": 0.43,
293
+ "learning_rate": 6.011607647335922e-05,
294
+ "loss": 2.0936,
295
+ "step": 45
296
+ },
297
+ {
298
+ "epoch": 0.44,
299
+ "learning_rate": 5.9840739817013386e-05,
300
+ "loss": 2.1971,
301
+ "step": 46
302
+ },
303
+ {
304
+ "epoch": 0.45,
305
+ "learning_rate": 5.955852027563006e-05,
306
+ "loss": 2.033,
307
+ "step": 47
308
+ },
309
+ {
310
+ "epoch": 0.46,
311
+ "learning_rate": 5.9269488896469564e-05,
312
+ "loss": 2.1994,
313
+ "step": 48
314
+ },
315
+ {
316
+ "epoch": 0.47,
317
+ "learning_rate": 5.897371844163591e-05,
318
+ "loss": 2.0339,
319
+ "step": 49
320
+ },
321
+ {
322
+ "epoch": 0.48,
323
+ "learning_rate": 5.86712833697594e-05,
324
+ "loss": 2.0523,
325
+ "step": 50
326
+ },
327
+ {
328
+ "epoch": 0.49,
329
+ "learning_rate": 5.8362259817252044e-05,
330
+ "loss": 2.0054,
331
+ "step": 51
332
+ },
333
+ {
334
+ "epoch": 0.5,
335
+ "learning_rate": 5.8046725579140586e-05,
336
+ "loss": 2.0617,
337
+ "step": 52
338
+ },
339
+ {
340
+ "epoch": 0.51,
341
+ "learning_rate": 5.772476008948209e-05,
342
+ "loss": 2.0812,
343
+ "step": 53
344
+ },
345
+ {
346
+ "epoch": 0.52,
347
+ "learning_rate": 5.739644440136678e-05,
348
+ "loss": 1.993,
349
+ "step": 54
350
+ },
351
+ {
352
+ "epoch": 0.53,
353
+ "learning_rate": 5.706186116651339e-05,
354
+ "loss": 2.0843,
355
+ "step": 55
356
+ },
357
+ {
358
+ "epoch": 0.54,
359
+ "learning_rate": 5.6721094614462025e-05,
360
+ "loss": 2.2769,
361
+ "step": 56
362
+ },
363
+ {
364
+ "epoch": 0.55,
365
+ "learning_rate": 5.637423053136982e-05,
366
+ "loss": 2.0877,
367
+ "step": 57
368
+ },
369
+ {
370
+ "epoch": 0.56,
371
+ "learning_rate": 5.6021356238414775e-05,
372
+ "loss": 1.995,
373
+ "step": 58
374
+ },
375
+ {
376
+ "epoch": 0.57,
377
+ "learning_rate": 5.5662560569813045e-05,
378
+ "loss": 2.0595,
379
+ "step": 59
380
+ },
381
+ {
382
+ "epoch": 0.58,
383
+ "learning_rate": 5.5297933850455436e-05,
384
+ "loss": 2.0496,
385
+ "step": 60
386
+ },
387
+ {
388
+ "epoch": 0.58,
389
+ "eval_loss": 2.146815299987793,
390
+ "eval_runtime": 2.6998,
391
+ "eval_samples_per_second": 4.074,
392
+ "eval_steps_per_second": 0.741,
393
+ "step": 60
394
+ },
395
+ {
396
+ "epoch": 0.59,
397
+ "learning_rate": 5.492756787316864e-05,
398
+ "loss": 2.0805,
399
+ "step": 61
400
+ },
401
+ {
402
+ "epoch": 0.6,
403
+ "learning_rate": 5.455155587560679e-05,
404
+ "loss": 2.055,
405
+ "step": 62
406
+ },
407
+ {
408
+ "epoch": 0.61,
409
+ "learning_rate": 5.416999251677947e-05,
410
+ "loss": 2.1191,
411
+ "step": 63
412
+ },
413
+ {
414
+ "epoch": 0.62,
415
+ "learning_rate": 5.3782973853221766e-05,
416
+ "loss": 2.0373,
417
+ "step": 64
418
+ },
419
+ {
420
+ "epoch": 0.62,
421
+ "learning_rate": 5.339059731481252e-05,
422
+ "loss": 1.9465,
423
+ "step": 65
424
+ },
425
+ {
426
+ "epoch": 0.63,
427
+ "learning_rate": 5.2992961680246975e-05,
428
+ "loss": 2.0444,
429
+ "step": 66
430
+ },
431
+ {
432
+ "epoch": 0.64,
433
+ "learning_rate": 5.259016705216966e-05,
434
+ "loss": 2.1411,
435
+ "step": 67
436
+ },
437
+ {
438
+ "epoch": 0.65,
439
+ "learning_rate": 5.218231483197416e-05,
440
+ "loss": 2.0416,
441
+ "step": 68
442
+ },
443
+ {
444
+ "epoch": 0.66,
445
+ "learning_rate": 5.176950769427582e-05,
446
+ "loss": 1.9937,
447
+ "step": 69
448
+ },
449
+ {
450
+ "epoch": 0.67,
451
+ "learning_rate": 5.135184956106394e-05,
452
+ "loss": 2.0153,
453
+ "step": 70
454
+ },
455
+ {
456
+ "epoch": 0.68,
457
+ "learning_rate": 5.092944557554004e-05,
458
+ "loss": 1.9772,
459
+ "step": 71
460
+ },
461
+ {
462
+ "epoch": 0.69,
463
+ "learning_rate": 5.0502402075648585e-05,
464
+ "loss": 2.0616,
465
+ "step": 72
466
+ },
467
+ {
468
+ "epoch": 0.7,
469
+ "learning_rate": 5.007082656730692e-05,
470
+ "loss": 2.0015,
471
+ "step": 73
472
+ },
473
+ {
474
+ "epoch": 0.71,
475
+ "learning_rate": 4.963482769734132e-05,
476
+ "loss": 1.951,
477
+ "step": 74
478
+ },
479
+ {
480
+ "epoch": 0.72,
481
+ "learning_rate": 4.9194515226135705e-05,
482
+ "loss": 2.0443,
483
+ "step": 75
484
+ },
485
+ {
486
+ "epoch": 0.73,
487
+ "learning_rate": 4.874999999999999e-05,
488
+ "loss": 2.0238,
489
+ "step": 76
490
+ },
491
+ {
492
+ "epoch": 0.74,
493
+ "learning_rate": 4.830139392326523e-05,
494
+ "loss": 1.9664,
495
+ "step": 77
496
+ },
497
+ {
498
+ "epoch": 0.75,
499
+ "learning_rate": 4.784880993011219e-05,
500
+ "loss": 2.0828,
501
+ "step": 78
502
+ },
503
+ {
504
+ "epoch": 0.76,
505
+ "learning_rate": 4.739236195614084e-05,
506
+ "loss": 2.0008,
507
+ "step": 79
508
+ },
509
+ {
510
+ "epoch": 0.77,
511
+ "learning_rate": 4.6932164909687655e-05,
512
+ "loss": 2.0787,
513
+ "step": 80
514
+ },
515
+ {
516
+ "epoch": 0.77,
517
+ "eval_loss": 2.13932204246521,
518
+ "eval_runtime": 2.7011,
519
+ "eval_samples_per_second": 4.072,
520
+ "eval_steps_per_second": 0.74,
521
+ "step": 80
522
+ },
523
+ {
524
+ "epoch": 0.78,
525
+ "learning_rate": 4.646833464289808e-05,
526
+ "loss": 2.0961,
527
+ "step": 81
528
+ },
529
+ {
530
+ "epoch": 0.79,
531
+ "learning_rate": 4.600098792256131e-05,
532
+ "loss": 1.986,
533
+ "step": 82
534
+ },
535
+ {
536
+ "epoch": 0.8,
537
+ "learning_rate": 4.5530242400714944e-05,
538
+ "loss": 2.1558,
539
+ "step": 83
540
+ },
541
+ {
542
+ "epoch": 0.81,
543
+ "learning_rate": 4.505621658502667e-05,
544
+ "loss": 1.9636,
545
+ "step": 84
546
+ },
547
+ {
548
+ "epoch": 0.82,
549
+ "learning_rate": 4.457902980896064e-05,
550
+ "loss": 2.0659,
551
+ "step": 85
552
+ },
553
+ {
554
+ "epoch": 0.83,
555
+ "learning_rate": 4.409880220173583e-05,
556
+ "loss": 2.0239,
557
+ "step": 86
558
+ },
559
+ {
560
+ "epoch": 0.84,
561
+ "learning_rate": 4.3615654658084234e-05,
562
+ "loss": 2.0932,
563
+ "step": 87
564
+ },
565
+ {
566
+ "epoch": 0.85,
567
+ "learning_rate": 4.3129708807816206e-05,
568
+ "loss": 1.8754,
569
+ "step": 88
570
+ },
571
+ {
572
+ "epoch": 0.86,
573
+ "learning_rate": 4.264108698520082e-05,
574
+ "loss": 2.0868,
575
+ "step": 89
576
+ },
577
+ {
578
+ "epoch": 0.87,
579
+ "learning_rate": 4.2149912198168935e-05,
580
+ "loss": 2.0433,
581
+ "step": 90
582
+ },
583
+ {
584
+ "epoch": 0.88,
585
+ "learning_rate": 4.1656308097346466e-05,
586
+ "loss": 2.1271,
587
+ "step": 91
588
+ },
589
+ {
590
+ "epoch": 0.88,
591
+ "learning_rate": 4.116039894492613e-05,
592
+ "loss": 2.1713,
593
+ "step": 92
594
+ },
595
+ {
596
+ "epoch": 0.89,
597
+ "learning_rate": 4.066230958338507e-05,
598
+ "loss": 2.021,
599
+ "step": 93
600
+ },
601
+ {
602
+ "epoch": 0.9,
603
+ "learning_rate": 4.0162165404056384e-05,
604
+ "loss": 2.021,
605
+ "step": 94
606
+ },
607
+ {
608
+ "epoch": 0.91,
609
+ "learning_rate": 3.9660092315562566e-05,
610
+ "loss": 2.0075,
611
+ "step": 95
612
+ },
613
+ {
614
+ "epoch": 0.92,
615
+ "learning_rate": 3.91562167121187e-05,
616
+ "loss": 2.0444,
617
+ "step": 96
618
+ },
619
+ {
620
+ "epoch": 0.93,
621
+ "learning_rate": 3.8650665441713327e-05,
622
+ "loss": 1.9967,
623
+ "step": 97
624
+ },
625
+ {
626
+ "epoch": 0.94,
627
+ "learning_rate": 3.814356577417524e-05,
628
+ "loss": 2.2019,
629
+ "step": 98
630
+ },
631
+ {
632
+ "epoch": 0.95,
633
+ "learning_rate": 3.763504536913387e-05,
634
+ "loss": 1.936,
635
+ "step": 99
636
+ },
637
+ {
638
+ "epoch": 0.96,
639
+ "learning_rate": 3.712523224388177e-05,
640
+ "loss": 1.9272,
641
+ "step": 100
642
+ },
643
+ {
644
+ "epoch": 0.96,
645
+ "eval_loss": 2.134587287902832,
646
+ "eval_runtime": 2.7065,
647
+ "eval_samples_per_second": 4.064,
648
+ "eval_steps_per_second": 0.739,
649
+ "step": 100
650
+ },
651
+ {
652
+ "epoch": 0.97,
653
+ "learning_rate": 3.661425474114685e-05,
654
+ "loss": 1.9903,
655
+ "step": 101
656
+ },
657
+ {
658
+ "epoch": 0.98,
659
+ "learning_rate": 3.610224149678286e-05,
660
+ "loss": 2.1719,
661
+ "step": 102
662
+ },
663
+ {
664
+ "epoch": 0.99,
665
+ "learning_rate": 3.558932140738593e-05,
666
+ "loss": 1.9709,
667
+ "step": 103
668
+ },
669
+ {
670
+ "epoch": 1.0,
671
+ "learning_rate": 3.507562359784562e-05,
672
+ "loss": 2.098,
673
+ "step": 104
674
+ },
675
+ {
676
+ "epoch": 1.01,
677
+ "learning_rate": 3.456127738883834e-05,
678
+ "loss": 2.21,
679
+ "step": 105
680
+ },
681
+ {
682
+ "epoch": 1.02,
683
+ "learning_rate": 3.404641226427163e-05,
684
+ "loss": 2.0854,
685
+ "step": 106
686
+ },
687
+ {
688
+ "epoch": 1.03,
689
+ "learning_rate": 3.3531157838687204e-05,
690
+ "loss": 2.0356,
691
+ "step": 107
692
+ },
693
+ {
694
+ "epoch": 1.04,
695
+ "learning_rate": 3.301564382463126e-05,
696
+ "loss": 2.0967,
697
+ "step": 108
698
+ },
699
+ {
700
+ "epoch": 1.05,
701
+ "learning_rate": 3.25e-05,
702
+ "loss": 1.9079,
703
+ "step": 109
704
+ },
705
+ {
706
+ "epoch": 1.06,
707
+ "learning_rate": 3.198435617536874e-05,
708
+ "loss": 1.9948,
709
+ "step": 110
710
+ },
711
+ {
712
+ "epoch": 1.07,
713
+ "learning_rate": 3.1468842161312804e-05,
714
+ "loss": 2.0195,
715
+ "step": 111
716
+ },
717
+ {
718
+ "epoch": 1.08,
719
+ "learning_rate": 3.095358773572837e-05,
720
+ "loss": 2.0118,
721
+ "step": 112
722
+ },
723
+ {
724
+ "epoch": 1.09,
725
+ "learning_rate": 3.043872261116165e-05,
726
+ "loss": 1.981,
727
+ "step": 113
728
+ },
729
+ {
730
+ "epoch": 1.1,
731
+ "learning_rate": 2.992437640215437e-05,
732
+ "loss": 2.0501,
733
+ "step": 114
734
+ },
735
+ {
736
+ "epoch": 1.11,
737
+ "learning_rate": 2.9410678592614068e-05,
738
+ "loss": 2.001,
739
+ "step": 115
740
+ },
741
+ {
742
+ "epoch": 1.12,
743
+ "learning_rate": 2.8897758503217142e-05,
744
+ "loss": 2.0048,
745
+ "step": 116
746
+ },
747
+ {
748
+ "epoch": 1.12,
749
+ "learning_rate": 2.838574525885314e-05,
750
+ "loss": 2.0039,
751
+ "step": 117
752
+ },
753
+ {
754
+ "epoch": 1.13,
755
+ "learning_rate": 2.7874767756118234e-05,
756
+ "loss": 2.0204,
757
+ "step": 118
758
+ },
759
+ {
760
+ "epoch": 1.14,
761
+ "learning_rate": 2.7364954630866125e-05,
762
+ "loss": 2.1493,
763
+ "step": 119
764
+ },
765
+ {
766
+ "epoch": 1.15,
767
+ "learning_rate": 2.6856434225824763e-05,
768
+ "loss": 2.0527,
769
+ "step": 120
770
+ },
771
+ {
772
+ "epoch": 1.15,
773
+ "eval_loss": 2.132925271987915,
774
+ "eval_runtime": 2.7003,
775
+ "eval_samples_per_second": 4.074,
776
+ "eval_steps_per_second": 0.741,
777
+ "step": 120
778
+ },
779
+ {
780
+ "epoch": 1.16,
781
+ "learning_rate": 2.6349334558286678e-05,
782
+ "loss": 2.0877,
783
+ "step": 121
784
+ },
785
+ {
786
+ "epoch": 1.17,
787
+ "learning_rate": 2.5843783287881306e-05,
788
+ "loss": 2.0201,
789
+ "step": 122
790
+ },
791
+ {
792
+ "epoch": 1.18,
793
+ "learning_rate": 2.5339907684437425e-05,
794
+ "loss": 2.0471,
795
+ "step": 123
796
+ },
797
+ {
798
+ "epoch": 1.19,
799
+ "learning_rate": 2.4837834595943607e-05,
800
+ "loss": 1.94,
801
+ "step": 124
802
+ },
803
+ {
804
+ "epoch": 1.2,
805
+ "learning_rate": 2.4337690416614925e-05,
806
+ "loss": 1.9542,
807
+ "step": 125
808
+ },
809
+ {
810
+ "epoch": 1.21,
811
+ "learning_rate": 2.3839601055073866e-05,
812
+ "loss": 2.0596,
813
+ "step": 126
814
+ },
815
+ {
816
+ "epoch": 1.22,
817
+ "learning_rate": 2.3343691902653532e-05,
818
+ "loss": 1.9224,
819
+ "step": 127
820
+ },
821
+ {
822
+ "epoch": 1.23,
823
+ "learning_rate": 2.2850087801831076e-05,
824
+ "loss": 2.0356,
825
+ "step": 128
826
+ },
827
+ {
828
+ "epoch": 1.24,
829
+ "learning_rate": 2.235891301479917e-05,
830
+ "loss": 1.9185,
831
+ "step": 129
832
+ },
833
+ {
834
+ "epoch": 1.25,
835
+ "learning_rate": 2.1870291192183795e-05,
836
+ "loss": 1.9719,
837
+ "step": 130
838
+ },
839
+ {
840
+ "epoch": 1.26,
841
+ "learning_rate": 2.1384345341915767e-05,
842
+ "loss": 2.0634,
843
+ "step": 131
844
+ },
845
+ {
846
+ "epoch": 1.27,
847
+ "learning_rate": 2.0901197798264166e-05,
848
+ "loss": 2.0109,
849
+ "step": 132
850
+ },
851
+ {
852
+ "epoch": 1.28,
853
+ "learning_rate": 2.0420970191039358e-05,
854
+ "loss": 1.9339,
855
+ "step": 133
856
+ },
857
+ {
858
+ "epoch": 1.29,
859
+ "learning_rate": 1.9943783414973313e-05,
860
+ "loss": 1.941,
861
+ "step": 134
862
+ },
863
+ {
864
+ "epoch": 1.3,
865
+ "learning_rate": 1.946975759928506e-05,
866
+ "loss": 2.0228,
867
+ "step": 135
868
+ },
869
+ {
870
+ "epoch": 1.31,
871
+ "learning_rate": 1.899901207743869e-05,
872
+ "loss": 2.0524,
873
+ "step": 136
874
+ },
875
+ {
876
+ "epoch": 1.32,
877
+ "learning_rate": 1.853166535710192e-05,
878
+ "loss": 2.0208,
879
+ "step": 137
880
+ },
881
+ {
882
+ "epoch": 1.33,
883
+ "learning_rate": 1.806783509031234e-05,
884
+ "loss": 1.951,
885
+ "step": 138
886
+ },
887
+ {
888
+ "epoch": 1.34,
889
+ "learning_rate": 1.760763804385916e-05,
890
+ "loss": 2.031,
891
+ "step": 139
892
+ },
893
+ {
894
+ "epoch": 1.35,
895
+ "learning_rate": 1.7151190069887813e-05,
896
+ "loss": 1.9507,
897
+ "step": 140
898
+ },
899
+ {
900
+ "epoch": 1.35,
901
+ "eval_loss": 2.131213903427124,
902
+ "eval_runtime": 2.7,
903
+ "eval_samples_per_second": 4.074,
904
+ "eval_steps_per_second": 0.741,
905
+ "step": 140
906
+ },
907
+ {
908
+ "epoch": 1.36,
909
+ "learning_rate": 1.669860607673477e-05,
910
+ "loss": 2.1314,
911
+ "step": 141
912
+ },
913
+ {
914
+ "epoch": 1.37,
915
+ "learning_rate": 1.6250000000000005e-05,
916
+ "loss": 2.0099,
917
+ "step": 142
918
+ },
919
+ {
920
+ "epoch": 1.38,
921
+ "learning_rate": 1.5805484773864303e-05,
922
+ "loss": 1.9891,
923
+ "step": 143
924
+ },
925
+ {
926
+ "epoch": 1.38,
927
+ "learning_rate": 1.536517230265867e-05,
928
+ "loss": 2.0866,
929
+ "step": 144
930
+ },
931
+ {
932
+ "epoch": 1.39,
933
+ "learning_rate": 1.4929173432693088e-05,
934
+ "loss": 2.0756,
935
+ "step": 145
936
+ },
937
+ {
938
+ "epoch": 1.4,
939
+ "learning_rate": 1.4497597924351416e-05,
940
+ "loss": 2.02,
941
+ "step": 146
942
+ },
943
+ {
944
+ "epoch": 1.41,
945
+ "learning_rate": 1.4070554424459953e-05,
946
+ "loss": 1.9735,
947
+ "step": 147
948
+ },
949
+ {
950
+ "epoch": 1.42,
951
+ "learning_rate": 1.3648150438936062e-05,
952
+ "loss": 1.9393,
953
+ "step": 148
954
+ },
955
+ {
956
+ "epoch": 1.43,
957
+ "learning_rate": 1.3230492305724189e-05,
958
+ "loss": 1.996,
959
+ "step": 149
960
+ },
961
+ {
962
+ "epoch": 1.44,
963
+ "learning_rate": 1.2817685168025838e-05,
964
+ "loss": 2.0682,
965
+ "step": 150
966
+ }
967
+ ],
968
+ "logging_steps": 1,
969
+ "max_steps": 208,
970
+ "num_train_epochs": 2,
971
+ "save_steps": 50,
972
+ "total_flos": 1.222374797040681e+17,
973
+ "trial_name": null,
974
+ "trial_params": null
975
+ }
multi-prompt/onechar/checkpoint-150/training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:deca3b521b13630aa127a677d66c86cecd19f6ba8eb8e593ac9a0d31193246e9
3
+ size 4475
multi-prompt/onechar/checkpoint-200/README.md ADDED
@@ -0,0 +1,21 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ library_name: peft
3
+ ---
4
+ ## Training procedure
5
+
6
+
7
+ The following `bitsandbytes` quantization config was used during training:
8
+ - quant_method: bitsandbytes
9
+ - load_in_8bit: False
10
+ - load_in_4bit: True
11
+ - llm_int8_threshold: 6.0
12
+ - llm_int8_skip_modules: None
13
+ - llm_int8_enable_fp32_cpu_offload: False
14
+ - llm_int8_has_fp16_weight: False
15
+ - bnb_4bit_quant_type: nf4
16
+ - bnb_4bit_use_double_quant: True
17
+ - bnb_4bit_compute_dtype: bfloat16
18
+ ### Framework versions
19
+
20
+
21
+ - PEFT 0.6.0.dev0
multi-prompt/onechar/checkpoint-200/adapter_config.json ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "auto_mapping": null,
3
+ "base_model_name_or_path": "meta-llama/Llama-2-13b-hf",
4
+ "bias": "none",
5
+ "fan_in_fan_out": null,
6
+ "inference_mode": true,
7
+ "init_lora_weights": true,
8
+ "layers_pattern": null,
9
+ "layers_to_transform": null,
10
+ "lora_alpha": 16,
11
+ "lora_dropout": 0.01,
12
+ "modules_to_save": null,
13
+ "peft_type": "LORA",
14
+ "r": 8,
15
+ "revision": null,
16
+ "target_modules": [
17
+ "down_proj",
18
+ "up_proj",
19
+ "k_proj",
20
+ "q_proj",
21
+ "o_proj",
22
+ "v_proj",
23
+ "gate_proj"
24
+ ],
25
+ "task_type": "CAUSAL_LM"
26
+ }
multi-prompt/onechar/checkpoint-200/adapter_model.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9e0dc335ec1113796038af12e416902af682cd1d2d6bcf511a7731a5f0fb7e24
3
+ size 125374989
multi-prompt/onechar/checkpoint-200/adapter_model/README.md ADDED
@@ -0,0 +1,21 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ library_name: peft
3
+ ---
4
+ ## Training procedure
5
+
6
+
7
+ The following `bitsandbytes` quantization config was used during training:
8
+ - quant_method: bitsandbytes
9
+ - load_in_8bit: False
10
+ - load_in_4bit: True
11
+ - llm_int8_threshold: 6.0
12
+ - llm_int8_skip_modules: None
13
+ - llm_int8_enable_fp32_cpu_offload: False
14
+ - llm_int8_has_fp16_weight: False
15
+ - bnb_4bit_quant_type: nf4
16
+ - bnb_4bit_use_double_quant: True
17
+ - bnb_4bit_compute_dtype: bfloat16
18
+ ### Framework versions
19
+
20
+
21
+ - PEFT 0.6.0.dev0
multi-prompt/onechar/checkpoint-200/adapter_model/adapter_config.json ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "auto_mapping": null,
3
+ "base_model_name_or_path": "meta-llama/Llama-2-13b-hf",
4
+ "bias": "none",
5
+ "fan_in_fan_out": null,
6
+ "inference_mode": true,
7
+ "init_lora_weights": true,
8
+ "layers_pattern": null,
9
+ "layers_to_transform": null,
10
+ "lora_alpha": 16,
11
+ "lora_dropout": 0.01,
12
+ "modules_to_save": null,
13
+ "peft_type": "LORA",
14
+ "r": 8,
15
+ "revision": null,
16
+ "target_modules": [
17
+ "down_proj",
18
+ "up_proj",
19
+ "k_proj",
20
+ "q_proj",
21
+ "o_proj",
22
+ "v_proj",
23
+ "gate_proj"
24
+ ],
25
+ "task_type": "CAUSAL_LM"
26
+ }
multi-prompt/onechar/checkpoint-200/adapter_model/adapter_model.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9e0dc335ec1113796038af12e416902af682cd1d2d6bcf511a7731a5f0fb7e24
3
+ size 125374989
multi-prompt/onechar/checkpoint-200/optimizer.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7a6452765ece252eca8ff0c0b9c9668b4fd8cb7950fc74aa92cd351428ac8d26
3
+ size 250681597
multi-prompt/onechar/checkpoint-200/rng_state_0.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:37101134cb18b14dcc0a89605f1ebcad5f5f724a7829ef0ab8c2375c1a35ab9b
3
+ size 21687
multi-prompt/onechar/checkpoint-200/rng_state_1.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9784f91de71853ebcbcec54e9436f2446c8d5f3503d1908fe1f3de175026941b
3
+ size 21687
multi-prompt/onechar/checkpoint-200/rng_state_2.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:886d5962d4b3ed49287bfec878352ac1771f41077883d9bad61da741a1264646
3
+ size 21687