Model save

README.md

@@ -32,16 +32,16 @@ datasets:
     type: alpaca
 dataset_prepared_path:
 val_set_size: 0.2
-output_dir: ./qlora-qwen25
+output_dir: ./qlora-qwen25-instruct-2
 
-sequence_len: 8192
+sequence_len: 2048
 sample_packing: true
 eval_sample_packing: true
 pad_to_sequence_len: true
 
 adapter: qlora
 lora_model_dir:
-lora_r: 256
+lora_r: 32
 lora_alpha: 128
 lora_dropout: 0.05
 lora_target_linear: true
@@ -54,8 +54,8 @@ wandb_name:
 wandb_log_model:
 
 gradient_accumulation_steps: 1
-micro_batch_size: 1
-num_epochs: 2
+micro_batch_size: 2
+num_epochs: 3
 optimizer: adamw_torch
 lr_scheduler: cosine
 learning_rate: 0.00002
@@ -97,7 +97,7 @@ fsdp_config:
   fsdp_sharding_strategy: FULL_SHARD
 special_tokens:
 
-wandb_project: qwen-25-7b-instruct
+wandb_project: qlora-qwen-25-7b-instruct
 wandb_entity: 
 wandb_watch:
 wandb_name: 
@@ -118,7 +118,7 @@ auto_resume_from_checkpoints: true
 
 This model is a fine-tuned version of [Qwen/Qwen2.5-7B-Instruct](https://huggingface.co/Qwen/Qwen2.5-7B-Instruct) on the medalpaca/medical_meadow_medqa dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.1257
+- Loss: 0.1429
 
 ## Model description
 
@@ -138,30 +138,34 @@ More information needed
 
 The following hyperparameters were used during training:
 - learning_rate: 2e-05
-- train_batch_size: 1
-- eval_batch_size: 1
+- train_batch_size: 2
+- eval_batch_size: 2
 - seed: 42
 - distributed_type: multi-GPU
 - num_devices: 4
-- total_train_batch_size: 4
-- total_eval_batch_size: 4
+- total_train_batch_size: 8
+- total_eval_batch_size: 8
 - optimizer: Use adamw_torch with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
 - lr_scheduler_type: cosine
-- lr_scheduler_warmup_steps: 4
-- num_epochs: 2
+- lr_scheduler_warmup_steps: 13
+- num_epochs: 3
 
 ### Training results
 
 | Training Loss | Epoch | Step | Validation Loss |
 |:-------------:|:-----:|:----:|:---------------:|
-| 0.1473        | 0.25  | 18   | 0.1576          |
-| 0.1456        | 0.5   | 36   | 0.1333          |
-| 0.121         | 0.75  | 54   | 0.1312          |
-| 0.1328        | 1.0   | 72   | 0.1303          |
-| 0.1336        | 1.25  | 90   | 0.1276          |
-| 0.1228        | 1.5   | 108  | 0.1263          |
-| 0.1199        | 1.75  | 126  | 0.1260          |
-| 0.1393        | 2.0   | 144  | 0.1257          |
+| 0.1255        | 0.25  | 37   | 0.1342          |
+| 0.1201        | 0.5   | 74   | 0.1235          |
+| 0.1227        | 0.75  | 111  | 0.1159          |
+| 0.1289        | 1.0   | 148  | 0.1116          |
+| 0.1004        | 1.25  | 185  | 0.1131          |
+| 0.0783        | 1.5   | 222  | 0.1124          |
+| 0.053         | 1.75  | 259  | 0.1171          |
+| 0.0747        | 2.0   | 296  | 0.1132          |
+| 0.0629        | 2.25  | 333  | 0.1366          |
+| 0.0655        | 2.5   | 370  | 0.1443          |
+| 0.0492        | 2.75  | 407  | 0.1435          |
+| 0.0509        | 3.0   | 444  | 0.1429          |
 
 
 ### Framework versions