Model save

adapter_config.json

@@ -21,11 +21,11 @@
   "revision": null,
   "target_modules": [
     "o_proj",
-    "q_proj",
     "gate_proj",
+    "q_proj",
     "v_proj",
-    "down_proj",
     "k_proj",
+    "down_proj",
     "up_proj"
   ],
   "task_type": "CAUSAL_LM",

all_results.json

@@ -2,8 +2,8 @@
     "epoch": 1.0,
     "total_flos": 1.1487027058043781e+19,
     "train_loss": 0.0,
-    "train_runtime": 0.0169,
+    "train_runtime": 0.0143,
     "train_samples": 207864,
-    "train_samples_per_second": 8184133.024,
-    "train_steps_per_second": 341027.672
+    "train_samples_per_second": 9715326.166,
+    "train_steps_per_second": 404831.526
 }

config.json

@@ -0,0 +1,43 @@
+{
+  "_attn_implementation_autoset": true,
+  "_name_or_path": "mistralai/Mistral-7B-v0.1",
+  "architectures": [
+    "MistralForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 32768,
+  "model_type": "mistral",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 8,
+  "quantization_config": {
+    "_load_in_4bit": true,
+    "_load_in_8bit": false,
+    "bnb_4bit_compute_dtype": "bfloat16",
+    "bnb_4bit_quant_storage": "uint8",
+    "bnb_4bit_quant_type": "nf4",
+    "bnb_4bit_use_double_quant": false,
+    "llm_int8_enable_fp32_cpu_offload": false,
+    "llm_int8_has_fp16_weight": false,
+    "llm_int8_skip_modules": null,
+    "llm_int8_threshold": 6.0,
+    "load_in_4bit": true,
+    "load_in_8bit": false,
+    "quant_method": "bitsandbytes"
+  },
+  "rms_norm_eps": 1e-05,
+  "rope_theta": 10000.0,
+  "sliding_window": 4096,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.46.2",
+  "use_cache": true,
+  "vocab_size": 32000
+}

runs/Nov20_07-40-49_d28f7adca72c/events.out.tfevents.1732088488.d28f7adca72c.2261562.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5e2e509964aed74faa557e627c0de135c83923305e7b4690aab75bbb4d9c825f
+size 6644

runs/Nov20_07-45-35_d28f7adca72c/events.out.tfevents.1732088781.d28f7adca72c.2263555.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:43168300e80ab14dedc9681882351d0382d6f044c740e17338fd467d232580d3
+size 6644

runs/Nov20_08-00-39_d28f7adca72c/events.out.tfevents.1732089682.d28f7adca72c.2269100.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8aa39bb29ac4889858038b98286c9be6cd73011a8faad911c09e08f6c816b725
+size 6644

runs/Nov20_08-03-58_d28f7adca72c/events.out.tfevents.1732089882.d28f7adca72c.2270466.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3bb5de9fee45edbfb36d13a6b3f444acc72e9eccaa32c351e7d912a217d403a1
+size 6644

runs/Nov20_08-06-30_d28f7adca72c/events.out.tfevents.1732090033.d28f7adca72c.2271569.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b03023784e0f0a70ff615754a4cc0d83db8fb3f8b44839a46bbaab0c3606eca0
+size 6644

runs/Nov20_08-08-06_d28f7adca72c/events.out.tfevents.1732090139.d28f7adca72c.2272351.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8f4c896a1c2e2918b1b9df3f8fb65cd626a9f4904d66f83c42e8136461d47a2b
+size 6998

train_results.json

@@ -2,8 +2,8 @@
     "epoch": 1.0,
     "total_flos": 1.1487027058043781e+19,
     "train_loss": 0.0,
-    "train_runtime": 0.0169,
+    "train_runtime": 0.0143,
     "train_samples": 207864,
-    "train_samples_per_second": 8184133.024,
-    "train_steps_per_second": 341027.672
+    "train_samples_per_second": 9715326.166,
+    "train_steps_per_second": 404831.526
 }

trainer_state.json

@@ -8105,9 +8105,9 @@
       "step": 5779,
       "total_flos": 1.1487027058043781e+19,
       "train_loss": 0.0,
-      "train_runtime": 0.0169,
-      "train_samples_per_second": 8184133.024,
-      "train_steps_per_second": 341027.672
+      "train_runtime": 0.0143,
+      "train_samples_per_second": 9715326.166,
+      "train_steps_per_second": 404831.526
     }
   ],
   "logging_steps": 5,

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:82763ac4d1451da25c7cb359dd9ee2581f99c3d2b8db998d96375db407c0978e
+oid sha256:952937bb8d6cdab0cd5cdf0d6bde0283fcedf864753ae3832fde55a9debaddd9
 size 6328