Upload MixtralForCausalLM

config.json

@@ -24,7 +24,56 @@
     "dataset": "wikitext2",
     "desc_act": false,
     "group_size": 128,
-    "modules_in_block_to_quantize": null,
+    "modules_in_block_to_quantize": [
+      [
+        "self_attn.k_proj",
+        "self_attn.v_proj",
+        "self_attn.q_proj"
+      ],
+      [
+        "self_attn.o_proj"
+      ],
+      [
+        "block_sparse_moe.experts.0.w1",
+        "block_sparse_moe.experts.0.w2",
+        "block_sparse_moe.experts.0.w3"
+      ],
+      [
+        "block_sparse_moe.experts.1.w1",
+        "block_sparse_moe.experts.1.w2",
+        "block_sparse_moe.experts.1.w3"
+      ],
+      [
+        "block_sparse_moe.experts.2.w1",
+        "block_sparse_moe.experts.2.w2",
+        "block_sparse_moe.experts.2.w3"
+      ],
+      [
+        "block_sparse_moe.experts.3.w1",
+        "block_sparse_moe.experts.3.w2",
+        "block_sparse_moe.experts.3.w3"
+      ],
+      [
+        "block_sparse_moe.experts.4.w1",
+        "block_sparse_moe.experts.4.w2",
+        "block_sparse_moe.experts.4.w3"
+      ],
+      [
+        "block_sparse_moe.experts.5.w1",
+        "block_sparse_moe.experts.5.w2",
+        "block_sparse_moe.experts.5.w3"
+      ],
+      [
+        "block_sparse_moe.experts.6.w1",
+        "block_sparse_moe.experts.6.w2",
+        "block_sparse_moe.experts.6.w3"
+      ],
+      [
+        "block_sparse_moe.experts.7.w1",
+        "block_sparse_moe.experts.7.w2",
+        "block_sparse_moe.experts.7.w3"
+      ]
+    ],
     "quant_method": "gptq",
     "sym": true,
     "true_sequential": true
@@ -36,6 +85,6 @@
   "tie_word_embeddings": false,
   "torch_dtype": "float16",
   "transformers_version": "4.39.3",
-  "use_cache": false,
+  "use_cache": true,
   "vocab_size": 32768
 }

model-00001-of-00015.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:04c046ae8f0ccf402ec6b08f6402308187be4409350232c6ac3cba2112796647
-size 4984890344
+oid sha256:ee7fe4d38e1c900d0e5e5a251bab06fac3c173df8818109e0af8e7f89a44b124
+size 4985132728

model-00002-of-00015.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fbe99adbf4bc6d7f28bb70c5b4b9c846979725ea5a8d5e431ba25701c0622351
-size 4999568144
+oid sha256:2fbe866a1d6c91cace9b51846bf298cbdd97c371e86475ab4ad52db3b8ad28ec
+size 4999760248

model-00003-of-00015.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:481d9f89c907617061f0132c67f1e3b29faff5d1ae0c832bfd3cc925ebf78502
-size 4950494384
+oid sha256:39992a82031ba7b33200137ed6a9b5786672e2c3d6bc05627a188b00ec2cdc95
+size 4950686368

model-00004-of-00015.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:14179eca9b205cecec039d2a090cfa30d6af62585307faa25445a1965ef661a0
-size 4955254808
+oid sha256:e799d93d64e39ea2808305fc9e5f043554e843bb7aec083722ab2f42a7c5876f
+size 4955398880

model-00005-of-00015.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1cc5f8af01512f2c955b80df5c589eb2d94a94d65251e54e36448add56760693
-size 4999568816
+oid sha256:a0d2de9aaf7de39c018fc23ef4f9cb370de165c195d711778cfa117912f9e536
+size 4999760800

model-00006-of-00015.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1a87a041552f53382a8c731f8a18f2a42fb1eeb468d0387f44112c2655a7ce17
-size 4950494632
+oid sha256:e9378fb98aa10078fd99764cefb73895acc1399f3ff7c1d342a58ca40bbaf01b
+size 4950686728

model-00007-of-00015.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f7baaabdb3a5aa973af09503ac31f3d2ad54246b4d890f344b8ae5c63b4ce869
-size 4999568824
+oid sha256:ed74705bb7f52550dc74e59464fabedc311fdebc728e925dddaba91d87e9de16
+size 4999760800

model-00008-of-00015.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6f32a4236c4511c8711144bab24bbfc9e551456dde2e5ed220d35a6d79db90de
-size 4950494632
+oid sha256:9b1a7379f8ffa1a847e7f320644d9d818d7c04ebcedeb22ab32d61d140bd4aa9
+size 4950686728

model-00009-of-00015.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2b797ebf00413c7ac2127115b282004940b33f6b348f9b5d0c2a4c169981ad03
-size 4999568824
+oid sha256:556b48d690c2598bcec7836a203e39af86126077f4618af669b52b8971e618c7
+size 4999760800

model-00010-of-00015.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ef47303d080a9bae1fafe5d3b51aac7ce8f65807dc5ba3f55b8a967cca6c647f
-size 4956885384
+oid sha256:bdb2b8a2b7721d090a77dc7095d4768526323f14c71b9a1698bf7c01b6b810a5
+size 4957029456

model-00011-of-00015.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a0421ea2a53380143d52173ef4479008204e55bdfe9261d32d151022b6900e0a
-size 4999568824
+oid sha256:8189ea0ca0d64236746e00d2a32f50cd8cfb5063a052edbc5cb8c385113cf0ad
+size 4999760800

model-00012-of-00015.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9deb8be90bafffca69cddccca415d610e1ec69d269da2442f95cff3afbcd49a5
-size 4950494632
+oid sha256:5614341acb2b11941f9a25cf6acb0a065c1d9b87f6c869c12eec4f0eecdc4ea0
+size 4950686728

model-00013-of-00015.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9ecd14eb036ef3ce831339ff64e1d109bb26d58b48941e2d4f72be0d7626855b
-size 4999568824
+oid sha256:bb7b10e681f268eb7753efd3adea3e64778a3b6485e970554386a6e330cc3dd9
+size 4999760800

model-00014-of-00015.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:bdd7d67a99dc337e73b1ea92028f4fd1df2866ebcf5cd29bf453fa170787913a
-size 4950494632
+oid sha256:e9de71eac6df45a8dc50a7f1383faa5b8a6767fe126501ffa629e1dc8cea147b
+size 4950686728

model-00015-of-00015.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:25c1afe5fa496c0caca169faa252113ef4d8f0106679d19d03c6f19521f742c0
-size 4107685704
+oid sha256:054531e9c6efdeb6531a74d19e435df5f60a367677816caf4cd1b4ce45dff328
+size 4107731232

model.safetensors.index.json

@@ -1,6 +1,6 @@
 {
   "metadata": {
-    "total_size": 73753582464
+    "total_size": 73756291072
   },
   "weight_map": {
     "lm_head.weight": "model-00015-of-00015.safetensors",
@@ -125,11 +125,7 @@
     "model.layers.0.block_sparse_moe.experts.7.w3.qweight": "model-00001-of-00015.safetensors",
     "model.layers.0.block_sparse_moe.experts.7.w3.qzeros": "model-00001-of-00015.safetensors",
     "model.layers.0.block_sparse_moe.experts.7.w3.scales": "model-00001-of-00015.safetensors",
-    "model.layers.0.block_sparse_moe.gate.bias": "model-00001-of-00015.safetensors",
-    "model.layers.0.block_sparse_moe.gate.g_idx": "model-00001-of-00015.safetensors",
-    "model.layers.0.block_sparse_moe.gate.qweight": "model-00001-of-00015.safetensors",
-    "model.layers.0.block_sparse_moe.gate.qzeros": "model-00001-of-00015.safetensors",
-    "model.layers.0.block_sparse_moe.gate.scales": "model-00001-of-00015.safetensors",
+    "model.layers.0.block_sparse_moe.gate.weight": "model-00001-of-00015.safetensors",
     "model.layers.0.input_layernorm.weight": "model-00001-of-00015.safetensors",
     "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00015.safetensors",
     "model.layers.0.self_attn.k_proj.bias": "model-00001-of-00015.safetensors",
@@ -272,10 +268,7 @@
     "model.layers.1.block_sparse_moe.experts.7.w3.qweight": "model-00001-of-00015.safetensors",
     "model.layers.1.block_sparse_moe.experts.7.w3.qzeros": "model-00001-of-00015.safetensors",
     "model.layers.1.block_sparse_moe.experts.7.w3.scales": "model-00001-of-00015.safetensors",
-    "model.layers.1.block_sparse_moe.gate.bias": "model-00001-of-00015.safetensors",
-    "model.layers.1.block_sparse_moe.gate.g_idx": "model-00001-of-00015.safetensors",
-    "model.layers.1.block_sparse_moe.gate.qweight": "model-00001-of-00015.safetensors",
-    "model.layers.1.block_sparse_moe.gate.scales": "model-00001-of-00015.safetensors",
+    "model.layers.1.block_sparse_moe.gate.weight": "model-00001-of-00015.safetensors",
     "model.layers.1.input_layernorm.weight": "model-00001-of-00015.safetensors",
     "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00015.safetensors",
     "model.layers.1.self_attn.k_proj.bias": "model-00001-of-00015.safetensors",
@@ -418,10 +411,7 @@
     "model.layers.10.block_sparse_moe.experts.7.w3.qweight": "model-00003-of-00015.safetensors",
     "model.layers.10.block_sparse_moe.experts.7.w3.qzeros": "model-00003-of-00015.safetensors",
     "model.layers.10.block_sparse_moe.experts.7.w3.scales": "model-00003-of-00015.safetensors",
-    "model.layers.10.block_sparse_moe.gate.bias": "model-00003-of-00015.safetensors",
-    "model.layers.10.block_sparse_moe.gate.g_idx": "model-00003-of-00015.safetensors",
-    "model.layers.10.block_sparse_moe.gate.qweight": "model-00003-of-00015.safetensors",
-    "model.layers.10.block_sparse_moe.gate.scales": "model-00003-of-00015.safetensors",
+    "model.layers.10.block_sparse_moe.gate.weight": "model-00003-of-00015.safetensors",
     "model.layers.10.input_layernorm.weight": "model-00003-of-00015.safetensors",
     "model.layers.10.post_attention_layernorm.weight": "model-00003-of-00015.safetensors",
     "model.layers.10.self_attn.k_proj.bias": "model-00003-of-00015.safetensors",
@@ -564,10 +554,7 @@
     "model.layers.11.block_sparse_moe.experts.7.w3.qweight": "model-00004-of-00015.safetensors",
     "model.layers.11.block_sparse_moe.experts.7.w3.qzeros": "model-00004-of-00015.safetensors",
     "model.layers.11.block_sparse_moe.experts.7.w3.scales": "model-00004-of-00015.safetensors",
-    "model.layers.11.block_sparse_moe.gate.bias": "model-00004-of-00015.safetensors",
-    "model.layers.11.block_sparse_moe.gate.g_idx": "model-00004-of-00015.safetensors",
-    "model.layers.11.block_sparse_moe.gate.qweight": "model-00004-of-00015.safetensors",
-    "model.layers.11.block_sparse_moe.gate.scales": "model-00004-of-00015.safetensors",
+    "model.layers.11.block_sparse_moe.gate.weight": "model-00003-of-00015.safetensors",
     "model.layers.11.input_layernorm.weight": "model-00004-of-00015.safetensors",
     "model.layers.11.post_attention_layernorm.weight": "model-00004-of-00015.safetensors",
     "model.layers.11.self_attn.k_proj.bias": "model-00003-of-00015.safetensors",
@@ -710,10 +697,7 @@
     "model.layers.12.block_sparse_moe.experts.7.w3.qweight": "model-00004-of-00015.safetensors",
     "model.layers.12.block_sparse_moe.experts.7.w3.qzeros": "model-00004-of-00015.safetensors",
     "model.layers.12.block_sparse_moe.experts.7.w3.scales": "model-00004-of-00015.safetensors",
-    "model.layers.12.block_sparse_moe.gate.bias": "model-00004-of-00015.safetensors",
-    "model.layers.12.block_sparse_moe.gate.g_idx": "model-00004-of-00015.safetensors",
-    "model.layers.12.block_sparse_moe.gate.qweight": "model-00004-of-00015.safetensors",
-    "model.layers.12.block_sparse_moe.gate.scales": "model-00004-of-00015.safetensors",
+    "model.layers.12.block_sparse_moe.gate.weight": "model-00004-of-00015.safetensors",
     "model.layers.12.input_layernorm.weight": "model-00004-of-00015.safetensors",
     "model.layers.12.post_attention_layernorm.weight": "model-00004-of-00015.safetensors",
     "model.layers.12.self_attn.k_proj.bias": "model-00004-of-00015.safetensors",
@@ -856,10 +840,7 @@
     "model.layers.13.block_sparse_moe.experts.7.w3.qweight": "model-00004-of-00015.safetensors",
     "model.layers.13.block_sparse_moe.experts.7.w3.qzeros": "model-00004-of-00015.safetensors",
     "model.layers.13.block_sparse_moe.experts.7.w3.scales": "model-00004-of-00015.safetensors",
-    "model.layers.13.block_sparse_moe.gate.bias": "model-00004-of-00015.safetensors",
-    "model.layers.13.block_sparse_moe.gate.g_idx": "model-00004-of-00015.safetensors",
-    "model.layers.13.block_sparse_moe.gate.qweight": "model-00004-of-00015.safetensors",
-    "model.layers.13.block_sparse_moe.gate.scales": "model-00004-of-00015.safetensors",
+    "model.layers.13.block_sparse_moe.gate.weight": "model-00004-of-00015.safetensors",
     "model.layers.13.input_layernorm.weight": "model-00004-of-00015.safetensors",
     "model.layers.13.post_attention_layernorm.weight": "model-00004-of-00015.safetensors",
     "model.layers.13.self_attn.k_proj.bias": "model-00004-of-00015.safetensors",
@@ -1002,10 +983,7 @@
     "model.layers.14.block_sparse_moe.experts.7.w3.qweight": "model-00005-of-00015.safetensors",
     "model.layers.14.block_sparse_moe.experts.7.w3.qzeros": "model-00005-of-00015.safetensors",
     "model.layers.14.block_sparse_moe.experts.7.w3.scales": "model-00005-of-00015.safetensors",
-    "model.layers.14.block_sparse_moe.gate.bias": "model-00005-of-00015.safetensors",
-    "model.layers.14.block_sparse_moe.gate.g_idx": "model-00005-of-00015.safetensors",
-    "model.layers.14.block_sparse_moe.gate.qweight": "model-00005-of-00015.safetensors",
-    "model.layers.14.block_sparse_moe.gate.scales": "model-00005-of-00015.safetensors",
+    "model.layers.14.block_sparse_moe.gate.weight": "model-00004-of-00015.safetensors",
     "model.layers.14.input_layernorm.weight": "model-00005-of-00015.safetensors",
     "model.layers.14.post_attention_layernorm.weight": "model-00005-of-00015.safetensors",
     "model.layers.14.self_attn.k_proj.bias": "model-00004-of-00015.safetensors",
@@ -1148,11 +1126,7 @@
     "model.layers.15.block_sparse_moe.experts.7.w3.qweight": "model-00005-of-00015.safetensors",
     "model.layers.15.block_sparse_moe.experts.7.w3.qzeros": "model-00005-of-00015.safetensors",
     "model.layers.15.block_sparse_moe.experts.7.w3.scales": "model-00005-of-00015.safetensors",
-    "model.layers.15.block_sparse_moe.gate.bias": "model-00005-of-00015.safetensors",
-    "model.layers.15.block_sparse_moe.gate.g_idx": "model-00005-of-00015.safetensors",
-    "model.layers.15.block_sparse_moe.gate.qweight": "model-00005-of-00015.safetensors",
-    "model.layers.15.block_sparse_moe.gate.qzeros": "model-00005-of-00015.safetensors",
-    "model.layers.15.block_sparse_moe.gate.scales": "model-00005-of-00015.safetensors",
+    "model.layers.15.block_sparse_moe.gate.weight": "model-00005-of-00015.safetensors",
     "model.layers.15.input_layernorm.weight": "model-00005-of-00015.safetensors",
     "model.layers.15.post_attention_layernorm.weight": "model-00005-of-00015.safetensors",
     "model.layers.15.self_attn.k_proj.bias": "model-00005-of-00015.safetensors",
@@ -1295,10 +1269,7 @@
     "model.layers.16.block_sparse_moe.experts.7.w3.qweight": "model-00005-of-00015.safetensors",
     "model.layers.16.block_sparse_moe.experts.7.w3.qzeros": "model-00005-of-00015.safetensors",
     "model.layers.16.block_sparse_moe.experts.7.w3.scales": "model-00005-of-00015.safetensors",
-    "model.layers.16.block_sparse_moe.gate.bias": "model-00005-of-00015.safetensors",
-    "model.layers.16.block_sparse_moe.gate.g_idx": "model-00005-of-00015.safetensors",
-    "model.layers.16.block_sparse_moe.gate.qweight": "model-00005-of-00015.safetensors",
-    "model.layers.16.block_sparse_moe.gate.scales": "model-00005-of-00015.safetensors",
+    "model.layers.16.block_sparse_moe.gate.weight": "model-00005-of-00015.safetensors",
     "model.layers.16.input_layernorm.weight": "model-00005-of-00015.safetensors",
     "model.layers.16.post_attention_layernorm.weight": "model-00005-of-00015.safetensors",
     "model.layers.16.self_attn.k_proj.bias": "model-00005-of-00015.safetensors",
@@ -1441,10 +1412,7 @@
     "model.layers.17.block_sparse_moe.experts.7.w3.qweight": "model-00005-of-00015.safetensors",
     "model.layers.17.block_sparse_moe.experts.7.w3.qzeros": "model-00005-of-00015.safetensors",
     "model.layers.17.block_sparse_moe.experts.7.w3.scales": "model-00005-of-00015.safetensors",
-    "model.layers.17.block_sparse_moe.gate.bias": "model-00005-of-00015.safetensors",
-    "model.layers.17.block_sparse_moe.gate.g_idx": "model-00005-of-00015.safetensors",
-    "model.layers.17.block_sparse_moe.gate.qweight": "model-00005-of-00015.safetensors",
-    "model.layers.17.block_sparse_moe.gate.scales": "model-00005-of-00015.safetensors",
+    "model.layers.17.block_sparse_moe.gate.weight": "model-00005-of-00015.safetensors",
     "model.layers.17.input_layernorm.weight": "model-00005-of-00015.safetensors",
     "model.layers.17.post_attention_layernorm.weight": "model-00005-of-00015.safetensors",
     "model.layers.17.self_attn.k_proj.bias": "model-00005-of-00015.safetensors",
@@ -1587,10 +1555,7 @@
     "model.layers.18.block_sparse_moe.experts.7.w3.qweight": "model-00006-of-00015.safetensors",
     "model.layers.18.block_sparse_moe.experts.7.w3.qzeros": "model-00006-of-00015.safetensors",
     "model.layers.18.block_sparse_moe.experts.7.w3.scales": "model-00006-of-00015.safetensors",
-    "model.layers.18.block_sparse_moe.gate.bias": "model-00006-of-00015.safetensors",
-    "model.layers.18.block_sparse_moe.gate.g_idx": "model-00006-of-00015.safetensors",
-    "model.layers.18.block_sparse_moe.gate.qweight": "model-00006-of-00015.safetensors",
-    "model.layers.18.block_sparse_moe.gate.scales": "model-00006-of-00015.safetensors",
+    "model.layers.18.block_sparse_moe.gate.weight": "model-00005-of-00015.safetensors",
     "model.layers.18.input_layernorm.weight": "model-00006-of-00015.safetensors",
     "model.layers.18.post_attention_layernorm.weight": "model-00006-of-00015.safetensors",
     "model.layers.18.self_attn.k_proj.bias": "model-00005-of-00015.safetensors",
@@ -1733,10 +1698,7 @@
     "model.layers.19.block_sparse_moe.experts.7.w3.qweight": "model-00006-of-00015.safetensors",
     "model.layers.19.block_sparse_moe.experts.7.w3.qzeros": "model-00006-of-00015.safetensors",
     "model.layers.19.block_sparse_moe.experts.7.w3.scales": "model-00006-of-00015.safetensors",
-    "model.layers.19.block_sparse_moe.gate.bias": "model-00006-of-00015.safetensors",
-    "model.layers.19.block_sparse_moe.gate.g_idx": "model-00006-of-00015.safetensors",
-    "model.layers.19.block_sparse_moe.gate.qweight": "model-00006-of-00015.safetensors",
-    "model.layers.19.block_sparse_moe.gate.scales": "model-00006-of-00015.safetensors",
+    "model.layers.19.block_sparse_moe.gate.weight": "model-00006-of-00015.safetensors",
     "model.layers.19.input_layernorm.weight": "model-00006-of-00015.safetensors",
     "model.layers.19.post_attention_layernorm.weight": "model-00006-of-00015.safetensors",
     "model.layers.19.self_attn.k_proj.bias": "model-00006-of-00015.safetensors",
@@ -1879,10 +1841,7 @@
     "model.layers.2.block_sparse_moe.experts.7.w3.qweight": "model-00001-of-00015.safetensors",
     "model.layers.2.block_sparse_moe.experts.7.w3.qzeros": "model-00001-of-00015.safetensors",
     "model.layers.2.block_sparse_moe.experts.7.w3.scales": "model-00001-of-00015.safetensors",
-    "model.layers.2.block_sparse_moe.gate.bias": "model-00001-of-00015.safetensors",
-    "model.layers.2.block_sparse_moe.gate.g_idx": "model-00001-of-00015.safetensors",
-    "model.layers.2.block_sparse_moe.gate.qweight": "model-00001-of-00015.safetensors",
-    "model.layers.2.block_sparse_moe.gate.scales": "model-00001-of-00015.safetensors",
+    "model.layers.2.block_sparse_moe.gate.weight": "model-00001-of-00015.safetensors",
     "model.layers.2.input_layernorm.weight": "model-00001-of-00015.safetensors",
     "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00015.safetensors",
     "model.layers.2.self_attn.k_proj.bias": "model-00001-of-00015.safetensors",
@@ -2025,10 +1984,7 @@
     "model.layers.20.block_sparse_moe.experts.7.w3.qweight": "model-00006-of-00015.safetensors",
     "model.layers.20.block_sparse_moe.experts.7.w3.qzeros": "model-00006-of-00015.safetensors",
     "model.layers.20.block_sparse_moe.experts.7.w3.scales": "model-00006-of-00015.safetensors",
-    "model.layers.20.block_sparse_moe.gate.bias": "model-00006-of-00015.safetensors",
-    "model.layers.20.block_sparse_moe.gate.g_idx": "model-00006-of-00015.safetensors",
-    "model.layers.20.block_sparse_moe.gate.qweight": "model-00006-of-00015.safetensors",
-    "model.layers.20.block_sparse_moe.gate.scales": "model-00006-of-00015.safetensors",
+    "model.layers.20.block_sparse_moe.gate.weight": "model-00006-of-00015.safetensors",
     "model.layers.20.input_layernorm.weight": "model-00006-of-00015.safetensors",
     "model.layers.20.post_attention_layernorm.weight": "model-00006-of-00015.safetensors",
     "model.layers.20.self_attn.k_proj.bias": "model-00006-of-00015.safetensors",
@@ -2171,10 +2127,7 @@
     "model.layers.21.block_sparse_moe.experts.7.w3.qweight": "model-00006-of-00015.safetensors",
     "model.layers.21.block_sparse_moe.experts.7.w3.qzeros": "model-00006-of-00015.safetensors",
     "model.layers.21.block_sparse_moe.experts.7.w3.scales": "model-00006-of-00015.safetensors",
-    "model.layers.21.block_sparse_moe.gate.bias": "model-00006-of-00015.safetensors",
-    "model.layers.21.block_sparse_moe.gate.g_idx": "model-00006-of-00015.safetensors",
-    "model.layers.21.block_sparse_moe.gate.qweight": "model-00006-of-00015.safetensors",
-    "model.layers.21.block_sparse_moe.gate.scales": "model-00006-of-00015.safetensors",
+    "model.layers.21.block_sparse_moe.gate.weight": "model-00006-of-00015.safetensors",
     "model.layers.21.input_layernorm.weight": "model-00006-of-00015.safetensors",
     "model.layers.21.post_attention_layernorm.weight": "model-00006-of-00015.safetensors",
     "model.layers.21.self_attn.k_proj.bias": "model-00006-of-00015.safetensors",
@@ -2317,10 +2270,7 @@
     "model.layers.22.block_sparse_moe.experts.7.w3.qweight": "model-00007-of-00015.safetensors",
     "model.layers.22.block_sparse_moe.experts.7.w3.qzeros": "model-00007-of-00015.safetensors",
     "model.layers.22.block_sparse_moe.experts.7.w3.scales": "model-00007-of-00015.safetensors",
-    "model.layers.22.block_sparse_moe.gate.bias": "model-00007-of-00015.safetensors",
-    "model.layers.22.block_sparse_moe.gate.g_idx": "model-00007-of-00015.safetensors",
-    "model.layers.22.block_sparse_moe.gate.qweight": "model-00007-of-00015.safetensors",
-    "model.layers.22.block_sparse_moe.gate.scales": "model-00007-of-00015.safetensors",
+    "model.layers.22.block_sparse_moe.gate.weight": "model-00006-of-00015.safetensors",
     "model.layers.22.input_layernorm.weight": "model-00007-of-00015.safetensors",
     "model.layers.22.post_attention_layernorm.weight": "model-00007-of-00015.safetensors",
     "model.layers.22.self_attn.k_proj.bias": "model-00006-of-00015.safetensors",
@@ -2463,11 +2413,7 @@
     "model.layers.23.block_sparse_moe.experts.7.w3.qweight": "model-00007-of-00015.safetensors",
     "model.layers.23.block_sparse_moe.experts.7.w3.qzeros": "model-00007-of-00015.safetensors",
     "model.layers.23.block_sparse_moe.experts.7.w3.scales": "model-00007-of-00015.safetensors",
-    "model.layers.23.block_sparse_moe.gate.bias": "model-00007-of-00015.safetensors",
-    "model.layers.23.block_sparse_moe.gate.g_idx": "model-00007-of-00015.safetensors",
-    "model.layers.23.block_sparse_moe.gate.qweight": "model-00007-of-00015.safetensors",
-    "model.layers.23.block_sparse_moe.gate.qzeros": "model-00007-of-00015.safetensors",
-    "model.layers.23.block_sparse_moe.gate.scales": "model-00007-of-00015.safetensors",
+    "model.layers.23.block_sparse_moe.gate.weight": "model-00007-of-00015.safetensors",
     "model.layers.23.input_layernorm.weight": "model-00007-of-00015.safetensors",
     "model.layers.23.post_attention_layernorm.weight": "model-00007-of-00015.safetensors",
     "model.layers.23.self_attn.k_proj.bias": "model-00007-of-00015.safetensors",
@@ -2610,10 +2556,7 @@
     "model.layers.24.block_sparse_moe.experts.7.w3.qweight": "model-00007-of-00015.safetensors",
     "model.layers.24.block_sparse_moe.experts.7.w3.qzeros": "model-00007-of-00015.safetensors",
     "model.layers.24.block_sparse_moe.experts.7.w3.scales": "model-00007-of-00015.safetensors",
-    "model.layers.24.block_sparse_moe.gate.bias": "model-00007-of-00015.safetensors",
-    "model.layers.24.block_sparse_moe.gate.g_idx": "model-00007-of-00015.safetensors",
-    "model.layers.24.block_sparse_moe.gate.qweight": "model-00007-of-00015.safetensors",
-    "model.layers.24.block_sparse_moe.gate.scales": "model-00007-of-00015.safetensors",
+    "model.layers.24.block_sparse_moe.gate.weight": "model-00007-of-00015.safetensors",
     "model.layers.24.input_layernorm.weight": "model-00007-of-00015.safetensors",
     "model.layers.24.post_attention_layernorm.weight": "model-00007-of-00015.safetensors",
     "model.layers.24.self_attn.k_proj.bias": "model-00007-of-00015.safetensors",
@@ -2756,10 +2699,7 @@
     "model.layers.25.block_sparse_moe.experts.7.w3.qweight": "model-00007-of-00015.safetensors",
     "model.layers.25.block_sparse_moe.experts.7.w3.qzeros": "model-00007-of-00015.safetensors",
     "model.layers.25.block_sparse_moe.experts.7.w3.scales": "model-00007-of-00015.safetensors",
-    "model.layers.25.block_sparse_moe.gate.bias": "model-00007-of-00015.safetensors",
-    "model.layers.25.block_sparse_moe.gate.g_idx": "model-00007-of-00015.safetensors",
-    "model.layers.25.block_sparse_moe.gate.qweight": "model-00007-of-00015.safetensors",
-    "model.layers.25.block_sparse_moe.gate.scales": "model-00007-of-00015.safetensors",
+    "model.layers.25.block_sparse_moe.gate.weight": "model-00007-of-00015.safetensors",
     "model.layers.25.input_layernorm.weight": "model-00007-of-00015.safetensors",
     "model.layers.25.post_attention_layernorm.weight": "model-00007-of-00015.safetensors",
     "model.layers.25.self_attn.k_proj.bias": "model-00007-of-00015.safetensors",
@@ -2902,10 +2842,7 @@
     "model.layers.26.block_sparse_moe.experts.7.w3.qweight": "model-00008-of-00015.safetensors",
     "model.layers.26.block_sparse_moe.experts.7.w3.qzeros": "model-00008-of-00015.safetensors",
     "model.layers.26.block_sparse_moe.experts.7.w3.scales": "model-00008-of-00015.safetensors",
-    "model.layers.26.block_sparse_moe.gate.bias": "model-00008-of-00015.safetensors",
-    "model.layers.26.block_sparse_moe.gate.g_idx": "model-00008-of-00015.safetensors",
-    "model.layers.26.block_sparse_moe.gate.qweight": "model-00008-of-00015.safetensors",
-    "model.layers.26.block_sparse_moe.gate.scales": "model-00008-of-00015.safetensors",
+    "model.layers.26.block_sparse_moe.gate.weight": "model-00007-of-00015.safetensors",
     "model.layers.26.input_layernorm.weight": "model-00008-of-00015.safetensors",
     "model.layers.26.post_attention_layernorm.weight": "model-00008-of-00015.safetensors",
     "model.layers.26.self_attn.k_proj.bias": "model-00007-of-00015.safetensors",
@@ -3048,10 +2985,7 @@
     "model.layers.27.block_sparse_moe.experts.7.w3.qweight": "model-00008-of-00015.safetensors",
     "model.layers.27.block_sparse_moe.experts.7.w3.qzeros": "model-00008-of-00015.safetensors",
     "model.layers.27.block_sparse_moe.experts.7.w3.scales": "model-00008-of-00015.safetensors",
-    "model.layers.27.block_sparse_moe.gate.bias": "model-00008-of-00015.safetensors",
-    "model.layers.27.block_sparse_moe.gate.g_idx": "model-00008-of-00015.safetensors",
-    "model.layers.27.block_sparse_moe.gate.qweight": "model-00008-of-00015.safetensors",
-    "model.layers.27.block_sparse_moe.gate.scales": "model-00008-of-00015.safetensors",
+    "model.layers.27.block_sparse_moe.gate.weight": "model-00008-of-00015.safetensors",
     "model.layers.27.input_layernorm.weight": "model-00008-of-00015.safetensors",
     "model.layers.27.post_attention_layernorm.weight": "model-00008-of-00015.safetensors",
     "model.layers.27.self_attn.k_proj.bias": "model-00008-of-00015.safetensors",
@@ -3194,10 +3128,7 @@
     "model.layers.28.block_sparse_moe.experts.7.w3.qweight": "model-00008-of-00015.safetensors",
     "model.layers.28.block_sparse_moe.experts.7.w3.qzeros": "model-00008-of-00015.safetensors",
     "model.layers.28.block_sparse_moe.experts.7.w3.scales": "model-00008-of-00015.safetensors",
-    "model.layers.28.block_sparse_moe.gate.bias": "model-00008-of-00015.safetensors",
-    "model.layers.28.block_sparse_moe.gate.g_idx": "model-00008-of-00015.safetensors",
-    "model.layers.28.block_sparse_moe.gate.qweight": "model-00008-of-00015.safetensors",
-    "model.layers.28.block_sparse_moe.gate.scales": "model-00008-of-00015.safetensors",
+    "model.layers.28.block_sparse_moe.gate.weight": "model-00008-of-00015.safetensors",
     "model.layers.28.input_layernorm.weight": "model-00008-of-00015.safetensors",
     "model.layers.28.post_attention_layernorm.weight": "model-00008-of-00015.safetensors",
     "model.layers.28.self_attn.k_proj.bias": "model-00008-of-00015.safetensors",
@@ -3340,10 +3271,7 @@
     "model.layers.29.block_sparse_moe.experts.7.w3.qweight": "model-00008-of-00015.safetensors",
     "model.layers.29.block_sparse_moe.experts.7.w3.qzeros": "model-00008-of-00015.safetensors",
     "model.layers.29.block_sparse_moe.experts.7.w3.scales": "model-00008-of-00015.safetensors",
-    "model.layers.29.block_sparse_moe.gate.bias": "model-00008-of-00015.safetensors",
-    "model.layers.29.block_sparse_moe.gate.g_idx": "model-00008-of-00015.safetensors",
-    "model.layers.29.block_sparse_moe.gate.qweight": "model-00008-of-00015.safetensors",
-    "model.layers.29.block_sparse_moe.gate.scales": "model-00008-of-00015.safetensors",
+    "model.layers.29.block_sparse_moe.gate.weight": "model-00008-of-00015.safetensors",
     "model.layers.29.input_layernorm.weight": "model-00008-of-00015.safetensors",
     "model.layers.29.post_attention_layernorm.weight": "model-00008-of-00015.safetensors",
     "model.layers.29.self_attn.k_proj.bias": "model-00008-of-00015.safetensors",
@@ -3486,10 +3414,7 @@
     "model.layers.3.block_sparse_moe.experts.7.w3.qweight": "model-00002-of-00015.safetensors",
     "model.layers.3.block_sparse_moe.experts.7.w3.qzeros": "model-00002-of-00015.safetensors",
     "model.layers.3.block_sparse_moe.experts.7.w3.scales": "model-00002-of-00015.safetensors",
-    "model.layers.3.block_sparse_moe.gate.bias": "model-00002-of-00015.safetensors",
-    "model.layers.3.block_sparse_moe.gate.g_idx": "model-00002-of-00015.safetensors",
-    "model.layers.3.block_sparse_moe.gate.qweight": "model-00002-of-00015.safetensors",
-    "model.layers.3.block_sparse_moe.gate.scales": "model-00002-of-00015.safetensors",
+    "model.layers.3.block_sparse_moe.gate.weight": "model-00001-of-00015.safetensors",
     "model.layers.3.input_layernorm.weight": "model-00002-of-00015.safetensors",
     "model.layers.3.post_attention_layernorm.weight": "model-00002-of-00015.safetensors",
     "model.layers.3.self_attn.k_proj.bias": "model-00001-of-00015.safetensors",
@@ -3632,10 +3557,7 @@
     "model.layers.30.block_sparse_moe.experts.7.w3.qweight": "model-00009-of-00015.safetensors",
     "model.layers.30.block_sparse_moe.experts.7.w3.qzeros": "model-00009-of-00015.safetensors",
     "model.layers.30.block_sparse_moe.experts.7.w3.scales": "model-00009-of-00015.safetensors",
-    "model.layers.30.block_sparse_moe.gate.bias": "model-00009-of-00015.safetensors",
-    "model.layers.30.block_sparse_moe.gate.g_idx": "model-00009-of-00015.safetensors",
-    "model.layers.30.block_sparse_moe.gate.qweight": "model-00009-of-00015.safetensors",
-    "model.layers.30.block_sparse_moe.gate.scales": "model-00009-of-00015.safetensors",
+    "model.layers.30.block_sparse_moe.gate.weight": "model-00008-of-00015.safetensors",
     "model.layers.30.input_layernorm.weight": "model-00009-of-00015.safetensors",
     "model.layers.30.post_attention_layernorm.weight": "model-00009-of-00015.safetensors",
     "model.layers.30.self_attn.k_proj.bias": "model-00008-of-00015.safetensors",
@@ -3778,11 +3700,7 @@
     "model.layers.31.block_sparse_moe.experts.7.w3.qweight": "model-00009-of-00015.safetensors",
     "model.layers.31.block_sparse_moe.experts.7.w3.qzeros": "model-00009-of-00015.safetensors",
     "model.layers.31.block_sparse_moe.experts.7.w3.scales": "model-00009-of-00015.safetensors",
-    "model.layers.31.block_sparse_moe.gate.bias": "model-00009-of-00015.safetensors",
-    "model.layers.31.block_sparse_moe.gate.g_idx": "model-00009-of-00015.safetensors",
-    "model.layers.31.block_sparse_moe.gate.qweight": "model-00009-of-00015.safetensors",
-    "model.layers.31.block_sparse_moe.gate.qzeros": "model-00009-of-00015.safetensors",
-    "model.layers.31.block_sparse_moe.gate.scales": "model-00009-of-00015.safetensors",
+    "model.layers.31.block_sparse_moe.gate.weight": "model-00009-of-00015.safetensors",
     "model.layers.31.input_layernorm.weight": "model-00009-of-00015.safetensors",
     "model.layers.31.post_attention_layernorm.weight": "model-00009-of-00015.safetensors",
     "model.layers.31.self_attn.k_proj.bias": "model-00009-of-00015.safetensors",
@@ -3925,10 +3843,7 @@
     "model.layers.32.block_sparse_moe.experts.7.w3.qweight": "model-00009-of-00015.safetensors",
     "model.layers.32.block_sparse_moe.experts.7.w3.qzeros": "model-00009-of-00015.safetensors",
     "model.layers.32.block_sparse_moe.experts.7.w3.scales": "model-00009-of-00015.safetensors",
-    "model.layers.32.block_sparse_moe.gate.bias": "model-00009-of-00015.safetensors",
-    "model.layers.32.block_sparse_moe.gate.g_idx": "model-00009-of-00015.safetensors",
-    "model.layers.32.block_sparse_moe.gate.qweight": "model-00009-of-00015.safetensors",
-    "model.layers.32.block_sparse_moe.gate.scales": "model-00009-of-00015.safetensors",
+    "model.layers.32.block_sparse_moe.gate.weight": "model-00009-of-00015.safetensors",
     "model.layers.32.input_layernorm.weight": "model-00009-of-00015.safetensors",
     "model.layers.32.post_attention_layernorm.weight": "model-00009-of-00015.safetensors",
     "model.layers.32.self_attn.k_proj.bias": "model-00009-of-00015.safetensors",
@@ -4071,10 +3986,7 @@
     "model.layers.33.block_sparse_moe.experts.7.w3.qweight": "model-00009-of-00015.safetensors",
     "model.layers.33.block_sparse_moe.experts.7.w3.qzeros": "model-00009-of-00015.safetensors",
     "model.layers.33.block_sparse_moe.experts.7.w3.scales": "model-00009-of-00015.safetensors",
-    "model.layers.33.block_sparse_moe.gate.bias": "model-00009-of-00015.safetensors",
-    "model.layers.33.block_sparse_moe.gate.g_idx": "model-00009-of-00015.safetensors",
-    "model.layers.33.block_sparse_moe.gate.qweight": "model-00009-of-00015.safetensors",
-    "model.layers.33.block_sparse_moe.gate.scales": "model-00009-of-00015.safetensors",
+    "model.layers.33.block_sparse_moe.gate.weight": "model-00009-of-00015.safetensors",
     "model.layers.33.input_layernorm.weight": "model-00009-of-00015.safetensors",
     "model.layers.33.post_attention_layernorm.weight": "model-00009-of-00015.safetensors",
     "model.layers.33.self_attn.k_proj.bias": "model-00009-of-00015.safetensors",
@@ -4217,10 +4129,7 @@
     "model.layers.34.block_sparse_moe.experts.7.w3.qweight": "model-00010-of-00015.safetensors",
     "model.layers.34.block_sparse_moe.experts.7.w3.qzeros": "model-00010-of-00015.safetensors",
     "model.layers.34.block_sparse_moe.experts.7.w3.scales": "model-00010-of-00015.safetensors",
-    "model.layers.34.block_sparse_moe.gate.bias": "model-00010-of-00015.safetensors",
-    "model.layers.34.block_sparse_moe.gate.g_idx": "model-00010-of-00015.safetensors",
-    "model.layers.34.block_sparse_moe.gate.qweight": "model-00010-of-00015.safetensors",
-    "model.layers.34.block_sparse_moe.gate.scales": "model-00010-of-00015.safetensors",
+    "model.layers.34.block_sparse_moe.gate.weight": "model-00009-of-00015.safetensors",
     "model.layers.34.input_layernorm.weight": "model-00010-of-00015.safetensors",
     "model.layers.34.post_attention_layernorm.weight": "model-00010-of-00015.safetensors",
     "model.layers.34.self_attn.k_proj.bias": "model-00009-of-00015.safetensors",
@@ -4363,10 +4272,7 @@
     "model.layers.35.block_sparse_moe.experts.7.w3.qweight": "model-00010-of-00015.safetensors",
     "model.layers.35.block_sparse_moe.experts.7.w3.qzeros": "model-00010-of-00015.safetensors",
     "model.layers.35.block_sparse_moe.experts.7.w3.scales": "model-00010-of-00015.safetensors",
-    "model.layers.35.block_sparse_moe.gate.bias": "model-00010-of-00015.safetensors",
-    "model.layers.35.block_sparse_moe.gate.g_idx": "model-00010-of-00015.safetensors",
-    "model.layers.35.block_sparse_moe.gate.qweight": "model-00010-of-00015.safetensors",
-    "model.layers.35.block_sparse_moe.gate.scales": "model-00010-of-00015.safetensors",
+    "model.layers.35.block_sparse_moe.gate.weight": "model-00010-of-00015.safetensors",
     "model.layers.35.input_layernorm.weight": "model-00010-of-00015.safetensors",
     "model.layers.35.post_attention_layernorm.weight": "model-00010-of-00015.safetensors",
     "model.layers.35.self_attn.k_proj.bias": "model-00010-of-00015.safetensors",
@@ -4509,10 +4415,7 @@
     "model.layers.36.block_sparse_moe.experts.7.w3.qweight": "model-00010-of-00015.safetensors",
     "model.layers.36.block_sparse_moe.experts.7.w3.qzeros": "model-00010-of-00015.safetensors",
     "model.layers.36.block_sparse_moe.experts.7.w3.scales": "model-00010-of-00015.safetensors",
-    "model.layers.36.block_sparse_moe.gate.bias": "model-00010-of-00015.safetensors",
-    "model.layers.36.block_sparse_moe.gate.g_idx": "model-00010-of-00015.safetensors",
-    "model.layers.36.block_sparse_moe.gate.qweight": "model-00010-of-00015.safetensors",
-    "model.layers.36.block_sparse_moe.gate.scales": "model-00010-of-00015.safetensors",
+    "model.layers.36.block_sparse_moe.gate.weight": "model-00010-of-00015.safetensors",
     "model.layers.36.input_layernorm.weight": "model-00010-of-00015.safetensors",
     "model.layers.36.post_attention_layernorm.weight": "model-00010-of-00015.safetensors",
     "model.layers.36.self_attn.k_proj.bias": "model-00010-of-00015.safetensors",
@@ -4655,10 +4558,7 @@
     "model.layers.37.block_sparse_moe.experts.7.w3.qweight": "model-00011-of-00015.safetensors",
     "model.layers.37.block_sparse_moe.experts.7.w3.qzeros": "model-00011-of-00015.safetensors",
     "model.layers.37.block_sparse_moe.experts.7.w3.scales": "model-00011-of-00015.safetensors",
-    "model.layers.37.block_sparse_moe.gate.bias": "model-00011-of-00015.safetensors",
-    "model.layers.37.block_sparse_moe.gate.g_idx": "model-00011-of-00015.safetensors",
-    "model.layers.37.block_sparse_moe.gate.qweight": "model-00011-of-00015.safetensors",
-    "model.layers.37.block_sparse_moe.gate.scales": "model-00011-of-00015.safetensors",
+    "model.layers.37.block_sparse_moe.gate.weight": "model-00010-of-00015.safetensors",
     "model.layers.37.input_layernorm.weight": "model-00011-of-00015.safetensors",
     "model.layers.37.post_attention_layernorm.weight": "model-00011-of-00015.safetensors",
     "model.layers.37.self_attn.k_proj.bias": "model-00010-of-00015.safetensors",
@@ -4801,10 +4701,7 @@
     "model.layers.38.block_sparse_moe.experts.7.w3.qweight": "model-00011-of-00015.safetensors",
     "model.layers.38.block_sparse_moe.experts.7.w3.qzeros": "model-00011-of-00015.safetensors",
     "model.layers.38.block_sparse_moe.experts.7.w3.scales": "model-00011-of-00015.safetensors",
-    "model.layers.38.block_sparse_moe.gate.bias": "model-00011-of-00015.safetensors",
-    "model.layers.38.block_sparse_moe.gate.g_idx": "model-00011-of-00015.safetensors",
-    "model.layers.38.block_sparse_moe.gate.qweight": "model-00011-of-00015.safetensors",
-    "model.layers.38.block_sparse_moe.gate.scales": "model-00011-of-00015.safetensors",
+    "model.layers.38.block_sparse_moe.gate.weight": "model-00011-of-00015.safetensors",
     "model.layers.38.input_layernorm.weight": "model-00011-of-00015.safetensors",
     "model.layers.38.post_attention_layernorm.weight": "model-00011-of-00015.safetensors",
     "model.layers.38.self_attn.k_proj.bias": "model-00011-of-00015.safetensors",
@@ -4947,11 +4844,7 @@
     "model.layers.39.block_sparse_moe.experts.7.w3.qweight": "model-00011-of-00015.safetensors",
     "model.layers.39.block_sparse_moe.experts.7.w3.qzeros": "model-00011-of-00015.safetensors",
     "model.layers.39.block_sparse_moe.experts.7.w3.scales": "model-00011-of-00015.safetensors",
-    "model.layers.39.block_sparse_moe.gate.bias": "model-00011-of-00015.safetensors",
-    "model.layers.39.block_sparse_moe.gate.g_idx": "model-00011-of-00015.safetensors",
-    "model.layers.39.block_sparse_moe.gate.qweight": "model-00011-of-00015.safetensors",
-    "model.layers.39.block_sparse_moe.gate.qzeros": "model-00011-of-00015.safetensors",
-    "model.layers.39.block_sparse_moe.gate.scales": "model-00011-of-00015.safetensors",
+    "model.layers.39.block_sparse_moe.gate.weight": "model-00011-of-00015.safetensors",
     "model.layers.39.input_layernorm.weight": "model-00011-of-00015.safetensors",
     "model.layers.39.post_attention_layernorm.weight": "model-00011-of-00015.safetensors",
     "model.layers.39.self_attn.k_proj.bias": "model-00011-of-00015.safetensors",
@@ -5094,10 +4987,7 @@
     "model.layers.4.block_sparse_moe.experts.7.w3.qweight": "model-00002-of-00015.safetensors",
     "model.layers.4.block_sparse_moe.experts.7.w3.qzeros": "model-00002-of-00015.safetensors",
     "model.layers.4.block_sparse_moe.experts.7.w3.scales": "model-00002-of-00015.safetensors",
-    "model.layers.4.block_sparse_moe.gate.bias": "model-00002-of-00015.safetensors",
-    "model.layers.4.block_sparse_moe.gate.g_idx": "model-00002-of-00015.safetensors",
-    "model.layers.4.block_sparse_moe.gate.qweight": "model-00002-of-00015.safetensors",
-    "model.layers.4.block_sparse_moe.gate.scales": "model-00002-of-00015.safetensors",
+    "model.layers.4.block_sparse_moe.gate.weight": "model-00002-of-00015.safetensors",
     "model.layers.4.input_layernorm.weight": "model-00002-of-00015.safetensors",
     "model.layers.4.post_attention_layernorm.weight": "model-00002-of-00015.safetensors",
     "model.layers.4.self_attn.k_proj.bias": "model-00002-of-00015.safetensors",
@@ -5240,10 +5130,7 @@
     "model.layers.40.block_sparse_moe.experts.7.w3.qweight": "model-00011-of-00015.safetensors",
     "model.layers.40.block_sparse_moe.experts.7.w3.qzeros": "model-00011-of-00015.safetensors",
     "model.layers.40.block_sparse_moe.experts.7.w3.scales": "model-00011-of-00015.safetensors",
-    "model.layers.40.block_sparse_moe.gate.bias": "model-00011-of-00015.safetensors",
-    "model.layers.40.block_sparse_moe.gate.g_idx": "model-00011-of-00015.safetensors",
-    "model.layers.40.block_sparse_moe.gate.qweight": "model-00011-of-00015.safetensors",
-    "model.layers.40.block_sparse_moe.gate.scales": "model-00011-of-00015.safetensors",
+    "model.layers.40.block_sparse_moe.gate.weight": "model-00011-of-00015.safetensors",
     "model.layers.40.input_layernorm.weight": "model-00011-of-00015.safetensors",
     "model.layers.40.post_attention_layernorm.weight": "model-00011-of-00015.safetensors",
     "model.layers.40.self_attn.k_proj.bias": "model-00011-of-00015.safetensors",
@@ -5386,10 +5273,7 @@
     "model.layers.41.block_sparse_moe.experts.7.w3.qweight": "model-00012-of-00015.safetensors",
     "model.layers.41.block_sparse_moe.experts.7.w3.qzeros": "model-00012-of-00015.safetensors",
     "model.layers.41.block_sparse_moe.experts.7.w3.scales": "model-00012-of-00015.safetensors",
-    "model.layers.41.block_sparse_moe.gate.bias": "model-00012-of-00015.safetensors",
-    "model.layers.41.block_sparse_moe.gate.g_idx": "model-00012-of-00015.safetensors",
-    "model.layers.41.block_sparse_moe.gate.qweight": "model-00012-of-00015.safetensors",
-    "model.layers.41.block_sparse_moe.gate.scales": "model-00012-of-00015.safetensors",
+    "model.layers.41.block_sparse_moe.gate.weight": "model-00011-of-00015.safetensors",
     "model.layers.41.input_layernorm.weight": "model-00012-of-00015.safetensors",
     "model.layers.41.post_attention_layernorm.weight": "model-00012-of-00015.safetensors",
     "model.layers.41.self_attn.k_proj.bias": "model-00011-of-00015.safetensors",
@@ -5532,10 +5416,7 @@
     "model.layers.42.block_sparse_moe.experts.7.w3.qweight": "model-00012-of-00015.safetensors",
     "model.layers.42.block_sparse_moe.experts.7.w3.qzeros": "model-00012-of-00015.safetensors",
     "model.layers.42.block_sparse_moe.experts.7.w3.scales": "model-00012-of-00015.safetensors",
-    "model.layers.42.block_sparse_moe.gate.bias": "model-00012-of-00015.safetensors",
-    "model.layers.42.block_sparse_moe.gate.g_idx": "model-00012-of-00015.safetensors",
-    "model.layers.42.block_sparse_moe.gate.qweight": "model-00012-of-00015.safetensors",
-    "model.layers.42.block_sparse_moe.gate.scales": "model-00012-of-00015.safetensors",
+    "model.layers.42.block_sparse_moe.gate.weight": "model-00012-of-00015.safetensors",
     "model.layers.42.input_layernorm.weight": "model-00012-of-00015.safetensors",
     "model.layers.42.post_attention_layernorm.weight": "model-00012-of-00015.safetensors",
     "model.layers.42.self_attn.k_proj.bias": "model-00012-of-00015.safetensors",
@@ -5678,10 +5559,7 @@
     "model.layers.43.block_sparse_moe.experts.7.w3.qweight": "model-00012-of-00015.safetensors",
     "model.layers.43.block_sparse_moe.experts.7.w3.qzeros": "model-00012-of-00015.safetensors",
     "model.layers.43.block_sparse_moe.experts.7.w3.scales": "model-00012-of-00015.safetensors",
-    "model.layers.43.block_sparse_moe.gate.bias": "model-00012-of-00015.safetensors",
-    "model.layers.43.block_sparse_moe.gate.g_idx": "model-00012-of-00015.safetensors",
-    "model.layers.43.block_sparse_moe.gate.qweight": "model-00012-of-00015.safetensors",
-    "model.layers.43.block_sparse_moe.gate.scales": "model-00012-of-00015.safetensors",
+    "model.layers.43.block_sparse_moe.gate.weight": "model-00012-of-00015.safetensors",
     "model.layers.43.input_layernorm.weight": "model-00012-of-00015.safetensors",
     "model.layers.43.post_attention_layernorm.weight": "model-00012-of-00015.safetensors",
     "model.layers.43.self_attn.k_proj.bias": "model-00012-of-00015.safetensors",
@@ -5824,10 +5702,7 @@
     "model.layers.44.block_sparse_moe.experts.7.w3.qweight": "model-00012-of-00015.safetensors",
     "model.layers.44.block_sparse_moe.experts.7.w3.qzeros": "model-00012-of-00015.safetensors",
     "model.layers.44.block_sparse_moe.experts.7.w3.scales": "model-00012-of-00015.safetensors",
-    "model.layers.44.block_sparse_moe.gate.bias": "model-00012-of-00015.safetensors",
-    "model.layers.44.block_sparse_moe.gate.g_idx": "model-00012-of-00015.safetensors",
-    "model.layers.44.block_sparse_moe.gate.qweight": "model-00012-of-00015.safetensors",
-    "model.layers.44.block_sparse_moe.gate.scales": "model-00012-of-00015.safetensors",
+    "model.layers.44.block_sparse_moe.gate.weight": "model-00012-of-00015.safetensors",
     "model.layers.44.input_layernorm.weight": "model-00012-of-00015.safetensors",
     "model.layers.44.post_attention_layernorm.weight": "model-00012-of-00015.safetensors",
     "model.layers.44.self_attn.k_proj.bias": "model-00012-of-00015.safetensors",
@@ -5970,10 +5845,7 @@
     "model.layers.45.block_sparse_moe.experts.7.w3.qweight": "model-00013-of-00015.safetensors",
     "model.layers.45.block_sparse_moe.experts.7.w3.qzeros": "model-00013-of-00015.safetensors",
     "model.layers.45.block_sparse_moe.experts.7.w3.scales": "model-00013-of-00015.safetensors",
-    "model.layers.45.block_sparse_moe.gate.bias": "model-00013-of-00015.safetensors",
-    "model.layers.45.block_sparse_moe.gate.g_idx": "model-00013-of-00015.safetensors",
-    "model.layers.45.block_sparse_moe.gate.qweight": "model-00013-of-00015.safetensors",
-    "model.layers.45.block_sparse_moe.gate.scales": "model-00013-of-00015.safetensors",
+    "model.layers.45.block_sparse_moe.gate.weight": "model-00012-of-00015.safetensors",
     "model.layers.45.input_layernorm.weight": "model-00013-of-00015.safetensors",
     "model.layers.45.post_attention_layernorm.weight": "model-00013-of-00015.safetensors",
     "model.layers.45.self_attn.k_proj.bias": "model-00012-of-00015.safetensors",
@@ -6116,10 +5988,7 @@
     "model.layers.46.block_sparse_moe.experts.7.w3.qweight": "model-00013-of-00015.safetensors",
     "model.layers.46.block_sparse_moe.experts.7.w3.qzeros": "model-00013-of-00015.safetensors",
     "model.layers.46.block_sparse_moe.experts.7.w3.scales": "model-00013-of-00015.safetensors",
-    "model.layers.46.block_sparse_moe.gate.bias": "model-00013-of-00015.safetensors",
-    "model.layers.46.block_sparse_moe.gate.g_idx": "model-00013-of-00015.safetensors",
-    "model.layers.46.block_sparse_moe.gate.qweight": "model-00013-of-00015.safetensors",
-    "model.layers.46.block_sparse_moe.gate.scales": "model-00013-of-00015.safetensors",
+    "model.layers.46.block_sparse_moe.gate.weight": "model-00013-of-00015.safetensors",
     "model.layers.46.input_layernorm.weight": "model-00013-of-00015.safetensors",
     "model.layers.46.post_attention_layernorm.weight": "model-00013-of-00015.safetensors",
     "model.layers.46.self_attn.k_proj.bias": "model-00013-of-00015.safetensors",
@@ -6262,11 +6131,7 @@
     "model.layers.47.block_sparse_moe.experts.7.w3.qweight": "model-00013-of-00015.safetensors",
     "model.layers.47.block_sparse_moe.experts.7.w3.qzeros": "model-00013-of-00015.safetensors",
     "model.layers.47.block_sparse_moe.experts.7.w3.scales": "model-00013-of-00015.safetensors",
-    "model.layers.47.block_sparse_moe.gate.bias": "model-00013-of-00015.safetensors",
-    "model.layers.47.block_sparse_moe.gate.g_idx": "model-00013-of-00015.safetensors",
-    "model.layers.47.block_sparse_moe.gate.qweight": "model-00013-of-00015.safetensors",
-    "model.layers.47.block_sparse_moe.gate.qzeros": "model-00013-of-00015.safetensors",
-    "model.layers.47.block_sparse_moe.gate.scales": "model-00013-of-00015.safetensors",
+    "model.layers.47.block_sparse_moe.gate.weight": "model-00013-of-00015.safetensors",
     "model.layers.47.input_layernorm.weight": "model-00013-of-00015.safetensors",
     "model.layers.47.post_attention_layernorm.weight": "model-00013-of-00015.safetensors",
     "model.layers.47.self_attn.k_proj.bias": "model-00013-of-00015.safetensors",
@@ -6409,10 +6274,7 @@
     "model.layers.48.block_sparse_moe.experts.7.w3.qweight": "model-00013-of-00015.safetensors",
     "model.layers.48.block_sparse_moe.experts.7.w3.qzeros": "model-00013-of-00015.safetensors",
     "model.layers.48.block_sparse_moe.experts.7.w3.scales": "model-00013-of-00015.safetensors",
-    "model.layers.48.block_sparse_moe.gate.bias": "model-00013-of-00015.safetensors",
-    "model.layers.48.block_sparse_moe.gate.g_idx": "model-00013-of-00015.safetensors",
-    "model.layers.48.block_sparse_moe.gate.qweight": "model-00013-of-00015.safetensors",
-    "model.layers.48.block_sparse_moe.gate.scales": "model-00013-of-00015.safetensors",
+    "model.layers.48.block_sparse_moe.gate.weight": "model-00013-of-00015.safetensors",
     "model.layers.48.input_layernorm.weight": "model-00013-of-00015.safetensors",
     "model.layers.48.post_attention_layernorm.weight": "model-00013-of-00015.safetensors",
     "model.layers.48.self_attn.k_proj.bias": "model-00013-of-00015.safetensors",
@@ -6555,10 +6417,7 @@
     "model.layers.49.block_sparse_moe.experts.7.w3.qweight": "model-00014-of-00015.safetensors",
     "model.layers.49.block_sparse_moe.experts.7.w3.qzeros": "model-00014-of-00015.safetensors",
     "model.layers.49.block_sparse_moe.experts.7.w3.scales": "model-00014-of-00015.safetensors",
-    "model.layers.49.block_sparse_moe.gate.bias": "model-00014-of-00015.safetensors",
-    "model.layers.49.block_sparse_moe.gate.g_idx": "model-00014-of-00015.safetensors",
-    "model.layers.49.block_sparse_moe.gate.qweight": "model-00014-of-00015.safetensors",
-    "model.layers.49.block_sparse_moe.gate.scales": "model-00014-of-00015.safetensors",
+    "model.layers.49.block_sparse_moe.gate.weight": "model-00013-of-00015.safetensors",
     "model.layers.49.input_layernorm.weight": "model-00014-of-00015.safetensors",
     "model.layers.49.post_attention_layernorm.weight": "model-00014-of-00015.safetensors",
     "model.layers.49.self_attn.k_proj.bias": "model-00013-of-00015.safetensors",
@@ -6701,10 +6560,7 @@
     "model.layers.5.block_sparse_moe.experts.7.w3.qweight": "model-00002-of-00015.safetensors",
     "model.layers.5.block_sparse_moe.experts.7.w3.qzeros": "model-00002-of-00015.safetensors",
     "model.layers.5.block_sparse_moe.experts.7.w3.scales": "model-00002-of-00015.safetensors",
-    "model.layers.5.block_sparse_moe.gate.bias": "model-00002-of-00015.safetensors",
-    "model.layers.5.block_sparse_moe.gate.g_idx": "model-00002-of-00015.safetensors",
-    "model.layers.5.block_sparse_moe.gate.qweight": "model-00002-of-00015.safetensors",
-    "model.layers.5.block_sparse_moe.gate.scales": "model-00002-of-00015.safetensors",
+    "model.layers.5.block_sparse_moe.gate.weight": "model-00002-of-00015.safetensors",
     "model.layers.5.input_layernorm.weight": "model-00002-of-00015.safetensors",
     "model.layers.5.post_attention_layernorm.weight": "model-00002-of-00015.safetensors",
     "model.layers.5.self_attn.k_proj.bias": "model-00002-of-00015.safetensors",
@@ -6847,10 +6703,7 @@
     "model.layers.50.block_sparse_moe.experts.7.w3.qweight": "model-00014-of-00015.safetensors",
     "model.layers.50.block_sparse_moe.experts.7.w3.qzeros": "model-00014-of-00015.safetensors",
     "model.layers.50.block_sparse_moe.experts.7.w3.scales": "model-00014-of-00015.safetensors",
-    "model.layers.50.block_sparse_moe.gate.bias": "model-00014-of-00015.safetensors",
-    "model.layers.50.block_sparse_moe.gate.g_idx": "model-00014-of-00015.safetensors",
-    "model.layers.50.block_sparse_moe.gate.qweight": "model-00014-of-00015.safetensors",
-    "model.layers.50.block_sparse_moe.gate.scales": "model-00014-of-00015.safetensors",
+    "model.layers.50.block_sparse_moe.gate.weight": "model-00014-of-00015.safetensors",
     "model.layers.50.input_layernorm.weight": "model-00014-of-00015.safetensors",
     "model.layers.50.post_attention_layernorm.weight": "model-00014-of-00015.safetensors",
     "model.layers.50.self_attn.k_proj.bias": "model-00014-of-00015.safetensors",
@@ -6993,10 +6846,7 @@
     "model.layers.51.block_sparse_moe.experts.7.w3.qweight": "model-00014-of-00015.safetensors",
     "model.layers.51.block_sparse_moe.experts.7.w3.qzeros": "model-00014-of-00015.safetensors",
     "model.layers.51.block_sparse_moe.experts.7.w3.scales": "model-00014-of-00015.safetensors",
-    "model.layers.51.block_sparse_moe.gate.bias": "model-00014-of-00015.safetensors",
-    "model.layers.51.block_sparse_moe.gate.g_idx": "model-00014-of-00015.safetensors",
-    "model.layers.51.block_sparse_moe.gate.qweight": "model-00014-of-00015.safetensors",
-    "model.layers.51.block_sparse_moe.gate.scales": "model-00014-of-00015.safetensors",
+    "model.layers.51.block_sparse_moe.gate.weight": "model-00014-of-00015.safetensors",
     "model.layers.51.input_layernorm.weight": "model-00014-of-00015.safetensors",
     "model.layers.51.post_attention_layernorm.weight": "model-00014-of-00015.safetensors",
     "model.layers.51.self_attn.k_proj.bias": "model-00014-of-00015.safetensors",
@@ -7139,10 +6989,7 @@
     "model.layers.52.block_sparse_moe.experts.7.w3.qweight": "model-00014-of-00015.safetensors",
     "model.layers.52.block_sparse_moe.experts.7.w3.qzeros": "model-00014-of-00015.safetensors",
     "model.layers.52.block_sparse_moe.experts.7.w3.scales": "model-00014-of-00015.safetensors",
-    "model.layers.52.block_sparse_moe.gate.bias": "model-00014-of-00015.safetensors",
-    "model.layers.52.block_sparse_moe.gate.g_idx": "model-00014-of-00015.safetensors",
-    "model.layers.52.block_sparse_moe.gate.qweight": "model-00014-of-00015.safetensors",
-    "model.layers.52.block_sparse_moe.gate.scales": "model-00014-of-00015.safetensors",
+    "model.layers.52.block_sparse_moe.gate.weight": "model-00014-of-00015.safetensors",
     "model.layers.52.input_layernorm.weight": "model-00014-of-00015.safetensors",
     "model.layers.52.post_attention_layernorm.weight": "model-00014-of-00015.safetensors",
     "model.layers.52.self_attn.k_proj.bias": "model-00014-of-00015.safetensors",
@@ -7285,10 +7132,7 @@
     "model.layers.53.block_sparse_moe.experts.7.w3.qweight": "model-00015-of-00015.safetensors",
     "model.layers.53.block_sparse_moe.experts.7.w3.qzeros": "model-00015-of-00015.safetensors",
     "model.layers.53.block_sparse_moe.experts.7.w3.scales": "model-00015-of-00015.safetensors",
-    "model.layers.53.block_sparse_moe.gate.bias": "model-00015-of-00015.safetensors",
-    "model.layers.53.block_sparse_moe.gate.g_idx": "model-00015-of-00015.safetensors",
-    "model.layers.53.block_sparse_moe.gate.qweight": "model-00015-of-00015.safetensors",
-    "model.layers.53.block_sparse_moe.gate.scales": "model-00015-of-00015.safetensors",
+    "model.layers.53.block_sparse_moe.gate.weight": "model-00014-of-00015.safetensors",
     "model.layers.53.input_layernorm.weight": "model-00015-of-00015.safetensors",
     "model.layers.53.post_attention_layernorm.weight": "model-00015-of-00015.safetensors",
     "model.layers.53.self_attn.k_proj.bias": "model-00014-of-00015.safetensors",
@@ -7431,10 +7275,7 @@
     "model.layers.54.block_sparse_moe.experts.7.w3.qweight": "model-00015-of-00015.safetensors",
     "model.layers.54.block_sparse_moe.experts.7.w3.qzeros": "model-00015-of-00015.safetensors",
     "model.layers.54.block_sparse_moe.experts.7.w3.scales": "model-00015-of-00015.safetensors",
-    "model.layers.54.block_sparse_moe.gate.bias": "model-00015-of-00015.safetensors",
-    "model.layers.54.block_sparse_moe.gate.g_idx": "model-00015-of-00015.safetensors",
-    "model.layers.54.block_sparse_moe.gate.qweight": "model-00015-of-00015.safetensors",
-    "model.layers.54.block_sparse_moe.gate.scales": "model-00015-of-00015.safetensors",
+    "model.layers.54.block_sparse_moe.gate.weight": "model-00015-of-00015.safetensors",
     "model.layers.54.input_layernorm.weight": "model-00015-of-00015.safetensors",
     "model.layers.54.post_attention_layernorm.weight": "model-00015-of-00015.safetensors",
     "model.layers.54.self_attn.k_proj.bias": "model-00015-of-00015.safetensors",
@@ -7577,11 +7418,7 @@
     "model.layers.55.block_sparse_moe.experts.7.w3.qweight": "model-00015-of-00015.safetensors",
     "model.layers.55.block_sparse_moe.experts.7.w3.qzeros": "model-00015-of-00015.safetensors",
     "model.layers.55.block_sparse_moe.experts.7.w3.scales": "model-00015-of-00015.safetensors",
-    "model.layers.55.block_sparse_moe.gate.bias": "model-00015-of-00015.safetensors",
-    "model.layers.55.block_sparse_moe.gate.g_idx": "model-00015-of-00015.safetensors",
-    "model.layers.55.block_sparse_moe.gate.qweight": "model-00015-of-00015.safetensors",
-    "model.layers.55.block_sparse_moe.gate.qzeros": "model-00015-of-00015.safetensors",
-    "model.layers.55.block_sparse_moe.gate.scales": "model-00015-of-00015.safetensors",
+    "model.layers.55.block_sparse_moe.gate.weight": "model-00015-of-00015.safetensors",
     "model.layers.55.input_layernorm.weight": "model-00015-of-00015.safetensors",
     "model.layers.55.post_attention_layernorm.weight": "model-00015-of-00015.safetensors",
     "model.layers.55.self_attn.k_proj.bias": "model-00015-of-00015.safetensors",
@@ -7724,10 +7561,7 @@
     "model.layers.6.block_sparse_moe.experts.7.w3.qweight": "model-00002-of-00015.safetensors",
     "model.layers.6.block_sparse_moe.experts.7.w3.qzeros": "model-00002-of-00015.safetensors",
     "model.layers.6.block_sparse_moe.experts.7.w3.scales": "model-00002-of-00015.safetensors",
-    "model.layers.6.block_sparse_moe.gate.bias": "model-00002-of-00015.safetensors",
-    "model.layers.6.block_sparse_moe.gate.g_idx": "model-00002-of-00015.safetensors",
-    "model.layers.6.block_sparse_moe.gate.qweight": "model-00002-of-00015.safetensors",
-    "model.layers.6.block_sparse_moe.gate.scales": "model-00002-of-00015.safetensors",
+    "model.layers.6.block_sparse_moe.gate.weight": "model-00002-of-00015.safetensors",
     "model.layers.6.input_layernorm.weight": "model-00002-of-00015.safetensors",
     "model.layers.6.post_attention_layernorm.weight": "model-00002-of-00015.safetensors",
     "model.layers.6.self_attn.k_proj.bias": "model-00002-of-00015.safetensors",
@@ -7870,11 +7704,7 @@
     "model.layers.7.block_sparse_moe.experts.7.w3.qweight": "model-00003-of-00015.safetensors",
     "model.layers.7.block_sparse_moe.experts.7.w3.qzeros": "model-00003-of-00015.safetensors",
     "model.layers.7.block_sparse_moe.experts.7.w3.scales": "model-00003-of-00015.safetensors",
-    "model.layers.7.block_sparse_moe.gate.bias": "model-00003-of-00015.safetensors",
-    "model.layers.7.block_sparse_moe.gate.g_idx": "model-00003-of-00015.safetensors",
-    "model.layers.7.block_sparse_moe.gate.qweight": "model-00003-of-00015.safetensors",
-    "model.layers.7.block_sparse_moe.gate.qzeros": "model-00003-of-00015.safetensors",
-    "model.layers.7.block_sparse_moe.gate.scales": "model-00003-of-00015.safetensors",
+    "model.layers.7.block_sparse_moe.gate.weight": "model-00002-of-00015.safetensors",
     "model.layers.7.input_layernorm.weight": "model-00003-of-00015.safetensors",
     "model.layers.7.post_attention_layernorm.weight": "model-00003-of-00015.safetensors",
     "model.layers.7.self_attn.k_proj.bias": "model-00002-of-00015.safetensors",
@@ -8017,10 +7847,7 @@
     "model.layers.8.block_sparse_moe.experts.7.w3.qweight": "model-00003-of-00015.safetensors",
     "model.layers.8.block_sparse_moe.experts.7.w3.qzeros": "model-00003-of-00015.safetensors",
     "model.layers.8.block_sparse_moe.experts.7.w3.scales": "model-00003-of-00015.safetensors",
-    "model.layers.8.block_sparse_moe.gate.bias": "model-00003-of-00015.safetensors",
-    "model.layers.8.block_sparse_moe.gate.g_idx": "model-00003-of-00015.safetensors",
-    "model.layers.8.block_sparse_moe.gate.qweight": "model-00003-of-00015.safetensors",
-    "model.layers.8.block_sparse_moe.gate.scales": "model-00003-of-00015.safetensors",
+    "model.layers.8.block_sparse_moe.gate.weight": "model-00003-of-00015.safetensors",
     "model.layers.8.input_layernorm.weight": "model-00003-of-00015.safetensors",
     "model.layers.8.post_attention_layernorm.weight": "model-00003-of-00015.safetensors",
     "model.layers.8.self_attn.k_proj.bias": "model-00003-of-00015.safetensors",
@@ -8163,10 +7990,7 @@
     "model.layers.9.block_sparse_moe.experts.7.w3.qweight": "model-00003-of-00015.safetensors",
     "model.layers.9.block_sparse_moe.experts.7.w3.qzeros": "model-00003-of-00015.safetensors",
     "model.layers.9.block_sparse_moe.experts.7.w3.scales": "model-00003-of-00015.safetensors",
-    "model.layers.9.block_sparse_moe.gate.bias": "model-00003-of-00015.safetensors",
-    "model.layers.9.block_sparse_moe.gate.g_idx": "model-00003-of-00015.safetensors",
-    "model.layers.9.block_sparse_moe.gate.qweight": "model-00003-of-00015.safetensors",
-    "model.layers.9.block_sparse_moe.gate.scales": "model-00003-of-00015.safetensors",
+    "model.layers.9.block_sparse_moe.gate.weight": "model-00003-of-00015.safetensors",
     "model.layers.9.input_layernorm.weight": "model-00003-of-00015.safetensors",
     "model.layers.9.post_attention_layernorm.weight": "model-00003-of-00015.safetensors",
     "model.layers.9.self_attn.k_proj.bias": "model-00003-of-00015.safetensors",