Upload 13 files

config.json

@@ -1,5 +1,5 @@
 {
-  "_name_or_path": "TroyDoesAI/MermaidMistral",
+  "_name_or_path": ".\\Mermaid\\",
   "architectures": [
     "MixtralForCausalLM"
   ],
@@ -24,7 +24,7 @@
   "sliding_window": null,
   "tie_word_embeddings": false,
   "torch_dtype": "float16",
-  "transformers_version": "4.38.1",
+  "transformers_version": "4.37.2",
   "use_cache": true,
   "vocab_size": 32000
 }

generation_config.json

@@ -1,7 +1,6 @@
 {
   "_from_model_config": true,
   "bos_token_id": 1,
-  "eos_token_id": 32000,
-  "transformers_version": "4.37.2",
-  "use_cache": false
+  "eos_token_id": 2,
+  "transformers_version": "4.37.2"
 }

model-00001-of-00006.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:44522bcf3f25332b8e004e7cfe4f6bfcb38d02e3586cdf05d1500041b8c12b9e
-size 4993541568
+oid sha256:6055c96a28b3790d73b13472f2c9ac64169de72e13baf585c6e23124ad173bbd
+size 4993525184

model-00002-of-00006.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f1a8647596fab63acafda05785556f7299166dcc256b523f61740bc6105b95f3
+oid sha256:774822e966ff0bdc71d0a42ff3f06a2de9026566f81a2cefb5846772d2a9f34b
 size 4932724792

model-00003-of-00006.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:431bae108d3e378fecf21e39329de635764f4f7d0c55cc2abd76af904580938f
+oid sha256:1f42ff3645133437984b1c2549cdd6898d6b11ccda8e228494d0c70b2e83dec3
 size 4966262448

model-00004-of-00006.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8d196420dea2fd74e4e8f2de36cd013e0cccea65c38145bac14d18264cf6d6bc
+oid sha256:cb643ce39094f7cc3d0d1d53ae09034c8611bf9d7946329cf1d0a3065367bf11
 size 4966262448

model-00005-of-00006.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6ba1e28683a7e29de6ecb57f968e07521866240c9c3f253563b7425caf89c652
+oid sha256:b69fa0caea5ab041722fe22e54b7a66c691517e63616041820ae2f2b081c4670
 size 4932741456

model-00006-of-00006.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4853b91046d24b059eba20d81922fd3eef8582d30ba850c79d4ac76f130a6d64
-size 966829248
+oid sha256:9a8e4e96f42f6abef053054d5790352094747d295d42380ec0aecd6f42ca48a4
+size 966812864

model.safetensors.index.json

@@ -1 +1,426 @@
-{"metadata": {"mergekit_version": "0.0.4.1", "total_size": 25758277632}, "weight_map": {"model.embed_tokens.weight": "model-00001-of-00003.safetensors", "model.norm.weight": "model-00001-of-00003.safetensors", "lm_head.weight": "model-00001-of-00003.safetensors", "model.layers.0.input_layernorm.weight": "model-00001-of-00003.safetensors", "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", "model.layers.0.block_sparse_moe.experts.0.w3.weight": "model-00001-of-00003.safetensors", "model.layers.0.block_sparse_moe.experts.1.w3.weight": "model-00001-of-00003.safetensors", "model.layers.0.block_sparse_moe.experts.0.w1.weight": "model-00001-of-00003.safetensors", "model.layers.0.block_sparse_moe.experts.1.w1.weight": "model-00001-of-00003.safetensors", "model.layers.0.block_sparse_moe.experts.0.w2.weight": "model-00001-of-00003.safetensors", "model.layers.0.block_sparse_moe.experts.1.w2.weight": "model-00001-of-00003.safetensors", "model.layers.1.input_layernorm.weight": "model-00001-of-00003.safetensors", "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", "model.layers.1.block_sparse_moe.experts.0.w3.weight": "model-00001-of-00003.safetensors", "model.layers.1.block_sparse_moe.experts.1.w3.weight": "model-00001-of-00003.safetensors", "model.layers.1.block_sparse_moe.experts.0.w1.weight": "model-00001-of-00003.safetensors", "model.layers.1.block_sparse_moe.experts.1.w1.weight": "model-00001-of-00003.safetensors", "model.layers.1.block_sparse_moe.experts.0.w2.weight": "model-00001-of-00003.safetensors", "model.layers.1.block_sparse_moe.experts.1.w2.weight": "model-00001-of-00003.safetensors", "model.layers.2.input_layernorm.weight": "model-00001-of-00003.safetensors", "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", "model.layers.2.block_sparse_moe.experts.0.w3.weight": "model-00001-of-00003.safetensors", "model.layers.2.block_sparse_moe.experts.1.w3.weight": "model-00001-of-00003.safetensors", "model.layers.2.block_sparse_moe.experts.0.w1.weight": "model-00001-of-00003.safetensors", "model.layers.2.block_sparse_moe.experts.1.w1.weight": "model-00001-of-00003.safetensors", "model.layers.2.block_sparse_moe.experts.0.w2.weight": "model-00001-of-00003.safetensors", "model.layers.2.block_sparse_moe.experts.1.w2.weight": "model-00001-of-00003.safetensors", "model.layers.3.input_layernorm.weight": "model-00001-of-00003.safetensors", "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", "model.layers.3.block_sparse_moe.experts.0.w3.weight": "model-00001-of-00003.safetensors", "model.layers.3.block_sparse_moe.experts.1.w3.weight": "model-00001-of-00003.safetensors", "model.layers.3.block_sparse_moe.experts.0.w1.weight": "model-00001-of-00003.safetensors", "model.layers.3.block_sparse_moe.experts.1.w1.weight": "model-00001-of-00003.safetensors", "model.layers.3.block_sparse_moe.experts.0.w2.weight": "model-00001-of-00003.safetensors", "model.layers.3.block_sparse_moe.experts.1.w2.weight": "model-00001-of-00003.safetensors", "model.layers.4.input_layernorm.weight": "model-00001-of-00003.safetensors", "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", "model.layers.4.block_sparse_moe.experts.0.w3.weight": "model-00001-of-00003.safetensors", "model.layers.4.block_sparse_moe.experts.1.w3.weight": "model-00001-of-00003.safetensors", "model.layers.4.block_sparse_moe.experts.0.w1.weight": "model-00001-of-00003.safetensors", "model.layers.4.block_sparse_moe.experts.1.w1.weight": "model-00001-of-00003.safetensors", "model.layers.4.block_sparse_moe.experts.0.w2.weight": "model-00001-of-00003.safetensors", "model.layers.4.block_sparse_moe.experts.1.w2.weight": "model-00001-of-00003.safetensors", "model.layers.5.input_layernorm.weight": "model-00001-of-00003.safetensors", "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", "model.layers.5.block_sparse_moe.experts.0.w3.weight": "model-00001-of-00003.safetensors", "model.layers.5.block_sparse_moe.experts.1.w3.weight": "model-00001-of-00003.safetensors", "model.layers.5.block_sparse_moe.experts.0.w1.weight": "model-00001-of-00003.safetensors", "model.layers.5.block_sparse_moe.experts.1.w1.weight": "model-00001-of-00003.safetensors", "model.layers.5.block_sparse_moe.experts.0.w2.weight": "model-00001-of-00003.safetensors", "model.layers.5.block_sparse_moe.experts.1.w2.weight": "model-00001-of-00003.safetensors", "model.layers.6.input_layernorm.weight": "model-00001-of-00003.safetensors", "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", "model.layers.6.block_sparse_moe.experts.0.w3.weight": "model-00001-of-00003.safetensors", "model.layers.6.block_sparse_moe.experts.1.w3.weight": "model-00001-of-00003.safetensors", "model.layers.6.block_sparse_moe.experts.0.w1.weight": "model-00001-of-00003.safetensors", "model.layers.6.block_sparse_moe.experts.1.w1.weight": "model-00001-of-00003.safetensors", "model.layers.6.block_sparse_moe.experts.0.w2.weight": "model-00001-of-00003.safetensors", "model.layers.6.block_sparse_moe.experts.1.w2.weight": "model-00001-of-00003.safetensors", "model.layers.7.input_layernorm.weight": "model-00001-of-00003.safetensors", "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", "model.layers.7.block_sparse_moe.experts.0.w3.weight": "model-00001-of-00003.safetensors", "model.layers.7.block_sparse_moe.experts.1.w3.weight": "model-00001-of-00003.safetensors", "model.layers.7.block_sparse_moe.experts.0.w1.weight": "model-00001-of-00003.safetensors", "model.layers.7.block_sparse_moe.experts.1.w1.weight": "model-00001-of-00003.safetensors", "model.layers.7.block_sparse_moe.experts.0.w2.weight": "model-00001-of-00003.safetensors", "model.layers.7.block_sparse_moe.experts.1.w2.weight": "model-00001-of-00003.safetensors", "model.layers.8.input_layernorm.weight": "model-00001-of-00003.safetensors", "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", "model.layers.8.block_sparse_moe.experts.0.w3.weight": "model-00001-of-00003.safetensors", "model.layers.8.block_sparse_moe.experts.1.w3.weight": "model-00001-of-00003.safetensors", "model.layers.8.block_sparse_moe.experts.0.w1.weight": "model-00001-of-00003.safetensors", "model.layers.8.block_sparse_moe.experts.1.w1.weight": "model-00001-of-00003.safetensors", "model.layers.8.block_sparse_moe.experts.0.w2.weight": "model-00001-of-00003.safetensors", "model.layers.8.block_sparse_moe.experts.1.w2.weight": "model-00001-of-00003.safetensors", "model.layers.9.input_layernorm.weight": "model-00001-of-00003.safetensors", "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", "model.layers.9.block_sparse_moe.experts.0.w3.weight": "model-00001-of-00003.safetensors", "model.layers.9.block_sparse_moe.experts.1.w3.weight": "model-00001-of-00003.safetensors", "model.layers.9.block_sparse_moe.experts.0.w1.weight": "model-00001-of-00003.safetensors", "model.layers.9.block_sparse_moe.experts.1.w1.weight": "model-00001-of-00003.safetensors", "model.layers.9.block_sparse_moe.experts.0.w2.weight": "model-00001-of-00003.safetensors", "model.layers.9.block_sparse_moe.experts.1.w2.weight": "model-00001-of-00003.safetensors", "model.layers.10.input_layernorm.weight": "model-00001-of-00003.safetensors", "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", "model.layers.10.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", "model.layers.10.block_sparse_moe.experts.0.w3.weight": "model-00001-of-00003.safetensors", "model.layers.10.block_sparse_moe.experts.1.w3.weight": "model-00001-of-00003.safetensors", "model.layers.10.block_sparse_moe.experts.0.w1.weight": "model-00001-of-00003.safetensors", "model.layers.10.block_sparse_moe.experts.1.w1.weight": "model-00001-of-00003.safetensors", "model.layers.10.block_sparse_moe.experts.0.w2.weight": "model-00001-of-00003.safetensors", "model.layers.10.block_sparse_moe.experts.1.w2.weight": "model-00001-of-00003.safetensors", "model.layers.11.input_layernorm.weight": "model-00001-of-00003.safetensors", "model.layers.11.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", "model.layers.11.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", "model.layers.11.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", "model.layers.11.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", "model.layers.11.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", "model.layers.11.block_sparse_moe.experts.0.w3.weight": "model-00001-of-00003.safetensors", "model.layers.11.block_sparse_moe.experts.1.w3.weight": "model-00001-of-00003.safetensors", "model.layers.11.block_sparse_moe.experts.0.w1.weight": "model-00001-of-00003.safetensors", "model.layers.11.block_sparse_moe.experts.1.w1.weight": "model-00001-of-00003.safetensors", "model.layers.11.block_sparse_moe.experts.0.w2.weight": "model-00001-of-00003.safetensors", "model.layers.11.block_sparse_moe.experts.1.w2.weight": "model-00001-of-00003.safetensors", "model.layers.12.input_layernorm.weight": "model-00001-of-00003.safetensors", "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", "model.layers.12.block_sparse_moe.experts.0.w3.weight": "model-00002-of-00003.safetensors", "model.layers.12.block_sparse_moe.experts.1.w3.weight": "model-00002-of-00003.safetensors", "model.layers.12.block_sparse_moe.experts.0.w1.weight": "model-00002-of-00003.safetensors", "model.layers.12.block_sparse_moe.experts.1.w1.weight": "model-00002-of-00003.safetensors", "model.layers.12.block_sparse_moe.experts.0.w2.weight": "model-00002-of-00003.safetensors", "model.layers.12.block_sparse_moe.experts.1.w2.weight": "model-00002-of-00003.safetensors", "model.layers.13.input_layernorm.weight": "model-00002-of-00003.safetensors", "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", "model.layers.13.block_sparse_moe.experts.0.w3.weight": "model-00002-of-00003.safetensors", "model.layers.13.block_sparse_moe.experts.1.w3.weight": "model-00002-of-00003.safetensors", "model.layers.13.block_sparse_moe.experts.0.w1.weight": "model-00002-of-00003.safetensors", "model.layers.13.block_sparse_moe.experts.1.w1.weight": "model-00002-of-00003.safetensors", "model.layers.13.block_sparse_moe.experts.0.w2.weight": "model-00002-of-00003.safetensors", "model.layers.13.block_sparse_moe.experts.1.w2.weight": "model-00002-of-00003.safetensors", "model.layers.14.input_layernorm.weight": "model-00002-of-00003.safetensors", "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", "model.layers.14.block_sparse_moe.experts.0.w3.weight": "model-00002-of-00003.safetensors", "model.layers.14.block_sparse_moe.experts.1.w3.weight": "model-00002-of-00003.safetensors", "model.layers.14.block_sparse_moe.experts.0.w1.weight": "model-00002-of-00003.safetensors", "model.layers.14.block_sparse_moe.experts.1.w1.weight": "model-00002-of-00003.safetensors", "model.layers.14.block_sparse_moe.experts.0.w2.weight": "model-00002-of-00003.safetensors", "model.layers.14.block_sparse_moe.experts.1.w2.weight": "model-00002-of-00003.safetensors", "model.layers.15.input_layernorm.weight": "model-00002-of-00003.safetensors", "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", "model.layers.15.block_sparse_moe.experts.0.w3.weight": "model-00002-of-00003.safetensors", "model.layers.15.block_sparse_moe.experts.1.w3.weight": "model-00002-of-00003.safetensors", "model.layers.15.block_sparse_moe.experts.0.w1.weight": "model-00002-of-00003.safetensors", "model.layers.15.block_sparse_moe.experts.1.w1.weight": "model-00002-of-00003.safetensors", "model.layers.15.block_sparse_moe.experts.0.w2.weight": "model-00002-of-00003.safetensors", "model.layers.15.block_sparse_moe.experts.1.w2.weight": "model-00002-of-00003.safetensors", "model.layers.16.input_layernorm.weight": "model-00002-of-00003.safetensors", "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", "model.layers.16.block_sparse_moe.experts.0.w3.weight": "model-00002-of-00003.safetensors", "model.layers.16.block_sparse_moe.experts.1.w3.weight": "model-00002-of-00003.safetensors", "model.layers.16.block_sparse_moe.experts.0.w1.weight": "model-00002-of-00003.safetensors", "model.layers.16.block_sparse_moe.experts.1.w1.weight": "model-00002-of-00003.safetensors", "model.layers.16.block_sparse_moe.experts.0.w2.weight": "model-00002-of-00003.safetensors", "model.layers.16.block_sparse_moe.experts.1.w2.weight": "model-00002-of-00003.safetensors", "model.layers.17.input_layernorm.weight": "model-00002-of-00003.safetensors", "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", "model.layers.17.block_sparse_moe.experts.0.w3.weight": "model-00002-of-00003.safetensors", "model.layers.17.block_sparse_moe.experts.1.w3.weight": "model-00002-of-00003.safetensors", "model.layers.17.block_sparse_moe.experts.0.w1.weight": "model-00002-of-00003.safetensors", "model.layers.17.block_sparse_moe.experts.1.w1.weight": "model-00002-of-00003.safetensors", "model.layers.17.block_sparse_moe.experts.0.w2.weight": "model-00002-of-00003.safetensors", "model.layers.17.block_sparse_moe.experts.1.w2.weight": "model-00002-of-00003.safetensors", "model.layers.18.input_layernorm.weight": "model-00002-of-00003.safetensors", "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", "model.layers.18.block_sparse_moe.experts.0.w3.weight": "model-00002-of-00003.safetensors", "model.layers.18.block_sparse_moe.experts.1.w3.weight": "model-00002-of-00003.safetensors", "model.layers.18.block_sparse_moe.experts.0.w1.weight": "model-00002-of-00003.safetensors", "model.layers.18.block_sparse_moe.experts.1.w1.weight": "model-00002-of-00003.safetensors", "model.layers.18.block_sparse_moe.experts.0.w2.weight": "model-00002-of-00003.safetensors", "model.layers.18.block_sparse_moe.experts.1.w2.weight": "model-00002-of-00003.safetensors", "model.layers.19.input_layernorm.weight": "model-00002-of-00003.safetensors", "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", "model.layers.19.block_sparse_moe.experts.0.w3.weight": "model-00002-of-00003.safetensors", "model.layers.19.block_sparse_moe.experts.1.w3.weight": "model-00002-of-00003.safetensors", "model.layers.19.block_sparse_moe.experts.0.w1.weight": "model-00002-of-00003.safetensors", "model.layers.19.block_sparse_moe.experts.1.w1.weight": "model-00002-of-00003.safetensors", "model.layers.19.block_sparse_moe.experts.0.w2.weight": "model-00002-of-00003.safetensors", "model.layers.19.block_sparse_moe.experts.1.w2.weight": "model-00002-of-00003.safetensors", "model.layers.20.input_layernorm.weight": "model-00002-of-00003.safetensors", "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", "model.layers.20.block_sparse_moe.experts.0.w3.weight": "model-00002-of-00003.safetensors", "model.layers.20.block_sparse_moe.experts.1.w3.weight": "model-00002-of-00003.safetensors", "model.layers.20.block_sparse_moe.experts.0.w1.weight": "model-00002-of-00003.safetensors", "model.layers.20.block_sparse_moe.experts.1.w1.weight": "model-00002-of-00003.safetensors", "model.layers.20.block_sparse_moe.experts.0.w2.weight": "model-00002-of-00003.safetensors", "model.layers.20.block_sparse_moe.experts.1.w2.weight": "model-00002-of-00003.safetensors", "model.layers.21.input_layernorm.weight": "model-00002-of-00003.safetensors", "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", "model.layers.21.block_sparse_moe.experts.0.w3.weight": "model-00002-of-00003.safetensors", "model.layers.21.block_sparse_moe.experts.1.w3.weight": "model-00002-of-00003.safetensors", "model.layers.21.block_sparse_moe.experts.0.w1.weight": "model-00002-of-00003.safetensors", "model.layers.21.block_sparse_moe.experts.1.w1.weight": "model-00002-of-00003.safetensors", "model.layers.21.block_sparse_moe.experts.0.w2.weight": "model-00002-of-00003.safetensors", "model.layers.21.block_sparse_moe.experts.1.w2.weight": "model-00002-of-00003.safetensors", "model.layers.22.input_layernorm.weight": "model-00002-of-00003.safetensors", "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", "model.layers.22.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", "model.layers.22.block_sparse_moe.experts.0.w3.weight": "model-00002-of-00003.safetensors", "model.layers.22.block_sparse_moe.experts.1.w3.weight": "model-00002-of-00003.safetensors", "model.layers.22.block_sparse_moe.experts.0.w1.weight": "model-00002-of-00003.safetensors", "model.layers.22.block_sparse_moe.experts.1.w1.weight": "model-00002-of-00003.safetensors", "model.layers.22.block_sparse_moe.experts.0.w2.weight": "model-00002-of-00003.safetensors", "model.layers.22.block_sparse_moe.experts.1.w2.weight": "model-00002-of-00003.safetensors", "model.layers.23.input_layernorm.weight": "model-00002-of-00003.safetensors", "model.layers.23.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", "model.layers.23.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", "model.layers.23.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", "model.layers.23.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", "model.layers.23.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", "model.layers.23.block_sparse_moe.experts.0.w3.weight": "model-00002-of-00003.safetensors", "model.layers.23.block_sparse_moe.experts.1.w3.weight": "model-00002-of-00003.safetensors", "model.layers.23.block_sparse_moe.experts.0.w1.weight": "model-00002-of-00003.safetensors", "model.layers.23.block_sparse_moe.experts.1.w1.weight": "model-00002-of-00003.safetensors", "model.layers.23.block_sparse_moe.experts.0.w2.weight": "model-00002-of-00003.safetensors", "model.layers.23.block_sparse_moe.experts.1.w2.weight": "model-00002-of-00003.safetensors", "model.layers.24.input_layernorm.weight": "model-00002-of-00003.safetensors", "model.layers.24.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", "model.layers.24.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", "model.layers.24.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", "model.layers.24.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", "model.layers.24.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", "model.layers.24.block_sparse_moe.experts.0.w3.weight": "model-00002-of-00003.safetensors", "model.layers.24.block_sparse_moe.experts.1.w3.weight": "model-00002-of-00003.safetensors", "model.layers.24.block_sparse_moe.experts.0.w1.weight": "model-00002-of-00003.safetensors", "model.layers.24.block_sparse_moe.experts.1.w1.weight": "model-00003-of-00003.safetensors", "model.layers.24.block_sparse_moe.experts.0.w2.weight": "model-00003-of-00003.safetensors", "model.layers.24.block_sparse_moe.experts.1.w2.weight": "model-00003-of-00003.safetensors", "model.layers.25.input_layernorm.weight": "model-00003-of-00003.safetensors", "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00003.safetensors", "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00003.safetensors", "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00003.safetensors", "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00003.safetensors", "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00003.safetensors", "model.layers.25.block_sparse_moe.experts.0.w3.weight": "model-00003-of-00003.safetensors", "model.layers.25.block_sparse_moe.experts.1.w3.weight": "model-00003-of-00003.safetensors", "model.layers.25.block_sparse_moe.experts.0.w1.weight": "model-00003-of-00003.safetensors", "model.layers.25.block_sparse_moe.experts.1.w1.weight": "model-00003-of-00003.safetensors", "model.layers.25.block_sparse_moe.experts.0.w2.weight": "model-00003-of-00003.safetensors", "model.layers.25.block_sparse_moe.experts.1.w2.weight": "model-00003-of-00003.safetensors", "model.layers.26.input_layernorm.weight": "model-00003-of-00003.safetensors", "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00003.safetensors", "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00003.safetensors", "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00003.safetensors", "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00003.safetensors", "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00003.safetensors", "model.layers.26.block_sparse_moe.experts.0.w3.weight": "model-00003-of-00003.safetensors", "model.layers.26.block_sparse_moe.experts.1.w3.weight": "model-00003-of-00003.safetensors", "model.layers.26.block_sparse_moe.experts.0.w1.weight": "model-00003-of-00003.safetensors", "model.layers.26.block_sparse_moe.experts.1.w1.weight": "model-00003-of-00003.safetensors", "model.layers.26.block_sparse_moe.experts.0.w2.weight": "model-00003-of-00003.safetensors", "model.layers.26.block_sparse_moe.experts.1.w2.weight": "model-00003-of-00003.safetensors", "model.layers.27.input_layernorm.weight": "model-00003-of-00003.safetensors", "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00003.safetensors", "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00003.safetensors", "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00003.safetensors", "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00003.safetensors", "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00003.safetensors", "model.layers.27.block_sparse_moe.experts.0.w3.weight": "model-00003-of-00003.safetensors", "model.layers.27.block_sparse_moe.experts.1.w3.weight": "model-00003-of-00003.safetensors", "model.layers.27.block_sparse_moe.experts.0.w1.weight": "model-00003-of-00003.safetensors", "model.layers.27.block_sparse_moe.experts.1.w1.weight": "model-00003-of-00003.safetensors", "model.layers.27.block_sparse_moe.experts.0.w2.weight": "model-00003-of-00003.safetensors", "model.layers.27.block_sparse_moe.experts.1.w2.weight": "model-00003-of-00003.safetensors", "model.layers.28.input_layernorm.weight": "model-00003-of-00003.safetensors", "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00003.safetensors", "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00003.safetensors", "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00003.safetensors", "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00003.safetensors", "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00003.safetensors", "model.layers.28.block_sparse_moe.experts.0.w3.weight": "model-00003-of-00003.safetensors", "model.layers.28.block_sparse_moe.experts.1.w3.weight": "model-00003-of-00003.safetensors", "model.layers.28.block_sparse_moe.experts.0.w1.weight": "model-00003-of-00003.safetensors", "model.layers.28.block_sparse_moe.experts.1.w1.weight": "model-00003-of-00003.safetensors", "model.layers.28.block_sparse_moe.experts.0.w2.weight": "model-00003-of-00003.safetensors", "model.layers.28.block_sparse_moe.experts.1.w2.weight": "model-00003-of-00003.safetensors", "model.layers.29.input_layernorm.weight": "model-00003-of-00003.safetensors", "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00003.safetensors", "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00003.safetensors", "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00003.safetensors", "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00003.safetensors", "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00003.safetensors", "model.layers.29.block_sparse_moe.experts.0.w3.weight": "model-00003-of-00003.safetensors", "model.layers.29.block_sparse_moe.experts.1.w3.weight": "model-00003-of-00003.safetensors", "model.layers.29.block_sparse_moe.experts.0.w1.weight": "model-00003-of-00003.safetensors", "model.layers.29.block_sparse_moe.experts.1.w1.weight": "model-00003-of-00003.safetensors", "model.layers.29.block_sparse_moe.experts.0.w2.weight": "model-00003-of-00003.safetensors", "model.layers.29.block_sparse_moe.experts.1.w2.weight": "model-00003-of-00003.safetensors", "model.layers.30.input_layernorm.weight": "model-00003-of-00003.safetensors", "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00003.safetensors", "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00003.safetensors", "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00003.safetensors", "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00003.safetensors", "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00003.safetensors", "model.layers.30.block_sparse_moe.experts.0.w3.weight": "model-00003-of-00003.safetensors", "model.layers.30.block_sparse_moe.experts.1.w3.weight": "model-00003-of-00003.safetensors", "model.layers.30.block_sparse_moe.experts.0.w1.weight": "model-00003-of-00003.safetensors", "model.layers.30.block_sparse_moe.experts.1.w1.weight": "model-00003-of-00003.safetensors", "model.layers.30.block_sparse_moe.experts.0.w2.weight": "model-00003-of-00003.safetensors", "model.layers.30.block_sparse_moe.experts.1.w2.weight": "model-00003-of-00003.safetensors", "model.layers.31.input_layernorm.weight": "model-00003-of-00003.safetensors", "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00003.safetensors", "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00003.safetensors", "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00003.safetensors", "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00003.safetensors", "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00003.safetensors", "model.layers.31.block_sparse_moe.experts.0.w3.weight": "model-00003-of-00003.safetensors", "model.layers.31.block_sparse_moe.experts.1.w3.weight": "model-00003-of-00003.safetensors", "model.layers.31.block_sparse_moe.experts.0.w1.weight": "model-00003-of-00003.safetensors", "model.layers.31.block_sparse_moe.experts.1.w1.weight": "model-00003-of-00003.safetensors", "model.layers.31.block_sparse_moe.experts.0.w2.weight": "model-00003-of-00003.safetensors", "model.layers.31.block_sparse_moe.experts.1.w2.weight": "model-00003-of-00003.safetensors", "model.layers.0.block_sparse_moe.gate.weight": "model-00003-of-00003.safetensors", "model.layers.1.block_sparse_moe.gate.weight": "model-00003-of-00003.safetensors", "model.layers.2.block_sparse_moe.gate.weight": "model-00003-of-00003.safetensors", "model.layers.3.block_sparse_moe.gate.weight": "model-00003-of-00003.safetensors", "model.layers.4.block_sparse_moe.gate.weight": "model-00003-of-00003.safetensors", "model.layers.5.block_sparse_moe.gate.weight": "model-00003-of-00003.safetensors", "model.layers.6.block_sparse_moe.gate.weight": "model-00003-of-00003.safetensors", "model.layers.7.block_sparse_moe.gate.weight": "model-00003-of-00003.safetensors", "model.layers.8.block_sparse_moe.gate.weight": "model-00003-of-00003.safetensors", "model.layers.9.block_sparse_moe.gate.weight": "model-00003-of-00003.safetensors", "model.layers.10.block_sparse_moe.gate.weight": "model-00003-of-00003.safetensors", "model.layers.11.block_sparse_moe.gate.weight": "model-00003-of-00003.safetensors", "model.layers.12.block_sparse_moe.gate.weight": "model-00003-of-00003.safetensors", "model.layers.13.block_sparse_moe.gate.weight": "model-00003-of-00003.safetensors", "model.layers.14.block_sparse_moe.gate.weight": "model-00003-of-00003.safetensors", "model.layers.15.block_sparse_moe.gate.weight": "model-00003-of-00003.safetensors", "model.layers.16.block_sparse_moe.gate.weight": "model-00003-of-00003.safetensors", "model.layers.17.block_sparse_moe.gate.weight": "model-00003-of-00003.safetensors", "model.layers.18.block_sparse_moe.gate.weight": "model-00003-of-00003.safetensors", "model.layers.19.block_sparse_moe.gate.weight": "model-00003-of-00003.safetensors", "model.layers.20.block_sparse_moe.gate.weight": "model-00003-of-00003.safetensors", "model.layers.21.block_sparse_moe.gate.weight": "model-00003-of-00003.safetensors", "model.layers.22.block_sparse_moe.gate.weight": "model-00003-of-00003.safetensors", "model.layers.23.block_sparse_moe.gate.weight": "model-00003-of-00003.safetensors", "model.layers.24.block_sparse_moe.gate.weight": "model-00003-of-00003.safetensors", "model.layers.25.block_sparse_moe.gate.weight": "model-00003-of-00003.safetensors", "model.layers.26.block_sparse_moe.gate.weight": "model-00003-of-00003.safetensors", "model.layers.27.block_sparse_moe.gate.weight": "model-00003-of-00003.safetensors", "model.layers.28.block_sparse_moe.gate.weight": "model-00003-of-00003.safetensors", "model.layers.29.block_sparse_moe.gate.weight": "model-00003-of-00003.safetensors", "model.layers.30.block_sparse_moe.gate.weight": "model-00003-of-00003.safetensors", "model.layers.31.block_sparse_moe.gate.weight": "model-00003-of-00003.safetensors"}}
+{
+  "metadata": {
+    "total_size": 25758277632
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00006-of-00006.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.block_sparse_moe.experts.0.w1.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.block_sparse_moe.experts.0.w2.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.block_sparse_moe.experts.0.w3.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.block_sparse_moe.experts.1.w1.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.block_sparse_moe.experts.1.w2.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.block_sparse_moe.experts.1.w3.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.block_sparse_moe.gate.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.block_sparse_moe.experts.0.w1.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.block_sparse_moe.experts.0.w2.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.block_sparse_moe.experts.0.w3.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.block_sparse_moe.experts.1.w1.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.block_sparse_moe.experts.1.w2.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.block_sparse_moe.experts.1.w3.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.block_sparse_moe.gate.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.10.block_sparse_moe.experts.0.w1.weight": "model-00002-of-00006.safetensors",
+    "model.layers.10.block_sparse_moe.experts.0.w2.weight": "model-00002-of-00006.safetensors",
+    "model.layers.10.block_sparse_moe.experts.0.w3.weight": "model-00002-of-00006.safetensors",
+    "model.layers.10.block_sparse_moe.experts.1.w1.weight": "model-00002-of-00006.safetensors",
+    "model.layers.10.block_sparse_moe.experts.1.w2.weight": "model-00002-of-00006.safetensors",
+    "model.layers.10.block_sparse_moe.experts.1.w3.weight": "model-00002-of-00006.safetensors",
+    "model.layers.10.block_sparse_moe.gate.weight": "model-00002-of-00006.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.11.block_sparse_moe.experts.0.w1.weight": "model-00002-of-00006.safetensors",
+    "model.layers.11.block_sparse_moe.experts.0.w2.weight": "model-00002-of-00006.safetensors",
+    "model.layers.11.block_sparse_moe.experts.0.w3.weight": "model-00002-of-00006.safetensors",
+    "model.layers.11.block_sparse_moe.experts.1.w1.weight": "model-00002-of-00006.safetensors",
+    "model.layers.11.block_sparse_moe.experts.1.w2.weight": "model-00002-of-00006.safetensors",
+    "model.layers.11.block_sparse_moe.experts.1.w3.weight": "model-00002-of-00006.safetensors",
+    "model.layers.11.block_sparse_moe.gate.weight": "model-00002-of-00006.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.12.block_sparse_moe.experts.0.w1.weight": "model-00002-of-00006.safetensors",
+    "model.layers.12.block_sparse_moe.experts.0.w2.weight": "model-00003-of-00006.safetensors",
+    "model.layers.12.block_sparse_moe.experts.0.w3.weight": "model-00003-of-00006.safetensors",
+    "model.layers.12.block_sparse_moe.experts.1.w1.weight": "model-00003-of-00006.safetensors",
+    "model.layers.12.block_sparse_moe.experts.1.w2.weight": "model-00003-of-00006.safetensors",
+    "model.layers.12.block_sparse_moe.experts.1.w3.weight": "model-00003-of-00006.safetensors",
+    "model.layers.12.block_sparse_moe.gate.weight": "model-00002-of-00006.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.13.block_sparse_moe.experts.0.w1.weight": "model-00003-of-00006.safetensors",
+    "model.layers.13.block_sparse_moe.experts.0.w2.weight": "model-00003-of-00006.safetensors",
+    "model.layers.13.block_sparse_moe.experts.0.w3.weight": "model-00003-of-00006.safetensors",
+    "model.layers.13.block_sparse_moe.experts.1.w1.weight": "model-00003-of-00006.safetensors",
+    "model.layers.13.block_sparse_moe.experts.1.w2.weight": "model-00003-of-00006.safetensors",
+    "model.layers.13.block_sparse_moe.experts.1.w3.weight": "model-00003-of-00006.safetensors",
+    "model.layers.13.block_sparse_moe.gate.weight": "model-00003-of-00006.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.14.block_sparse_moe.experts.0.w1.weight": "model-00003-of-00006.safetensors",
+    "model.layers.14.block_sparse_moe.experts.0.w2.weight": "model-00003-of-00006.safetensors",
+    "model.layers.14.block_sparse_moe.experts.0.w3.weight": "model-00003-of-00006.safetensors",
+    "model.layers.14.block_sparse_moe.experts.1.w1.weight": "model-00003-of-00006.safetensors",
+    "model.layers.14.block_sparse_moe.experts.1.w2.weight": "model-00003-of-00006.safetensors",
+    "model.layers.14.block_sparse_moe.experts.1.w3.weight": "model-00003-of-00006.safetensors",
+    "model.layers.14.block_sparse_moe.gate.weight": "model-00003-of-00006.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.15.block_sparse_moe.experts.0.w1.weight": "model-00003-of-00006.safetensors",
+    "model.layers.15.block_sparse_moe.experts.0.w2.weight": "model-00003-of-00006.safetensors",
+    "model.layers.15.block_sparse_moe.experts.0.w3.weight": "model-00003-of-00006.safetensors",
+    "model.layers.15.block_sparse_moe.experts.1.w1.weight": "model-00003-of-00006.safetensors",
+    "model.layers.15.block_sparse_moe.experts.1.w2.weight": "model-00003-of-00006.safetensors",
+    "model.layers.15.block_sparse_moe.experts.1.w3.weight": "model-00003-of-00006.safetensors",
+    "model.layers.15.block_sparse_moe.gate.weight": "model-00003-of-00006.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.16.block_sparse_moe.experts.0.w1.weight": "model-00003-of-00006.safetensors",
+    "model.layers.16.block_sparse_moe.experts.0.w2.weight": "model-00003-of-00006.safetensors",
+    "model.layers.16.block_sparse_moe.experts.0.w3.weight": "model-00003-of-00006.safetensors",
+    "model.layers.16.block_sparse_moe.experts.1.w1.weight": "model-00003-of-00006.safetensors",
+    "model.layers.16.block_sparse_moe.experts.1.w2.weight": "model-00003-of-00006.safetensors",
+    "model.layers.16.block_sparse_moe.experts.1.w3.weight": "model-00003-of-00006.safetensors",
+    "model.layers.16.block_sparse_moe.gate.weight": "model-00003-of-00006.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.17.block_sparse_moe.experts.0.w1.weight": "model-00003-of-00006.safetensors",
+    "model.layers.17.block_sparse_moe.experts.0.w2.weight": "model-00003-of-00006.safetensors",
+    "model.layers.17.block_sparse_moe.experts.0.w3.weight": "model-00003-of-00006.safetensors",
+    "model.layers.17.block_sparse_moe.experts.1.w1.weight": "model-00003-of-00006.safetensors",
+    "model.layers.17.block_sparse_moe.experts.1.w2.weight": "model-00003-of-00006.safetensors",
+    "model.layers.17.block_sparse_moe.experts.1.w3.weight": "model-00003-of-00006.safetensors",
+    "model.layers.17.block_sparse_moe.gate.weight": "model-00003-of-00006.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.18.block_sparse_moe.experts.0.w1.weight": "model-00003-of-00006.safetensors",
+    "model.layers.18.block_sparse_moe.experts.0.w2.weight": "model-00003-of-00006.safetensors",
+    "model.layers.18.block_sparse_moe.experts.0.w3.weight": "model-00003-of-00006.safetensors",
+    "model.layers.18.block_sparse_moe.experts.1.w1.weight": "model-00004-of-00006.safetensors",
+    "model.layers.18.block_sparse_moe.experts.1.w2.weight": "model-00004-of-00006.safetensors",
+    "model.layers.18.block_sparse_moe.experts.1.w3.weight": "model-00004-of-00006.safetensors",
+    "model.layers.18.block_sparse_moe.gate.weight": "model-00003-of-00006.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.19.block_sparse_moe.experts.0.w1.weight": "model-00004-of-00006.safetensors",
+    "model.layers.19.block_sparse_moe.experts.0.w2.weight": "model-00004-of-00006.safetensors",
+    "model.layers.19.block_sparse_moe.experts.0.w3.weight": "model-00004-of-00006.safetensors",
+    "model.layers.19.block_sparse_moe.experts.1.w1.weight": "model-00004-of-00006.safetensors",
+    "model.layers.19.block_sparse_moe.experts.1.w2.weight": "model-00004-of-00006.safetensors",
+    "model.layers.19.block_sparse_moe.experts.1.w3.weight": "model-00004-of-00006.safetensors",
+    "model.layers.19.block_sparse_moe.gate.weight": "model-00004-of-00006.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.2.block_sparse_moe.experts.0.w1.weight": "model-00001-of-00006.safetensors",
+    "model.layers.2.block_sparse_moe.experts.0.w2.weight": "model-00001-of-00006.safetensors",
+    "model.layers.2.block_sparse_moe.experts.0.w3.weight": "model-00001-of-00006.safetensors",
+    "model.layers.2.block_sparse_moe.experts.1.w1.weight": "model-00001-of-00006.safetensors",
+    "model.layers.2.block_sparse_moe.experts.1.w2.weight": "model-00001-of-00006.safetensors",
+    "model.layers.2.block_sparse_moe.experts.1.w3.weight": "model-00001-of-00006.safetensors",
+    "model.layers.2.block_sparse_moe.gate.weight": "model-00001-of-00006.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.20.block_sparse_moe.experts.0.w1.weight": "model-00004-of-00006.safetensors",
+    "model.layers.20.block_sparse_moe.experts.0.w2.weight": "model-00004-of-00006.safetensors",
+    "model.layers.20.block_sparse_moe.experts.0.w3.weight": "model-00004-of-00006.safetensors",
+    "model.layers.20.block_sparse_moe.experts.1.w1.weight": "model-00004-of-00006.safetensors",
+    "model.layers.20.block_sparse_moe.experts.1.w2.weight": "model-00004-of-00006.safetensors",
+    "model.layers.20.block_sparse_moe.experts.1.w3.weight": "model-00004-of-00006.safetensors",
+    "model.layers.20.block_sparse_moe.gate.weight": "model-00004-of-00006.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.21.block_sparse_moe.experts.0.w1.weight": "model-00004-of-00006.safetensors",
+    "model.layers.21.block_sparse_moe.experts.0.w2.weight": "model-00004-of-00006.safetensors",
+    "model.layers.21.block_sparse_moe.experts.0.w3.weight": "model-00004-of-00006.safetensors",
+    "model.layers.21.block_sparse_moe.experts.1.w1.weight": "model-00004-of-00006.safetensors",
+    "model.layers.21.block_sparse_moe.experts.1.w2.weight": "model-00004-of-00006.safetensors",
+    "model.layers.21.block_sparse_moe.experts.1.w3.weight": "model-00004-of-00006.safetensors",
+    "model.layers.21.block_sparse_moe.gate.weight": "model-00004-of-00006.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.22.block_sparse_moe.experts.0.w1.weight": "model-00004-of-00006.safetensors",
+    "model.layers.22.block_sparse_moe.experts.0.w2.weight": "model-00004-of-00006.safetensors",
+    "model.layers.22.block_sparse_moe.experts.0.w3.weight": "model-00004-of-00006.safetensors",
+    "model.layers.22.block_sparse_moe.experts.1.w1.weight": "model-00004-of-00006.safetensors",
+    "model.layers.22.block_sparse_moe.experts.1.w2.weight": "model-00004-of-00006.safetensors",
+    "model.layers.22.block_sparse_moe.experts.1.w3.weight": "model-00004-of-00006.safetensors",
+    "model.layers.22.block_sparse_moe.gate.weight": "model-00004-of-00006.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.23.block_sparse_moe.experts.0.w1.weight": "model-00004-of-00006.safetensors",
+    "model.layers.23.block_sparse_moe.experts.0.w2.weight": "model-00004-of-00006.safetensors",
+    "model.layers.23.block_sparse_moe.experts.0.w3.weight": "model-00004-of-00006.safetensors",
+    "model.layers.23.block_sparse_moe.experts.1.w1.weight": "model-00004-of-00006.safetensors",
+    "model.layers.23.block_sparse_moe.experts.1.w2.weight": "model-00004-of-00006.safetensors",
+    "model.layers.23.block_sparse_moe.experts.1.w3.weight": "model-00004-of-00006.safetensors",
+    "model.layers.23.block_sparse_moe.gate.weight": "model-00004-of-00006.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.24.block_sparse_moe.experts.0.w1.weight": "model-00004-of-00006.safetensors",
+    "model.layers.24.block_sparse_moe.experts.0.w2.weight": "model-00004-of-00006.safetensors",
+    "model.layers.24.block_sparse_moe.experts.0.w3.weight": "model-00004-of-00006.safetensors",
+    "model.layers.24.block_sparse_moe.experts.1.w1.weight": "model-00004-of-00006.safetensors",
+    "model.layers.24.block_sparse_moe.experts.1.w2.weight": "model-00004-of-00006.safetensors",
+    "model.layers.24.block_sparse_moe.experts.1.w3.weight": "model-00005-of-00006.safetensors",
+    "model.layers.24.block_sparse_moe.gate.weight": "model-00004-of-00006.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.25.block_sparse_moe.experts.0.w1.weight": "model-00005-of-00006.safetensors",
+    "model.layers.25.block_sparse_moe.experts.0.w2.weight": "model-00005-of-00006.safetensors",
+    "model.layers.25.block_sparse_moe.experts.0.w3.weight": "model-00005-of-00006.safetensors",
+    "model.layers.25.block_sparse_moe.experts.1.w1.weight": "model-00005-of-00006.safetensors",
+    "model.layers.25.block_sparse_moe.experts.1.w2.weight": "model-00005-of-00006.safetensors",
+    "model.layers.25.block_sparse_moe.experts.1.w3.weight": "model-00005-of-00006.safetensors",
+    "model.layers.25.block_sparse_moe.gate.weight": "model-00005-of-00006.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.26.block_sparse_moe.experts.0.w1.weight": "model-00005-of-00006.safetensors",
+    "model.layers.26.block_sparse_moe.experts.0.w2.weight": "model-00005-of-00006.safetensors",
+    "model.layers.26.block_sparse_moe.experts.0.w3.weight": "model-00005-of-00006.safetensors",
+    "model.layers.26.block_sparse_moe.experts.1.w1.weight": "model-00005-of-00006.safetensors",
+    "model.layers.26.block_sparse_moe.experts.1.w2.weight": "model-00005-of-00006.safetensors",
+    "model.layers.26.block_sparse_moe.experts.1.w3.weight": "model-00005-of-00006.safetensors",
+    "model.layers.26.block_sparse_moe.gate.weight": "model-00005-of-00006.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.27.block_sparse_moe.experts.0.w1.weight": "model-00005-of-00006.safetensors",
+    "model.layers.27.block_sparse_moe.experts.0.w2.weight": "model-00005-of-00006.safetensors",
+    "model.layers.27.block_sparse_moe.experts.0.w3.weight": "model-00005-of-00006.safetensors",
+    "model.layers.27.block_sparse_moe.experts.1.w1.weight": "model-00005-of-00006.safetensors",
+    "model.layers.27.block_sparse_moe.experts.1.w2.weight": "model-00005-of-00006.safetensors",
+    "model.layers.27.block_sparse_moe.experts.1.w3.weight": "model-00005-of-00006.safetensors",
+    "model.layers.27.block_sparse_moe.gate.weight": "model-00005-of-00006.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.28.block_sparse_moe.experts.0.w1.weight": "model-00005-of-00006.safetensors",
+    "model.layers.28.block_sparse_moe.experts.0.w2.weight": "model-00005-of-00006.safetensors",
+    "model.layers.28.block_sparse_moe.experts.0.w3.weight": "model-00005-of-00006.safetensors",
+    "model.layers.28.block_sparse_moe.experts.1.w1.weight": "model-00005-of-00006.safetensors",
+    "model.layers.28.block_sparse_moe.experts.1.w2.weight": "model-00005-of-00006.safetensors",
+    "model.layers.28.block_sparse_moe.experts.1.w3.weight": "model-00005-of-00006.safetensors",
+    "model.layers.28.block_sparse_moe.gate.weight": "model-00005-of-00006.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.29.block_sparse_moe.experts.0.w1.weight": "model-00005-of-00006.safetensors",
+    "model.layers.29.block_sparse_moe.experts.0.w2.weight": "model-00005-of-00006.safetensors",
+    "model.layers.29.block_sparse_moe.experts.0.w3.weight": "model-00005-of-00006.safetensors",
+    "model.layers.29.block_sparse_moe.experts.1.w1.weight": "model-00005-of-00006.safetensors",
+    "model.layers.29.block_sparse_moe.experts.1.w2.weight": "model-00005-of-00006.safetensors",
+    "model.layers.29.block_sparse_moe.experts.1.w3.weight": "model-00005-of-00006.safetensors",
+    "model.layers.29.block_sparse_moe.gate.weight": "model-00005-of-00006.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.3.block_sparse_moe.experts.0.w1.weight": "model-00001-of-00006.safetensors",
+    "model.layers.3.block_sparse_moe.experts.0.w2.weight": "model-00001-of-00006.safetensors",
+    "model.layers.3.block_sparse_moe.experts.0.w3.weight": "model-00001-of-00006.safetensors",
+    "model.layers.3.block_sparse_moe.experts.1.w1.weight": "model-00001-of-00006.safetensors",
+    "model.layers.3.block_sparse_moe.experts.1.w2.weight": "model-00001-of-00006.safetensors",
+    "model.layers.3.block_sparse_moe.experts.1.w3.weight": "model-00001-of-00006.safetensors",
+    "model.layers.3.block_sparse_moe.gate.weight": "model-00001-of-00006.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.30.block_sparse_moe.experts.0.w1.weight": "model-00005-of-00006.safetensors",
+    "model.layers.30.block_sparse_moe.experts.0.w2.weight": "model-00005-of-00006.safetensors",
+    "model.layers.30.block_sparse_moe.experts.0.w3.weight": "model-00005-of-00006.safetensors",
+    "model.layers.30.block_sparse_moe.experts.1.w1.weight": "model-00005-of-00006.safetensors",
+    "model.layers.30.block_sparse_moe.experts.1.w2.weight": "model-00005-of-00006.safetensors",
+    "model.layers.30.block_sparse_moe.experts.1.w3.weight": "model-00005-of-00006.safetensors",
+    "model.layers.30.block_sparse_moe.gate.weight": "model-00005-of-00006.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.31.block_sparse_moe.experts.0.w1.weight": "model-00006-of-00006.safetensors",
+    "model.layers.31.block_sparse_moe.experts.0.w2.weight": "model-00006-of-00006.safetensors",
+    "model.layers.31.block_sparse_moe.experts.0.w3.weight": "model-00006-of-00006.safetensors",
+    "model.layers.31.block_sparse_moe.experts.1.w1.weight": "model-00006-of-00006.safetensors",
+    "model.layers.31.block_sparse_moe.experts.1.w2.weight": "model-00006-of-00006.safetensors",
+    "model.layers.31.block_sparse_moe.experts.1.w3.weight": "model-00006-of-00006.safetensors",
+    "model.layers.31.block_sparse_moe.gate.weight": "model-00005-of-00006.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.4.block_sparse_moe.experts.0.w1.weight": "model-00001-of-00006.safetensors",
+    "model.layers.4.block_sparse_moe.experts.0.w2.weight": "model-00001-of-00006.safetensors",
+    "model.layers.4.block_sparse_moe.experts.0.w3.weight": "model-00001-of-00006.safetensors",
+    "model.layers.4.block_sparse_moe.experts.1.w1.weight": "model-00001-of-00006.safetensors",
+    "model.layers.4.block_sparse_moe.experts.1.w2.weight": "model-00001-of-00006.safetensors",
+    "model.layers.4.block_sparse_moe.experts.1.w3.weight": "model-00001-of-00006.safetensors",
+    "model.layers.4.block_sparse_moe.gate.weight": "model-00001-of-00006.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.5.block_sparse_moe.experts.0.w1.weight": "model-00001-of-00006.safetensors",
+    "model.layers.5.block_sparse_moe.experts.0.w2.weight": "model-00001-of-00006.safetensors",
+    "model.layers.5.block_sparse_moe.experts.0.w3.weight": "model-00001-of-00006.safetensors",
+    "model.layers.5.block_sparse_moe.experts.1.w1.weight": "model-00001-of-00006.safetensors",
+    "model.layers.5.block_sparse_moe.experts.1.w2.weight": "model-00001-of-00006.safetensors",
+    "model.layers.5.block_sparse_moe.experts.1.w3.weight": "model-00001-of-00006.safetensors",
+    "model.layers.5.block_sparse_moe.gate.weight": "model-00001-of-00006.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.6.block_sparse_moe.experts.0.w1.weight": "model-00002-of-00006.safetensors",
+    "model.layers.6.block_sparse_moe.experts.0.w2.weight": "model-00002-of-00006.safetensors",
+    "model.layers.6.block_sparse_moe.experts.0.w3.weight": "model-00002-of-00006.safetensors",
+    "model.layers.6.block_sparse_moe.experts.1.w1.weight": "model-00002-of-00006.safetensors",
+    "model.layers.6.block_sparse_moe.experts.1.w2.weight": "model-00002-of-00006.safetensors",
+    "model.layers.6.block_sparse_moe.experts.1.w3.weight": "model-00002-of-00006.safetensors",
+    "model.layers.6.block_sparse_moe.gate.weight": "model-00002-of-00006.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.7.block_sparse_moe.experts.0.w1.weight": "model-00002-of-00006.safetensors",
+    "model.layers.7.block_sparse_moe.experts.0.w2.weight": "model-00002-of-00006.safetensors",
+    "model.layers.7.block_sparse_moe.experts.0.w3.weight": "model-00002-of-00006.safetensors",
+    "model.layers.7.block_sparse_moe.experts.1.w1.weight": "model-00002-of-00006.safetensors",
+    "model.layers.7.block_sparse_moe.experts.1.w2.weight": "model-00002-of-00006.safetensors",
+    "model.layers.7.block_sparse_moe.experts.1.w3.weight": "model-00002-of-00006.safetensors",
+    "model.layers.7.block_sparse_moe.gate.weight": "model-00002-of-00006.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.8.block_sparse_moe.experts.0.w1.weight": "model-00002-of-00006.safetensors",
+    "model.layers.8.block_sparse_moe.experts.0.w2.weight": "model-00002-of-00006.safetensors",
+    "model.layers.8.block_sparse_moe.experts.0.w3.weight": "model-00002-of-00006.safetensors",
+    "model.layers.8.block_sparse_moe.experts.1.w1.weight": "model-00002-of-00006.safetensors",
+    "model.layers.8.block_sparse_moe.experts.1.w2.weight": "model-00002-of-00006.safetensors",
+    "model.layers.8.block_sparse_moe.experts.1.w3.weight": "model-00002-of-00006.safetensors",
+    "model.layers.8.block_sparse_moe.gate.weight": "model-00002-of-00006.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.block_sparse_moe.experts.0.w1.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.block_sparse_moe.experts.0.w2.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.block_sparse_moe.experts.0.w3.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.block_sparse_moe.experts.1.w1.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.block_sparse_moe.experts.1.w2.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.block_sparse_moe.experts.1.w3.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.block_sparse_moe.gate.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00002-of-00006.safetensors",
+    "model.norm.weight": "model-00006-of-00006.safetensors"
+  }
+}

special_tokens_map.json

@@ -18,7 +18,13 @@
     "rstrip": false,
     "single_word": false
   },
-  "pad_token": "<s>",
+  "pad_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
   "unk_token": {
     "content": "<unk>",
     "lstrip": false,