Upload merged model: FMixIA-FrankenMerge-9.5B-PT-9

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,45 @@
+---
+license: apache-2.0
+tags:
+- merge
+- mergekit
+- passthrough
+- "frankenmerge"
+- "7B"
+- "ZeroXClem/Qwen2.5-7B-HomerCreative-Mix"
+- "ZeroXClem/Qwen2.5-7B-HomerAnvita-NerdMix"
+---
+
+# FMixIA-FrankenMerge-9.5B-PT-9
+
+A merged model using Passthrough layer concatenation, creating a frankenmerge model using [mergekit](https://github.com/cg123/mergekit).
+
+## Model Details
+- **Base Models**:
+  * [ZeroXClem/Qwen2.5-7B-HomerCreative-Mix](https://huggingface.co/ZeroXClem/Qwen2.5-7B-HomerCreative-Mix)
+  * [ZeroXClem/Qwen2.5-7B-HomerAnvita-NerdMix](https://huggingface.co/ZeroXClem/Qwen2.5-7B-HomerAnvita-NerdMix)
+- **Merge Method**: passthrough
+- **Note**: This is a frankenmerge model with modified architecture
+
+## Configuration
+```yaml
+slices:
+  - sources:
+    - model: ZeroXClem/Qwen2.5-7B-HomerCreative-Mix
+      layer_range: [0, 28]
+  - sources:
+    - model: ZeroXClem/Qwen2.5-7B-HomerAnvita-NerdMix
+      layer_range: [0, 28]
+merge_method: passthrough
+dtype: bfloat16
+```
+
+## Usage
+This model can be used with the standard transformers library:
+
+```python
+from transformers import AutoModelForCausalLM, AutoTokenizer
+
+model = AutoModelForCausalLM.from_pretrained("Ro-xe/FMixIA-FrankenMerge-9.5B-PT-9")
+tokenizer = AutoTokenizer.from_pretrained("Ro-xe/FMixIA-FrankenMerge-9.5B-PT-9")
+```

added_tokens.json

@@ -0,0 +1,24 @@
+{
+  "</tool_call>": 151658,
+  "<tool_call>": 151657,
+  "<|box_end|>": 151649,
+  "<|box_start|>": 151648,
+  "<|endoftext|>": 151643,
+  "<|file_sep|>": 151664,
+  "<|fim_middle|>": 151660,
+  "<|fim_pad|>": 151662,
+  "<|fim_prefix|>": 151659,
+  "<|fim_suffix|>": 151661,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644,
+  "<|image_pad|>": 151655,
+  "<|object_ref_end|>": 151647,
+  "<|object_ref_start|>": 151646,
+  "<|quad_end|>": 151651,
+  "<|quad_start|>": 151650,
+  "<|repo_name|>": 151663,
+  "<|video_pad|>": 151656,
+  "<|vision_end|>": 151653,
+  "<|vision_pad|>": 151654,
+  "<|vision_start|>": 151652
+}

config.json

@@ -0,0 +1,29 @@
+{
+  "_name_or_path": "ZeroXClem/Qwen2.5-7B-HomerCreative-Mix",
+  "architectures": [
+    "Qwen2ForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 151643,
+  "eos_token_id": 151645,
+  "hidden_act": "silu",
+  "hidden_size": 3584,
+  "initializer_range": 0.02,
+  "intermediate_size": 18944,
+  "max_position_embeddings": 32768,
+  "max_window_layers": 28,
+  "model_type": "qwen2",
+  "num_attention_heads": 28,
+  "num_hidden_layers": 56,
+  "num_key_value_heads": 4,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 1000000.0,
+  "sliding_window": null,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.47.0",
+  "use_cache": true,
+  "use_sliding_window": false,
+  "vocab_size": 152064
+}

mergekit_config.yml

@@ -0,0 +1,9 @@
+slices:
+  - sources:
+    - model: ZeroXClem/Qwen2.5-7B-HomerCreative-Mix
+      layer_range: [0, 28]
+  - sources:
+    - model: ZeroXClem/Qwen2.5-7B-HomerAnvita-NerdMix
+      layer_range: [0, 28]
+merge_method: passthrough
+dtype: bfloat16

model-00001-of-00015.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8e3179168d48da50b3aea6648f2ebaa5981948f898028d790e3a634cd0d258dc
+size 960521544

model-00001-of-00030.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3e51fe7ca2ea39232b4bdc1831f088cf8772cbcfdfde1d74c43b918de24aa808
+size 1089994880

model-00002-of-00015.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:608bcaae953fe20abb77081886266d4853c3fff0615eab45ed8a0af5880b0429
+size 989890704

model-00002-of-00030.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:27d2ab2a01a6c4bbd8d3130f4c7f272a3890efb33ef095804c391ccc676d6cbb
+size 1089994896

model-00003-of-00015.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0e003c37da7e872c42d91b0cdb7b490b69445154c2785b846de38c353caedb32
+size 989890712

model-00003-of-00030.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:67cd1803f5c0a5cac73236502b6b96ed87ca7b1be94ddeb06bf83516cd1262bd
+size 932248448

model-00004-of-00015.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7d957312f3c875ee76a226bc9a5fd0ab301b27d32b1de9610fcb36d9c85b15f7
+size 998287736

model-00004-of-00030.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:34a7f6a23104f1d3d6990d5e3723199a462075480f539e69aa2f2ad8bc3faba8
+size 932233904

model-00005-of-00015.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4e82147b657c9015cd555d4f34efd5e566c177b7660f91bbda43913f8c373303
+size 947956192

model-00005-of-00030.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:312828ffdb5e483f0cb405b47373810faa3ea57c43f6cd4366d2226ede7ae82b
+size 932233912

model-00006-of-00015.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7a2359c4374de0f3f0f474a2b8d294b4fa4ae82111fab896175bbdb05d485d2c
+size 989890704

model-00006-of-00030.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c7d0886459e0ecf2993ee94a4b33b3a22e410eca4ba9e835d9bd97dab0ce2ff0
+size 932233912

model-00007-of-00015.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:efc9f282eeb8741878481eba11cfd6916ce324b2651afc46ad6b7302f067f292
+size 989890704

model-00007-of-00030.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2e12ff41eb53368f3d882dc3c305a33991f7d0627c4b0826046802329fcd98fa
+size 932233912

model-00008-of-00015.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:817e88d5b5c49421aa50006bfd8be023a5f6d4840a2d5d131dd36f1d5a68b1b4
+size 998287736

model-00008-of-00030.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:da71aacd6c80e5c94b58420453b8b5f8b4cf1df48be5baf30642f31ae9a7e4ac
+size 932233912

model-00009-of-00015.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b5c55d4836f65ec38599dcdab1c2b62d782214582132d7c47ad8ebd6af540e99
+size 947956192

model-00009-of-00030.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7efbc673f9a4f6f116e12b166d38c3a1ab925c84ea785db7d01c2e1278ade9a2
+size 932233912

model-00010-of-00015.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d2fe506514a207cc33a2daaae0e7e451eaad1c518d72b97072fb2c8edce78a72
+size 989890712

model-00010-of-00030.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:793849c5236a3af8e7db8a31467da3238720a1b000985fb182499987c850bea5
+size 932233912

model-00011-of-00015.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:746e90454abf25b89de0a1630b6deb42de45eb6b740ef12210900d26cd587e1e
+size 989890704

model-00011-of-00030.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f517728dc5b6f711878010b22f1d386ae5b101acb8340dee53c5b7b81c7b2715
+size 932233912

model-00012-of-00015.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b4c4425d84d5f7c1c9a59448ae4ac3f91cd964f0bab9beeaaedf8cea8eb66e87
+size 998287736

model-00012-of-00030.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e9139b706a7e1c70197961a2fe667ee0a4ef65424bb21b270bad06b764650982
+size 932233912

model-00013-of-00015.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8a275019bf5fb7cb519d439351bc21606b424735b92a9321127d912f7298e930
+size 947956176

model-00013-of-00030.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8b2f0a2ac6f9b4f7311a23c19972936f3858d23a02257f3bf4b64f950b95e30d
+size 932233912

model-00014-of-00015.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:57bec613dba0618d86c51df927ba5273a7ec48775ba396d78dfd404aa67467e9
+size 989890688

model-00014-of-00030.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9f4f663183319d208b5bc681923cc225b12c029e833768809c703729ae82d0a3
+size 932233912

model-00015-of-00015.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:88562da74888bcc892aa1d34579e5a4642ae41e42c1c03182e3151e9a839b1b0
+size 755009424

model-00015-of-00030.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1d28f68b4bf99b4193a71aa9ee01a83b8b982fa541cc92a374ddcf4462d279fe
+size 932233904

model-00016-of-00030.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bc08eba74e3e0c969a849a1f39d2172bf450e4317be70290a1c06b63256513bd
+size 932233912

model-00017-of-00030.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:89ccc6e0131baaa2b58c9a73587ee0f386e5b91db029747e0036bcd1e67d0b73
+size 932233912

model-00018-of-00030.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ab6cf7b53349e9f53433801ee078669011455c65e71efe6cc3737e94bd28943e
+size 932233912

model-00019-of-00030.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2586d358a0866ed7294b80bdef311c654f2881ec6c62f84dea120b55191f563f
+size 932233912

model-00020-of-00030.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:00496af46a97cf7d6df56ef61f0732e9c84886def72754b7380fd370ab699fdf
+size 932233912

model-00021-of-00030.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bb65590a5d7788924a9cd0cd293e11d86b5203da3deb0cfa59c689ea0a6dff8a
+size 932233912

model-00022-of-00030.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b31b26b2d02088fb8db55e78f1380b1a9e7c978a7508495126d3380536483638
+size 932233912

model-00023-of-00030.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e0fa191172d0a04aa18d30787ed2701314f6cbb840a238ce2e9c965a02e7f85a
+size 932233912

model-00024-of-00030.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2e5a49e71ef73248fc010722fad336d09cfd92abd7e0745f46c050fdf73eb5e9
+size 932233904

model-00025-of-00030.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0efe41650313b80e624c3c09af3773abebaa7be70ccd5d2c94e9333495b0a789
+size 932233904

model-00026-of-00030.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9c1574b5a7ddbf748abf9e50d05058f89244c65144e9c16d53b27c1b9302e488
+size 932233904

model-00027-of-00030.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3e157b8832b4aeaccf2e46e7b5f7d56af856bd28594610cafb9d68f8fed09856
+size 932233904

model-00028-of-00030.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4488cabc6fb74058bb835329b57799b0e6e0a54b24ff3fa6579b20b6159ef240
+size 932233904

model-00029-of-00030.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5e7e980b6291f180db5c55687b30fd8fcc19387025df7a121b62800e68ec16af
+size 932233904