Upload folder using huggingface_hub

README.md

@@ -1,24 +1,34 @@
 ---
-tags:
-- merge
-- mergekit
-- lazymergekit
-- Sao10K/Fimbulvetr-10.7B-v1
-- upstage/SOLAR-10.7B-Instruct-v1.0
 base_model:
-- Sao10K/Fimbulvetr-10.7B-v1
 - upstage/SOLAR-10.7B-Instruct-v1.0
+- Sao10K/Fimbulvetr-10.7B-v1
+library_name: transformers
+tags:
+- mergekit
+- merge
+
 ---
+# merge
 
-# sarry-10.7B-slerp
+This is a merge of pre-trained language models created using [mergekit](https://github.com/cg123/mergekit).
 
-sarry-10.7B-slerp is a merge of the following models using [LazyMergekit](https://colab.research.google.com/drive/1obulZ1ROXHjYLn6PPZJwRR6GzgQogxxb?usp=sharing):
-* [Sao10K/Fimbulvetr-10.7B-v1](https://huggingface.co/Sao10K/Fimbulvetr-10.7B-v1)
+## Merge Details
+### Merge Method
+
+This model was merged using the SLERP merge method.
+
+### Models Merged
+
+The following models were included in the merge:
 * [upstage/SOLAR-10.7B-Instruct-v1.0](https://huggingface.co/upstage/SOLAR-10.7B-Instruct-v1.0)
+* [Sao10K/Fimbulvetr-10.7B-v1](https://huggingface.co/Sao10K/Fimbulvetr-10.7B-v1)
 
-## 🧩 Configuration
+### Configuration
+
+The following YAML configuration was used to produce this model:
 
 ```yaml
+
 slices:
   - sources:
       - model: Sao10K/Fimbulvetr-10.7B-v1
@@ -26,7 +36,7 @@ slices:
       - model: upstage/SOLAR-10.7B-Instruct-v1.0
         layer_range: [0, 32]
 merge_method: slerp
-base_model: Sao10K/Fimbulvetr-10.7B-v1
+base_model: upstage/SOLAR-10.7B-Instruct-v1.0
 parameters:
   t:
     - filter: self_attn
@@ -35,29 +45,5 @@ parameters:
       value: [1, 0.5, 0.7, 0.3, 0]
     - value: 0.5
 dtype: bfloat16
-```
 
-## 💻 Usage
-
-```python
-!pip install -qU transformers accelerate
-
-from transformers import AutoTokenizer
-import transformers
-import torch
-
-model = "ssaryssane/sarry-10.7B-slerp"
-messages = [{"role": "user", "content": "What is a large language model?"}]
-
-tokenizer = AutoTokenizer.from_pretrained(model)
-prompt = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
-pipeline = transformers.pipeline(
-    "text-generation",
-    model=model,
-    torch_dtype=torch.float16,
-    device_map="auto",
-)
-
-outputs = pipeline(prompt, max_new_tokens=256, do_sample=True, temperature=0.7, top_k=50, top_p=0.95)
-print(outputs[0]["generated_text"])
-```
+```

config.json

@@ -1,10 +1,9 @@
 {
-  "_name_or_path": "Sao10K/Fimbulvetr-10.7B-v1",
+  "_name_or_path": "upstage/SOLAR-10.7B-Instruct-v1.0",
   "architectures": [
     "LlamaForCausalLM"
   ],
   "attention_bias": false,
-  "attention_dropout": 0.0,
   "bos_token_id": 1,
   "eos_token_id": 2,
   "hidden_act": "silu",
@@ -16,6 +15,7 @@
   "num_attention_heads": 32,
   "num_hidden_layers": 32,
   "num_key_value_heads": 8,
+  "pad_token_id": 2,
   "pretraining_tp": 1,
   "rms_norm_eps": 1e-05,
   "rope_scaling": null,

mergekit_config.yml

@@ -6,7 +6,7 @@ slices:
       - model: upstage/SOLAR-10.7B-Instruct-v1.0
         layer_range: [0, 32]
 merge_method: slerp
-base_model: Sao10K/Fimbulvetr-10.7B-v1
+base_model: upstage/SOLAR-10.7B-Instruct-v1.0
 parameters:
   t:
     - filter: self_attn

model-00001-of-00008.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7795f4c6534852f2b3f55875ab8cee7defaafc98baf5e1e62ade28bb847b44a2
+oid sha256:48525a4a0f783906648f6817cebce0f1eff22c1791098223a526adeec2a7bac3
 size 1979773128

model-00002-of-00008.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5bf25709948eb151bb2b9ff508b1738ff1378e0203e114367fa46f0ad21f33d4
+oid sha256:eab31689452a8111dd7bf69790ad21beffd0f1a4103bef6e734c2bc4be1bb4f9
 size 1946235640

model-00003-of-00008.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6b25b5026d4dd347168d14b4713c14356defac9d726217520364efe90dda084f
+oid sha256:876ca2e573d87537d07b8131373c00dd63cd371564d925b0470f1040cad493b5
 size 1973490216

model-00004-of-00008.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d5cb5eb53e4d699ff0615532387ebad3a73e4798f2362f2c1927d55ba397cab4
+oid sha256:cf9abd9329742be8437dfb8a6d26a58139f0ef15dac5482c34a91f47f8450148
 size 1979781464

model-00005-of-00008.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0c08e5f2905b841b28bf777e810c97635533a2a2201848db77e538d9856a0d43
+oid sha256:ba2fa338fe09e12b70885b507408c9281d2ee9775e7cfe8a2ff32bc46748faa6
 size 1946243984

model-00006-of-00008.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6e5140ee9413f3555956ae43830764d68bd27452e46f99e171314cbf12113ee7
+oid sha256:501fdcb9fc81aad0ee37315f2785b372ebf2460d2dfdb82db77e2b3e56215c6f
 size 1979798072

model-00007-of-00008.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d297a98cdcd4d915aff581ba02e84336fa11d17612d92ef65710f04b6705c07c
+oid sha256:9a5553968470e655f57ae92b71f1d001a8d8e324029c842312d01a5ddfa8eb37
 size 1979789776

model-00008-of-00008.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:731ad821946a4b4cac0a5035a77758ac85f0376691203a55a9528f711ce65989
+oid sha256:c8cc0b0392b79effef6bbeb2b5ec9188ec3180b88165b1dcb2be14a4c031b6ca
 size 698385744

special_tokens_map.json

@@ -13,6 +13,13 @@
     "rstrip": false,
     "single_word": false
   },
+  "pad_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
   "unk_token": {
     "content": "<unk>",
     "lstrip": false,

tokenizer_config.json

@@ -27,11 +27,12 @@
   },
   "additional_special_tokens": [],
   "bos_token": "<s>",
+  "chat_template": "{% for message in messages %}{% if message['role'] == 'system' %}{% if message['content']%}{{'### System:\n' + message['content']+'\n\n'}}{% endif %}{% elif message['role'] == 'user' %}{{'### User:\n' + message['content']+'\n\n'}}{% elif message['role'] == 'assistant' %}{{'### Assistant:\n'  + message['content']}}{% endif %}{% if loop.last and add_generation_prompt %}{{ '### Assistant:\n' }}{% endif %}{% endfor %}",
   "clean_up_tokenization_spaces": false,
   "eos_token": "</s>",
   "legacy": true,
   "model_max_length": 1000000000000000019884624838656,
-  "pad_token": null,
+  "pad_token": "</s>",
   "sp_model_kwargs": {},
   "spaces_between_special_tokens": false,
   "tokenizer_class": "LlamaTokenizer",