Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

1_Pooling/config.json

@@ -0,0 +1,10 @@
+{
+  "word_embedding_dimension": 1536,
+  "pooling_mode_cls_token": false,
+  "pooling_mode_mean_tokens": true,
+  "pooling_mode_max_tokens": false,
+  "pooling_mode_mean_sqrt_len_tokens": false,
+  "pooling_mode_weightedmean_tokens": false,
+  "pooling_mode_lasttoken": false,
+  "include_prompt": true
+}

2_Dense/config.json

@@ -0,0 +1 @@
+{"in_features": 1536, "out_features": 1024, "bias": true, "activation_function": "torch.nn.modules.linear.Identity"}

2_Dense/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a3b63b4ca37aee58c47ae8ac58f05af96341e165971e6a99adc2cb0f5e0a8f58
+size 6295712

Pipfile

@@ -0,0 +1,11 @@
+[[source]]
+url = "https://pypi.org/simple"
+verify_ssl = true
+name = "pypi"
+
+[packages]
+
+[dev-packages]
+
+[requires]
+python_version = "3.12"

README.md

@@ -0,0 +1,32 @@
+---
+license: cc-by-nc-2.0
+language:
+- en
+base_model:
+- dunzhang/stella_en_1.5B_v5
+library_name: sentence-transformers
+pipeline_tag: sentence-similarity
+tags:
+- sentence-transformers
+- sentence-similarity
+- medical
+- clinicaltrials
+- cancer
+- feature-extraction
+- generated_from_trainer
+- dataset_size:1395384
+- loss:OnlineContrastiveLoss
+- loss:MultipleNegativesRankingLoss
+---
+# SentenceTransformer
+
+This is a [sentence-transformers](https://www.SBERT.net) model trained. It maps sentences & paragraphs to a 1024-dimensional dense vector space and can be used for semantic textual similarity, semantic search, paraphrase mining, text classification, clustering, and more.
+
+## Model Details
+
+### Model Description
+- **Model Type:** Sentence Transformer
+<!-- - **Base model:** [dunzhang/stella_en_1.5B_v5](https://huggingface.co/dunzhang/stella_en_1.5B_v5) -->
+- **Maximum Sequence Length:** 512 tokens
+- **Output Dimensionality:** 1024 tokens
+- **Similarity Function:** Cosine Similarity

added_tokens.json

@@ -0,0 +1,5 @@
+{
+  "<|endoftext|>": 151643,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644
+}

backup/special_tokens_map.json

@@ -0,0 +1,20 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>"
+  ],
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

backup/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c8eab352ec6bb19236f8684bf40e504e86876bd4f2f43982b0561b2f07702666
+size 11418805

backup/tokenizer_config.json

@@ -0,0 +1,56 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>"
+  ],
+  "auto_map": {
+    "AutoTokenizer": [
+      "tokenization_qwen.Qwen2Tokenizer",
+      "tokenization_qwen.Qwen2TokenizerFast"
+    ]
+  },
+  "bos_token": null,
+  "chat_template": "{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "max_length": 512,
+  "model_max_length": 512,
+  "pad_to_multiple_of": null,
+  "pad_token": "<|endoftext|>",
+  "pad_token_type_id": 0,
+  "padding_side": "left",
+  "split_special_tokens": false,
+  "stride": 0,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "truncation_side": "right",
+  "truncation_strategy": "longest_first",
+  "unk_token": null
+}

config.json

@@ -0,0 +1,34 @@
+{
+  "_name_or_path": "reranker_round1.model",
+  "architectures": [
+    "Qwen2Model"
+  ],
+  "attention_dropout": 0.0,
+  "auto_map": {
+    "AutoModel": "dunzhang/stella_en_1.5b_v5--modeling_qwen.Qwen2Model",
+    "AutoModelForCausalLM": "dunzhang/stella_en_1.5b_v5--modeling_qwen.Qwen2ForCausalLM",
+    "AutoModelForSequenceClassification": "dunzhang/stella_en_1.5b_v5--modeling_qwen.Qwen2ForSequenceClassification"
+  },
+  "bos_token_id": 151643,
+  "eos_token_id": 151643,
+  "hidden_act": "silu",
+  "hidden_size": 1536,
+  "initializer_range": 0.02,
+  "intermediate_size": 8960,
+  "max_position_embeddings": 131072,
+  "max_window_layers": 21,
+  "model_type": "qwen2",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 28,
+  "num_key_value_heads": 2,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 1000000.0,
+  "sliding_window": null,
+  "tie_word_embeddings": false,
+  "torch_dtype": "float32",
+  "transformers_version": "4.45.2",
+  "use_cache": true,
+  "use_sliding_window": false,
+  "vocab_size": 151646
+}

config_sentence_transformers.json

@@ -0,0 +1,13 @@
+{
+  "__version__": {
+    "sentence_transformers": "3.1.1",
+    "transformers": "4.45.2",
+    "pytorch": "2.4.0+cu121"
+  },
+  "prompts": {
+    "s2p_query": "Instruct: Given a web search query, retrieve relevant passages that answer the query.\nQuery: ",
+    "s2s_query": "Instruct: Retrieve semantically similar text.\nQuery: "
+  },
+  "default_prompt_name": null,
+  "similarity_fn_name": "cosine"
+}

model-00001-of-00002.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:69d74487fa399f069df4a729371169581c43f7d30ce4ede54cab549c8081cb0e
+size 4994887136

model-00002-of-00002.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8d984228e35c6b48621f15e90af3ffd386bedba69b73949aa84431282c38424d
+size 1178224504

model.safetensors.index.json

@@ -0,0 +1,345 @@
+{
+  "metadata": {
+    "total_size": 6173075456
+  },
+  "weight_map": {
+    "embed_tokens.weight": "model-00001-of-00002.safetensors",
+    "layers.0.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "layers.0.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.0.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.0.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.0.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "layers.0.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.0.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.0.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.0.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.0.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.0.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.0.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.1.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "layers.1.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.1.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.1.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.1.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "layers.1.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.1.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.1.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.1.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.1.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.1.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.1.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.10.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "layers.10.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.10.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.10.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.10.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "layers.10.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.10.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.10.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.10.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.10.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.10.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.10.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.11.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "layers.11.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.11.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.11.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.11.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "layers.11.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.11.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.11.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.11.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.11.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.11.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.11.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.12.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "layers.12.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.12.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.12.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.12.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "layers.12.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.12.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.12.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.12.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.12.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.12.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.12.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.13.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "layers.13.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.13.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.13.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.13.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "layers.13.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.13.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.13.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.13.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.13.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.13.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.13.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.14.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "layers.14.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.14.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.14.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.14.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "layers.14.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.14.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.14.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.14.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.14.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.14.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.14.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.15.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "layers.15.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.15.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.15.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.15.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "layers.15.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.15.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.15.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.15.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.15.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.15.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.15.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.16.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "layers.16.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.16.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.16.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.16.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "layers.16.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.16.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.16.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.16.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.16.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.16.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.16.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.17.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "layers.17.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.17.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.17.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.17.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "layers.17.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.17.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.17.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.17.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.17.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.17.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.17.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.18.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "layers.18.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.18.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.18.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.18.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "layers.18.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.18.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.18.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.18.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.18.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.18.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.18.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.19.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "layers.19.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.19.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.19.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.19.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "layers.19.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.19.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.19.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.19.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.19.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.19.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.19.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.2.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "layers.2.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.2.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.2.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.2.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "layers.2.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.2.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.2.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.2.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.2.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.2.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.2.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.20.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "layers.20.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.20.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.20.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.20.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "layers.20.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.20.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.20.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.20.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.20.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.20.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.20.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.21.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "layers.21.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "layers.21.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.21.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.21.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "layers.21.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.21.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.21.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.21.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.21.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.21.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.21.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.22.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "layers.22.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "layers.22.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "layers.22.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "layers.22.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "layers.22.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "layers.22.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "layers.22.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "layers.22.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "layers.22.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "layers.22.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "layers.22.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "layers.23.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "layers.23.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "layers.23.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "layers.23.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "layers.23.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "layers.23.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "layers.23.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "layers.23.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "layers.23.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "layers.23.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "layers.23.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "layers.23.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "layers.24.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "layers.24.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "layers.24.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "layers.24.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "layers.24.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "layers.24.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "layers.24.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "layers.24.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "layers.24.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "layers.24.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "layers.24.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "layers.24.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "layers.25.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "layers.25.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "layers.25.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "layers.25.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "layers.25.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "layers.25.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "layers.25.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "layers.25.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "layers.25.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "layers.25.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "layers.25.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "layers.25.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "layers.26.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "layers.26.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "layers.26.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "layers.26.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "layers.26.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "layers.26.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "layers.26.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "layers.26.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "layers.26.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "layers.26.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "layers.26.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "layers.26.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "layers.27.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "layers.27.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "layers.27.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "layers.27.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "layers.27.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "layers.27.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "layers.27.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "layers.27.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "layers.27.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "layers.27.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "layers.27.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "layers.27.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "layers.3.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "layers.3.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.3.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.3.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.3.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "layers.3.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.3.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.3.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.3.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.3.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.3.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.3.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.4.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "layers.4.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.4.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.4.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.4.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "layers.4.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.4.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.4.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.4.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.4.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.4.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.4.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.5.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "layers.5.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.5.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.5.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.5.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "layers.5.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.5.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.5.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.5.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.5.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.5.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.5.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.6.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "layers.6.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.6.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.6.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.6.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "layers.6.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.6.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.6.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.6.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.6.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.6.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.6.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.7.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "layers.7.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.7.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.7.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.7.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "layers.7.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.7.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.7.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.7.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.7.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.7.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.7.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.8.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "layers.8.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.8.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.8.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.8.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "layers.8.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.8.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.8.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.8.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.8.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.8.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.8.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.9.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "layers.9.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.9.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.9.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.9.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "layers.9.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.9.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.9.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.9.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.9.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.9.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.9.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "norm.weight": "model-00002-of-00002.safetensors"
+  }
+}

modules.json

@@ -0,0 +1,20 @@
+[
+  {
+    "idx": 0,
+    "name": "0",
+    "path": "",
+    "type": "sentence_transformers.models.Transformer"
+  },
+  {
+    "idx": 1,
+    "name": "1",
+    "path": "1_Pooling",
+    "type": "sentence_transformers.models.Pooling"
+  },
+  {
+    "idx": 2,
+    "name": "2",
+    "path": "2_Dense",
+    "type": "sentence_transformers.models.Dense"
+  }
+]

sentence_bert_config.json

@@ -0,0 +1,4 @@
+{
+  "max_seq_length": 512,
+  "do_lower_case": false
+}

special_tokens_map.json

@@ -0,0 +1,20 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>"
+  ],
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

test.py

@@ -0,0 +1,14 @@
+# Wrap your own tokenizer
+from transformers import PreTrainedTokenizerFast
+
+wrapped_tokenizer = PreTrainedTokenizerFast(
+    tokenizer_file="tokenizer.json", # You can load from the tokenizer file
+    unk_token="[UNK]",
+    pad_token="[PAD]",
+    cls_token="[CLS]",
+    sep_token="[SEP]",
+    mask_token="[MASK]",
+)
+
+# Finally, save your own pretrained tokenizer
+wrapped_tokenizer.save_pretrained('my-tokenizer')

tokenization_qwen.py

@@ -0,0 +1,267 @@
+
+from typing import List, Optional
+from transformers.models.qwen2.tokenization_qwen2 import Qwen2Tokenizer as OriginalQwen2Tokenizer
+from transformers.models.qwen2.tokenization_qwen2_fast import Qwen2TokenizerFast as OriginalQwen2TokenizerFast
+from tokenizers import processors
+
+VOCAB_FILES_NAMES = {
+    "vocab_file": "vocab.json",
+    "merges_file": "merges.txt",
+    "tokenizer_file": "tokenizer.json",
+}
+
+class Qwen2Tokenizer(OriginalQwen2Tokenizer):
+    """
+    Construct a Qwen2 tokenizer. Based on byte-level Byte-Pair-Encoding.
+
+    Same with GPT2Tokenizer, this tokenizer has been trained to treat spaces like parts of the tokens so a word will
+    be encoded differently whether it is at the beginning of the sentence (without space) or not:
+
+    ```python
+    >>> from transformers import Qwen2Tokenizer
+
+    >>> tokenizer = Qwen2Tokenizer.from_pretrained("Qwen/Qwen-tokenizer")
+    >>> tokenizer("Hello world")["input_ids"]
+    [9707, 1879]
+
+    >>> tokenizer(" Hello world")["input_ids"]
+    [21927, 1879]
+    ```
+    This is expected.
+
+    You should not use GPT2Tokenizer instead, because of the different pretokenization rules.
+
+    This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods. Users should refer to
+    this superclass for more information regarding those methods.
+
+    Args:
+        vocab_file (`str`):
+            Path to the vocabulary file.
+        merges_file (`str`):
+            Path to the merges file.
+        errors (`str`, *optional*, defaults to `"replace"`):
+            Paradigm to follow when decoding bytes to UTF-8. See
+            [bytes.decode](https://docs.python.org/3/library/stdtypes.html#bytes.decode) for more information.
+        unk_token (`str`, *optional*, defaults to `"<|endoftext|>"`):
+            The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
+            token instead.
+        bos_token (`str`, *optional*):
+            The beginning of sequence token. Not applicable for this tokenizer.
+        eos_token (`str`, *optional*, defaults to `"<|endoftext|>"`):
+            The end of sequence token.
+        pad_token (`str`, *optional*, defaults to `"<|endoftext|>"`):
+            The token used for padding, for example when batching sequences of different lengths.
+        clean_up_tokenization_spaces (`bool`, *optional*, defaults to `False`):
+            Whether or not the model should cleanup the spaces that were added when splitting the input text during the
+            tokenization process. Not applicable to this tokenizer, since tokenization does not add spaces.
+        split_special_tokens (`bool`, *optional*, defaults to `False`):
+            Whether or not the special tokens should be split during the tokenization process. The default behavior is
+            to not split special tokens. This means that if `<|endoftext|>` is the `eos_token`, then `tokenizer.tokenize("<|endoftext|>") =
+            ['<|endoftext|>`]. Otherwise, if `split_special_tokens=True`, then `tokenizer.tokenize("<|endoftext|>")` will be give `['<',
+            '|', 'endo', 'ft', 'ext', '|', '>']`. This argument is only supported for `slow` tokenizers for the moment.
+        add_eos_token (`bool`, *optional*, defaults to `False`):
+            Whether or not to add an `eos_token` at the end of sequences.
+    """
+
+    def __init__(
+        self,
+        vocab_file,
+        merges_file,
+        errors="replace",
+        unk_token="<|endoftext|>",
+        bos_token=None,
+        eos_token="<|endoftext|>",
+        pad_token="<|endoftext|>",
+        clean_up_tokenization_spaces=False,
+        split_special_tokens=False,
+        add_eos_token=False,
+        **kwargs,
+    ):
+        # The add_eos_token code was inspired by the LlamaTokenizer
+        self.add_eos_token = add_eos_token
+
+        super().__init__(
+            vocab_file=vocab_file,
+            merges_file=merges_file,
+            errors=errors,
+            unk_token=unk_token,
+            bos_token=bos_token,
+            eos_token=eos_token,
+            pad_token=pad_token,
+            clean_up_tokenization_spaces=clean_up_tokenization_spaces,
+            split_special_tokens=split_special_tokens,
+            add_eos_token=add_eos_token,
+            **kwargs,
+        )
+
+    def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None):
+        eos_token_id = [self.eos_token_id] if self.add_eos_token else []
+
+        output = token_ids_0 + eos_token_id
+
+        if token_ids_1 is not None:
+            output = output + token_ids_1 + eos_token_id
+
+        return output
+
+    def get_special_tokens_mask(
+        self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None, already_has_special_tokens: bool = False
+    ) -> List[int]:
+        """
+        Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding
+        special tokens using the tokenizer `prepare_for_model` method.
+
+        Args:
+            token_ids_0 (`List[int]`):
+                List of IDs.
+            token_ids_1 (`List[int]`, *optional*):
+                Optional second list of IDs for sequence pairs.
+            already_has_special_tokens (`bool`, *optional*, defaults to `False`):
+                Whether or not the token list is already formatted with special tokens for the model.
+
+        Returns:
+            `List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.
+        """
+        if already_has_special_tokens:
+            return super().get_special_tokens_mask(
+                token_ids_0=token_ids_0, token_ids_1=token_ids_1, already_has_special_tokens=True
+            )
+
+        eos_token_id = [1] if self.add_eos_token else []
+
+        if token_ids_1 is None:
+            return ([0] * len(token_ids_0)) + eos_token_id
+        return (
+            ([0] * len(token_ids_0))
+            + eos_token_id
+            + ([0] * len(token_ids_1))
+            + eos_token_id
+        )
+
+    def create_token_type_ids_from_sequences(
+        self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
+    ) -> List[int]:
+        """
+        Creates a mask from the two sequences passed to be used in a sequence-pair classification task. An ALBERT
+        sequence pair mask has the following format:
+
+        ```
+        0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1
+        | first sequence    | second sequence |
+        ```
+
+        if token_ids_1 is None, only returns the first portion of the mask (0s).
+
+        Args:
+            token_ids_0 (`List[int]`):
+                List of ids.
+            token_ids_1 (`List[int]`, *optional*):
+                Optional second list of IDs for sequence pairs.
+
+        Returns:
+            `List[int]`: List of [token type IDs](../glossary#token-type-ids) according to the given sequence(s).
+        """
+        eos_token_id = [self.eos_token_id] if self.add_eos_token else []
+
+        output = [0] * len(token_ids_0 + eos_token_id)
+
+        if token_ids_1 is not None:
+            output += [1] * len(token_ids_1 + eos_token_id)
+
+        return output
+
+class Qwen2TokenizerFast(OriginalQwen2TokenizerFast):
+    """
+    Construct a "fast" Qwen2 tokenizer (backed by HuggingFace's *tokenizers* library). Based on byte-level
+    Byte-Pair-Encoding.
+
+    Same with GPT2Tokenizer, this tokenizer has been trained to treat spaces like parts of the tokens so a word will
+    be encoded differently whether it is at the beginning of the sentence (without space) or not:
+
+    ```python
+    >>> from transformers import Qwen2TokenizerFast
+
+    >>> tokenizer = Qwen2TokenizerFast.from_pretrained("Qwen/Qwen-tokenizer")
+    >>> tokenizer("Hello world")["input_ids"]
+    [9707, 1879]
+
+    >>> tokenizer(" Hello world")["input_ids"]
+    [21927, 1879]
+    ```
+    This is expected.
+
+    This tokenizer inherits from [`PreTrainedTokenizerFast`] which contains most of the main methods. Users should
+    refer to this superclass for more information regarding those methods.
+
+    Args:
+        vocab_file (`str`, *optional*):
+            Path to the vocabulary file.
+        merges_file (`str`, *optional*):
+            Path to the merges file.
+        tokenizer_file (`str`, *optional*):
+            Path to [tokenizers](https://github.com/huggingface/tokenizers) file (generally has a .json extension) that
+            contains everything needed to load the tokenizer.
+        unk_token (`str`, *optional*, defaults to `"<|endoftext|>"`):
+            The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
+            token instead. Not applicable to this tokenizer.
+        bos_token (`str`, *optional*):
+            The beginning of sequence token. Not applicable for this tokenizer.
+        eos_token (`str`, *optional*, defaults to `"<|endoftext|>"`):
+            The end of sequence token.
+        pad_token (`str`, *optional*, defaults to `"<|endoftext|>"`):
+            The token used for padding, for example when batching sequences of different lengths.
+        add_eos_token (`bool`, *optional*, defaults to `False`):
+            Whether or not to add an `eos_token` at the end of sequences.
+    """
+
+    slow_tokenizer_class = Qwen2Tokenizer
+    padding_side = "left"
+
+    def __init__(
+        self,
+        vocab_file=None,
+        merges_file=None,
+        tokenizer_file=None,
+        unk_token="<|endoftext|>",
+        bos_token=None,
+        eos_token="<|endoftext|>",
+        pad_token="<|endoftext|>",
+        add_eos_token=False,
+        **kwargs,
+    ):
+        super().__init__(
+            vocab_file=vocab_file,
+            merges_file=merges_file,
+            tokenizer_file=tokenizer_file,
+            unk_token=unk_token,
+            bos_token=bos_token,
+            eos_token=eos_token,
+            pad_token=pad_token,
+            **kwargs,
+        )
+
+        self._add_eos_token = add_eos_token
+        self.update_post_processor()
+
+    def update_post_processor(self):
+        """
+        Updates the underlying post processor with the current `eos_token`.
+        """
+        eos = self.eos_token
+        eos_token_id = self.eos_token_id
+        if eos is None and self.add_eos_token:
+            raise ValueError("add_eos_token = True but eos_token = None")
+
+        single = f"$A:0{(' '+eos+':0') if self.add_eos_token else ''}"
+        pair = f"{single} $B:1{(' '+eos+':1') if self.add_eos_token else ''}"
+
+        special_tokens = []
+        if self.add_eos_token:
+            special_tokens.append((eos, eos_token_id))
+        self._tokenizer.post_processor = processors.TemplateProcessing(
+            single=single, pair=pair, special_tokens=special_tokens
+        )
+
+    @property
+    def add_eos_token(self):
+        return self._add_eos_token

tokenizer_config.json

@@ -0,0 +1,47 @@
+{
+  "add_eos_token": true,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>"
+  ],
+  "auto_map": {
+    "AutoTokenizer": ["tokenization_qwen.Qwen2Tokenizer", "tokenization_qwen.Qwen2TokenizerFast"]
+  },
+  "bos_token": null,
+  "chat_template": "{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "model_max_length": 32768,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}