Upload tokenizer

added_tokens.json

@@ -1,3 +1,4 @@
 {
-  "<|im_start|>": 32000
+  "<|im_end|>": 32000,
+  "<|im_start|>": 32001
 }

special_tokens_map.json

@@ -14,7 +14,7 @@
     "single_word": false
   },
   "pad_token": {
-    "content": "<|im_end|>",
+    "content": "<unk>",
     "lstrip": false,
     "normalized": false,
     "rstrip": false,

tokenizer.json

@@ -23,7 +23,7 @@
     },
     {
       "id": 2,
-      "content": "<|im_end|>",
+      "content": "</s>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -32,12 +32,21 @@
     },
     {
       "id": 32000,
+      "content": "<|im_end|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 32001,
       "content": "<|im_start|>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
       "normalized": false,
-      "special": false
+      "special": true
     }
   ],
   "normalizer": {
@@ -147,7 +156,7 @@
     "vocab": {
       "<unk>": 0,
       "<s>": 1,
-      "<|im_end|>": 2,
+      "</s>": 2,
       "<0x00>": 3,
       "<0x01>": 4,
       "<0x02>": 5,