juliehunter
commited on
Update tokenizer.json
Browse files- tokenizer.json +21 -0
tokenizer.json
CHANGED
@@ -289,6 +289,27 @@
|
|
289 |
{
|
290 |
"type": "Fuse"
|
291 |
},
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
292 |
{
|
293 |
"type": "Replace",
|
294 |
"pattern": {
|
|
|
289 |
{
|
290 |
"type": "Fuse"
|
291 |
},
|
292 |
+
{
|
293 |
+
"type": "Replace",
|
294 |
+
"pattern": {
|
295 |
+
"String": "<|start_header_id|> "
|
296 |
+
},
|
297 |
+
"content": "<|start_header_id|>"
|
298 |
+
},
|
299 |
+
{
|
300 |
+
"type": "Replace",
|
301 |
+
"pattern": {
|
302 |
+
"String": "<|end_header_id|> "
|
303 |
+
},
|
304 |
+
"content": "<|end_header_id|>"
|
305 |
+
},
|
306 |
+
{
|
307 |
+
"type": "Replace",
|
308 |
+
"pattern": {
|
309 |
+
"String": "<|eot_id|> "
|
310 |
+
},
|
311 |
+
"content": "<|eot_id|>"
|
312 |
+
},
|
313 |
{
|
314 |
"type": "Replace",
|
315 |
"pattern": {
|