juliehunter commited on
Commit
4834048
·
verified ·
1 Parent(s): 11c21c8

Update tokenizer.json

Browse files
Files changed (1) hide show
  1. tokenizer.json +21 -0
tokenizer.json CHANGED
@@ -289,6 +289,27 @@
289
  {
290
  "type": "Fuse"
291
  },
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
292
  {
293
  "type": "Replace",
294
  "pattern": {
 
289
  {
290
  "type": "Fuse"
291
  },
292
+ {
293
+ "type": "Replace",
294
+ "pattern": {
295
+ "String": "<|start_header_id|> "
296
+ },
297
+ "content": "<|start_header_id|>"
298
+ },
299
+ {
300
+ "type": "Replace",
301
+ "pattern": {
302
+ "String": "<|end_header_id|> "
303
+ },
304
+ "content": "<|end_header_id|>"
305
+ },
306
+ {
307
+ "type": "Replace",
308
+ "pattern": {
309
+ "String": "<|eot_id|> "
310
+ },
311
+ "content": "<|eot_id|>"
312
+ },
313
  {
314
  "type": "Replace",
315
  "pattern": {