naist-nlp
/

mitre_466m

Model card Files Files and versions Community

zhiqu22 commited on 25 days ago

Commit

fe0ae35

·

1 Parent(s): 07b10ca

update codes

Files changed (1) hide show

tokenizer_config.json +2 -1

tokenizer_config.json CHANGED Viewed

@@ -9,7 +9,8 @@
     "model_max_length": 256,
     "name_or_path": "naist-nlp/mitre_466m",
     "tokenizer_class": "MitreTokenizer",
     "auto_map": {
-        "AutoTokenizer": "tokenization_mitre.MitreTokenizer"
     }
 }

     "model_max_length": 256,
     "name_or_path": "naist-nlp/mitre_466m",
     "tokenizer_class": "MitreTokenizer",
+    "use_fast": false,
     "auto_map": {
+        "AutoTokenizer": ["tokenization_mitre.MitreTokenizer"]
     }
 }