update codes

tokenization_mitre.py

@@ -207,3 +207,5 @@ def load_json(path: str) -> Union[Dict, List]:
 def save_json(data, path: str) -> None:
     with open(path, "w") as f:
         json.dump(data, f, indent=2)
+
+MitreTokenizer.register_for_auto_class("AutoTokenizer")

tokenizer_config.json

@@ -6,5 +6,10 @@
     "sep_token": "</s>",
     "unk_token": "<unk>",
     "pad_token": "<pad>",
-    "model_max_length": 256
+    "model_max_length": 256,
+    "name_or_path": "naist-nlp/mitre_466m",
+    "tokenizer_class": "MitreTokenizer",
+    "auto_map": {
+        "AutoTokenizer": "tokenization_mitre.MitreTokenizer"
+    }
 }