| """Helper script do dump/freeze the current tokenizer""" | |
| from tokenizers import Tokenizer | |
| from pathlib import Path | |
| HERE = Path(__file__).parent.absolute() | |
| def main(): | |
| MODEL_NAME = "sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2" | |
| tokenizer = Tokenizer.from_pretrained(MODEL_NAME) | |
| tokenizer.save((HERE.parent / "iscc_sct/tokenizer.json").as_posix(), pretty=False) | |
| if __name__ == "__main__": | |
| main() | |