test.py

# Wrap your own tokenizer
from transformers import PreTrainedTokenizerFast

wrapped_tokenizer = PreTrainedTokenizerFast(
    tokenizer_file="tokenizer.json", # You can load from the tokenizer file
    unk_token="[UNK]",
    pad_token="[PAD]",
    cls_token="[CLS]",
    sep_token="[SEP]",
    mask_token="[MASK]",
)

# Finally, save your own pretrained tokenizer
wrapped_tokenizer.save_pretrained('my-tokenizer')