add tokenizer

vocab.json

@@ -1 +1 @@
-{"!": 1, "\"": 2, "%": 3, "'": 4, ",": 5, "-": 6, ".": 7, "?": 8, "A": 9, "B": 10, "C": 11, "D": 12, "E": 13, "F": 14, "G": 15, "H": 16, "I": 17, "J": 18, "K": 19, "L": 20, "M": 21, "N": 22, "O": 23, "P": 24, "Q": 25, "R": 26, "S": 27, "T": 28, "U": 29, "V": 30, "W": 31, "Y": 32, "Z": 33, "a": 34, "b": 35, "c": 36, "d": 37, "e": 38, "f": 39, "g": 40, "h": 41, "i": 42, "j": 43, "k": 44, "l": 45, "m": 46, "n": 47, "o": 48, "p": 49, "r": 50, "s": 51, "t": 52, "u": 53, "v": 54, "w": 55, "x": 56, "y": 57, "z": 58, "Ç": 59, "Ö": 60, "Ü": 61, "â": 62, "ç": 63, "ë": 64, "ö": 65, "ü": 66, "ğ": 67, "İ": 68, "ı": 69, "Ş": 70, "ş": 71, "‘": 72, "“": 73, "”": 74, "�": 75, "|": 0, "[UNK]": 76, "[PAD]": 77}
+{"!": 1, "\"": 2, "%": 3, "'": 4, ",": 5, "-": 6, ".": 7, "?": 8, "A": 9, "B": 10, "C": 11, "D": 12, "E": 13, "F": 14, "G": 15, "H": 16, "I": 17, "J": 18, "K": 19, "L": 20, "M": 21, "N": 22, "O": 23, "P": 24, "Q": 25, "R": 26, "S": 27, "T": 28, "U": 29, "V": 30, "W": 31, "X": 32, "Y": 33, "Z": 34, "a": 35, "b": 36, "c": 37, "d": 38, "e": 39, "f": 40, "g": 41, "h": 42, "i": 43, "j": 44, "k": 45, "l": 46, "m": 47, "n": 48, "o": 49, "p": 50, "r": 51, "s": 52, "t": 53, "u": 54, "v": 55, "w": 56, "x": 57, "y": 58, "z": 59, "Ç": 60, "Ö": 61, "Ü": 62, "â": 63, "ç": 64, "ë": 65, "î": 66, "ö": 67, "ü": 68, "ğ": 69, "İ": 70, "ı": 71, "Ş": 72, "ş": 73, "‘": 74, "“": 75, "”": 76, "�": 77, "|": 0, "[UNK]": 78, "[PAD]": 79}