Simple demonstration of a cjk tokenizer.
from common import load_tiktoken_cl100k


def iscjk(token):
  # simple check
  for c in token:
    if ord("\u4e00") < ord(c) < ord("\u9fa5"):
      return True
  return False


tiktoken_cl100k = load_tiktoken_cl100k()
cjk_tokens = {k: v for k, v in tiktoken_cl100k.items() if iscjk(v)}

print(cjk_tokens)
print(f"Total cjk tokens: {len(cjk_tokens)}, ratio: {len(cjk_tokens)/len(tiktoken_cl100k)*100:.2f}%")

check_cl100_cjk.py
SwordYork/cjk100k
from common import load_tiktoken_cl100k, load_cjk100k

tiktoken_cl100k = load_tiktoken_cl100k()
cjk100k = load_cjk100k()

tiktoken_set = set(tiktoken_cl100k.values())
for topk in [1000, 10000, len(cjk100k)]:
  cnt = 0
  for i in range(topk):
    if cjk100k[i] in tiktoken_set:
      cnt += 1
  print(f"coverage cjk100k@{topk}", cnt / topk)

check_converage.py
SwordYork/cjk100k
import json

from tiktoken.load import load_tiktoken_bpe
from tokenizers import decoders


def load_tiktoken_cl100k():
  """Load a TikToken BPE from a URL."""
  decoder = decoders.ByteLevel()
  mergeable_ranks = load_tiktoken_bpe("https://openaipublic.blob.core.windows.net/encodings/cl100k_base.tiktoken")

  # copy from https://github.com/openai/tiktoken/blob/main/tiktoken/load.py
  rank_to_intbyte = [b for b in range(2**8) if chr(b).isprintable() and chr(b) != " "]
  data_gym_byte_to_byte = {chr(b): b for b in rank_to_intbyte}
  n = 0
  for b in range(2**8):
    if b not in rank_to_intbyte:
      rank_to_intbyte.append(b)
      data_gym_byte_to_byte[chr(2**8 + n)] = b
      n += 1
  assert len(rank_to_intbyte) == 2**8
  int_to_gym = {v: k for k, v in data_gym_byte_to_byte.items()}

  def encode_data_gym(value: bytes) -> str:
    return "".join(int_to_gym[byte] for byte in value)

  id2token = {}
  for idx, k in enumerate(mergeable_ranks):
    # convert to str token from bytes by ByteLevel decoder
    d = decoder.decode([encode_data_gym(k)])
    id2token[idx] = d
  return id2token


def load_cjk100k():
  decoder = decoders.ByteLevel()
  ddict = json.loads(open("tokenizer.json").read())
  id2token = {}
  for idx, v in enumerate(ddict["model"]["vocab"]):
    d = decoder.decode([v])
    id2token[idx] = d
  return id2token
common.py
SwordYork/cjk100k
import json
from tiktoken import Encoding
from tiktoken.load import data_gym_to_mergeable_bpe_ranks

tokenizer_config = "tokenizer.json"


def data_gym_to_mergeable_bpe_ranks(bpe_merges, start_idx=0):
  # NB: do not add caching to this function
  rank_to_intbyte = [b for b in range(2**8) if chr(b).isprintable() and chr(b) != " "]

  data_gym_byte_to_byte = {chr(b): b for b in rank_to_intbyte}
  n = 0
  for b in range(2**8):
    if b not in rank_to_intbyte:
      rank_to_intbyte.append(b)
      data_gym_byte_to_byte[chr(2**8 + n)] = b
      n += 1
  assert len(rank_to_intbyte) == 2**8
  bpe_merges = [tuple(merge_str.split()) for merge_str in bpe_merges]

  def decode_data_gym(value: str) -> bytes:
    return bytes(data_gym_byte_to_byte[b] for b in value)

  # add the single byte tokens
  bpe_ranks = {bytes([b]): i + start_idx for i, b in enumerate(rank_to_intbyte)}
  # add the merged tokens
  n = len(bpe_ranks) + start_idx
  for first, second in bpe_merges:
    bpe_ranks[decode_data_gym(first) + decode_data_gym(second)] = n
    n += 1
  return bpe_ranks


def create_tokenizer():
  special_tokens = [
    "[PAD]", "<|endoftext|>", "[UNK]", "[CLS]", "[SEP]", "[MASK]", "<|beginoftext|>", "<|fim_prefix|>",
    "<|fim_middle|>", "<|fim_suffix|>", "<|beginofprompt|>", "<|endofprompt|>", "[PAD1]", "[PAD2]", "[PAD3]", "[PAD4]",
    "[PAD5]", "[PAD6]", "[PAD7]", "[PAD8]", "[PAD9]", "[PAD10]", "[PAD11]", "[PAD12]", "[PAD13]", "[PAD14]", "[PAD15]",
    "[PAD16]"
  ]

  cjk_regex = r"""(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?[\p{L}--[\u4e00-\u9fa5]]+|[\u4e00-\u9fa5]|\p{N}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?![\S--[\u4e00-\u9fa5]])|\s+"""
  with open(tokenizer_config) as fr:
    ddict = json.loads(fr.read())
    bpe_ranks = data_gym_to_mergeable_bpe_ranks(ddict["model"]["merges"], start_idx=len(special_tokens))
    sp_ranks = {t: i for i, t in enumerate(special_tokens)}
    targs = {"name": "cjk100k", "pat_str": cjk_regex, "mergeable_ranks": bpe_ranks, "special_tokens": sp_ranks}
  return Encoding(**targs)


batch_docs = ["你好，世界", "hello, world!"]
tokenizer = create_tokenizer()
tokenized_docs = tokenizer.encode_batch(batch_docs, allowed_special="all", num_threads=8)
recovered = tokenizer.decode_batch(tokenized_docs)

print(tokenized_docs, recovered)
demo.py
SwordYork/cjk100k