(self, text)
return "".join(output)
Adds whitespace around any CJK character.
Adds whitespace around any CJK character.
Adds whitespace around any CJK character .
def _tokenize_chinese_chars(self, text):
    """Adds whitespace around any CJK character."""
    output = []
    for char in text:
      cp = ord(char)
      if self._is_chinese_char(cp):
        output.append(" ")
        output.append(char)
        output.append(" ")
      else:
        output.append(char)
    return "".join(output)
def _tokenize_chinese_chars ( self , text ) : output = [ ] for char in text : cp = ord ( char ) if self . _is_chinese_char ( cp ) : output . append ( " " ) output . append ( char ) output . append ( " " ) else : output . append ( char ) return "" . join ( output )