(self, text)
return "".join(output)
Adds whitespace around any CJK character.
Adds whitespace around any CJK character.
Adds whitespace around any CJK character .
def _tokenize_chinese_chars(self, text):
		"""Adds whitespace around any CJK character."""
		output = []
		for char in text:
			cp = ord(char)
			if self._is_chinese_char(cp):
				output.append(" ")
				output.append(char)
				output.append(" ")
			else:
				output.append(char)
		return "".join(output)
def _tokenize_chinese_chars ( self , text ) : output = [ ] for char in text : cp = ord ( char ) if self . _is_chinese_char ( cp ) : output . append ( " " ) output . append ( char ) output . append ( " " ) else : output . append ( char ) return "" . join ( output )