(self, text)
return output_tokens
Tokenizes a piece of text.
Tokenizes a piece of text.
Tokenizes a piece of text .
def tokenize(self, text):
    """Tokenizes a piece of text."""
    text = native_to_unicode(text)
    text = self._clean_text(text)

    text = self._tokenize_chinese_chars(text)

    orig_tokens = whitespace_tokenize(text)
    split_tokens = []
    for token in orig_tokens:
      if self.do_lower_case:
        token = token.lower()
        token = self._run_strip_accents(token)
      split_tokens.extend(self._run_split_on_punc(token))

    output_tokens = whitespace_tokenize(" ".join(split_tokens))
    return output_tokens
def tokenize ( self , text ) : text = native_to_unicode ( text ) text = self . _clean_text ( text ) text = self . _tokenize_chinese_chars ( text ) orig_tokens = whitespace_tokenize ( text ) split_tokens = [ ] for token in orig_tokens : if self . do_lower_case : token = token . lower ( ) token = self . _run_strip_accents ( token ) split_tokens . extend ( self . _run_split_on_punc ( token ) ) output_tokens = whitespace_tokenize ( " " . join ( split_tokens ) ) return output_tokens