(self, text)
return output_tokens
Tokenizes a piece of text.
Tokenizes a piece of text.
Tokenizes a piece of text .
def tokenize(self, text):
        """Tokenizes a piece of text."""
        text = self._clean_text(text)
        # This was added on November 1st, 2018 for the multilingual and Chinese
        # models. This is also applied to the English models now, but it doesn't
        # matter since the English models were not trained on any Chinese data
        # and generally don't have any Chinese data in them (there are Chinese
        # characters in the vocabulary because Wikipedia does have some Chinese
        # words in the English Wikipedia.).
        text = self._tokenize_chinese_chars(text)
        orig_tokens = whitespace_tokenize(text)
        split_tokens = []
        for token in orig_tokens:
            if self.do_lower_case:
                token = token.lower()
                token = self._run_strip_accents(token)
            split_tokens.extend(self._run_split_on_punc(token))

        output_tokens = whitespace_tokenize(" ".join(split_tokens))
        return output_tokens
def tokenize ( self , text ) : text = self . _clean_text ( text ) # This was added on November 1st, 2018 for the multilingual and Chinese # models. This is also applied to the English models now, but it doesn't # matter since the English models were not trained on any Chinese data # and generally don't have any Chinese data in them (there are Chinese # characters in the vocabulary because Wikipedia does have some Chinese # words in the English Wikipedia.). text = self . _tokenize_chinese_chars ( text ) orig_tokens = whitespace_tokenize ( text ) split_tokens = [ ] for token in orig_tokens : if self . do_lower_case : token = token . lower ( ) token = self . _run_strip_accents ( token ) split_tokens . extend ( self . _run_split_on_punc ( token ) ) output_tokens = whitespace_tokenize ( " " . join ( split_tokens ) ) return output_tokens