(self, text)
return output_tokens
Tokenizes a piece of text into its word pieces.

    This uses a greedy longest-match-first algorithm to perform tokenization
    using the given vocabulary.
    For example:
      input = "unaffable"
      output = ["un", "##aff", "##able"]
    Args:
      text: A single token or whitespace separated tokens. This should have
        already been passed through `BasicTokenizer.

    Returns:
      A list of wordpiece tokens.
Tokenizes a piece of text into its word pieces.
Tokenizes a piece of text into its word pieces .
def tokenize(self, text):
    """Tokenizes a piece of text into its word pieces.

    This uses a greedy longest-match-first algorithm to perform tokenization
    using the given vocabulary.
    For example:
      input = "unaffable"
      output = ["un", "##aff", "##able"]
    Args:
      text: A single token or whitespace separated tokens. This should have
        already been passed through `BasicTokenizer.

    Returns:
      A list of wordpiece tokens.
    """

    text = native_to_unicode(text)

    output_tokens = []
    for token in whitespace_tokenize(text):
      chars = list(token)
      if len(chars) > self.max_input_chars_per_word:
        output_tokens.append(self.unk_token)
        continue

      is_bad = False
      start = 0
      sub_tokens = []
      while start < len(chars):
        end = len(chars)
        cur_substr = None
        while start < end:
          substr = "".join(chars[start:end])
          if start > 0:
            substr = "##" + substr
          if substr in self.vocab:
            cur_substr = substr
            break
          end -= 1
        if cur_substr is None:
          is_bad = True
          break
        sub_tokens.append(cur_substr)
        start = end

      if is_bad:
        output_tokens.append(self.unk_token)
      else:
        output_tokens.extend(sub_tokens)
    return output_tokens
def tokenize ( self , text ) : text = native_to_unicode ( text ) output_tokens = [ ] for token in whitespace_tokenize ( text ) : chars = list ( token ) if len ( chars ) > self . max_input_chars_per_word : output_tokens . append ( self . unk_token ) continue is_bad = False start = 0 sub_tokens = [ ] while start < len ( chars ) : end = len ( chars ) cur_substr = None while start < end : substr = "" . join ( chars [ start : end ] ) if start > 0 : substr = "##" + substr if substr in self . vocab : cur_substr = substr break end -= 1 if cur_substr is None : is_bad = True break sub_tokens . append ( cur_substr ) start = end if is_bad : output_tokens . append ( self . unk_token ) else : output_tokens . extend ( sub_tokens ) return output_tokens