(self, text)
return output_tokens
Tokenizes a piece of text into its word pieces.

		This uses a greedy longest-match-first algorithm to perform tokenization
		using the given vocabulary.

		For example:
		  input = "unaffable"
		  output = ["un", "##aff", "##able"]

		Args:
		  text: A single token or whitespace separated tokens. This should have
			already been passed through `BasicTokenizer`.

		Returns:
		  A list of wordpiece tokens.
Tokenizes a piece of text into its word pieces.
Tokenizes a piece of text into its word pieces .
def tokenize(self, text):
		"""Tokenizes a piece of text into its word pieces.

		This uses a greedy longest-match-first algorithm to perform tokenization
		using the given vocabulary.

		For example:
		  input = "unaffable"
		  output = ["un", "##aff", "##able"]

		Args:
		  text: A single token or whitespace separated tokens. This should have
			already been passed through `BasicTokenizer`.

		Returns:
		  A list of wordpiece tokens.
		"""

		output_tokens = []
		for token in whitespace_tokenize(text):
			chars = list(token)
			if len(chars) > self.max_input_chars_per_word:
				output_tokens.append(self.unk_token)
				continue

			is_bad = False
			start = 0
			sub_tokens = []
			while start < len(chars):
				end = len(chars)
				cur_substr = None
				while start < end:
					substr = "".join(chars[start:end])
					### joonho.lim @ 2019-03-15
					# if start > 0:
						# substr = "##" + substr
					# print ( '[substr]\t%s\t%s\t%d\t%d' % ( substr, substr in self.vocab, start, end))
					if substr in self.vocab:
						cur_substr = substr
						break
					end -= 1
				if cur_substr is None:
					is_bad = True
					break
				sub_tokens.append(cur_substr)
				start = end

			if is_bad:
				output_tokens.append(self.unk_token)
			else:
				output_tokens.extend(sub_tokens)
		return output_tokens
def tokenize ( self , text ) : output_tokens = [ ] for token in whitespace_tokenize ( text ) : chars = list ( token ) if len ( chars ) > self . max_input_chars_per_word : output_tokens . append ( self . unk_token ) continue is_bad = False start = 0 sub_tokens = [ ] while start < len ( chars ) : end = len ( chars ) cur_substr = None while start < end : substr = "" . join ( chars [ start : end ] ) ### joonho.lim @ 2019-03-15 # if start > 0: # substr = "##" + substr # print ( '[substr]\t%s\t%s\t%d\t%d' % ( substr, substr in self.vocab, start, end)) if substr in self . vocab : cur_substr = substr break end -= 1 if cur_substr is None : is_bad = True break sub_tokens . append ( cur_substr ) start = end if is_bad : output_tokens . append ( self . unk_token ) else : output_tokens . extend ( sub_tokens ) return output_tokens