(texts, max_seq_len, vocab_path)
return token_ids
Tokenize sentence
Tokenize sentence
Tokenize sentence
def tokenize_sentence(texts, max_seq_len, vocab_path):
  """Tokenize sentence"""
  vocabs = read_lines_from_text_file(vocab_path)
  token_ids, _ = py_x_ops.sentence_to_ids(
      texts,
      maxlen=max_seq_len,
      use_vocab_file=False,
      vocab=vocabs,
      load_token_ids_from_vocab=True,
      pad_id=utils.PAD_IDX,
      check_tokens=False)
  return token_ids
def tokenize_sentence ( texts , max_seq_len , vocab_path ) : vocabs = read_lines_from_text_file ( vocab_path ) token_ids , _ = py_x_ops . sentence_to_ids ( texts , maxlen = max_seq_len , use_vocab_file = False , vocab = vocabs , load_token_ids_from_vocab = True , pad_id = utils . PAD_IDX , check_tokens = False ) return token_ids