(doc_t, split_token, padding_token,
                                   max_doc_len, max_sen_len)
Split a document to sentences with true sentence lengths.
  doc_t: [doc_word_len]
  out_t: [max_doc_len, max_sen_len]
Split a document to sentences with true sentence lengths.
  doc_t: [doc_word_len]
  out_t: [max_doc_len, max_sen_len]
Split a document to sentences with true sentence lengths . doc_t : [ doc_word_len ] out_t : [ max_doc_len max_sen_len ]
def split_one_doc_to_true_len_sens(doc_t, split_token, padding_token,
                                   max_doc_len, max_sen_len):
  """
  Split a document to sentences with true sentence lengths.
  doc_t: [doc_word_len]
  out_t: [max_doc_len, max_sen_len]
  """
  if len(doc_t.get_shape()) == 1:
    split_token_index = tf.squeeze(
        tf.where(tf.equal(doc_t, split_token)), axis=1)
    split_token_index.set_shape([None])
    split_len_part_1 = split_token_index[:1] + 1
    split_len_part_2 = split_token_index[1:] - split_token_index[:-1]
    split_lens = tf.concat([split_len_part_1, split_len_part_2], axis=0)
    split_lens = cut_or_padding(
        split_lens, max_doc_len, padding_token=padding_token)
    new_doc_len = tf.reduce_sum(split_lens)
    splited_sentences = tf.split(doc_t[:new_doc_len], split_lens)
    splited_sentences = [
        cut_or_padding(s, max_sen_len) for s in splited_sentences
    ]
    out_t = tf.stack(splited_sentences)
    padding_tokens = tf.multiply(
        tf.ones_like(out_t, dtype=tf.int32), padding_token)
    out_t = tf.where(tf.equal(out_t, split_token), padding_tokens, out_t)
    return out_t

  raise ValueError("doc_t should be a tensor with rank 1.")
def split_one_doc_to_true_len_sens ( doc_t , split_token , padding_token , max_doc_len , max_sen_len ) : if len ( doc_t . get_shape ( ) ) == 1 : split_token_index = tf . squeeze ( tf . where ( tf . equal ( doc_t , split_token ) ) , axis = 1 ) split_token_index . set_shape ( [ None ] ) split_len_part_1 = split_token_index [ : 1 ] + 1 split_len_part_2 = split_token_index [ 1 : ] - split_token_index [ : - 1 ] split_lens = tf . concat ( [ split_len_part_1 , split_len_part_2 ] , axis = 0 ) split_lens = cut_or_padding ( split_lens , max_doc_len , padding_token = padding_token ) new_doc_len = tf . reduce_sum ( split_lens ) splited_sentences = tf . split ( doc_t [ : new_doc_len ] , split_lens ) splited_sentences = [ cut_or_padding ( s , max_sen_len ) for s in splited_sentences ] out_t = tf . stack ( splited_sentences ) padding_tokens = tf . multiply ( tf . ones_like ( out_t , dtype = tf . int32 ) , padding_token ) out_t = tf . where ( tf . equal ( out_t , split_token ) , padding_tokens , out_t ) return out_t raise ValueError ( "doc_t should be a tensor with rank 1." )