(data_dir, word_to_id, cat_to_id, seq_length=512)
return np.array(data_id), np.array(label_id)
:param data_dir:
    :param word_to_id:
    :param cat_to_id:
    :param seq_length:
    :return:
def process_file(data_dir, word_to_id, cat_to_id, seq_length=512):
    """

    :param data_dir:
    :param word_to_id:
    :param cat_to_id:
    :param seq_length:
    :return:
    """
    contents, labels = read_file(data_dir)

    data_id, label_id = [], []
    for i in range(len(contents)):
        sent_ids = [word_to_id.get(w) if w in word_to_id else word_to_id.get("<UNK>") for w in contents[i]]
        # pad to the required length
        if len(sent_ids) > seq_length:
            sent_ids = sent_ids[:seq_length]
        else:
            padding = [0] * (seq_length - len(sent_ids))
            sent_ids += padding
        data_id.append(sent_ids)
        y_pad = [0] * len(cat_to_id)
        y_pad[cat_to_id[labels[i]]] = 1
        label_id.append(y_pad)

    return np.array(data_id), np.array(label_id)
def process_file ( data_dir , word_to_id , cat_to_id , seq_length = 512 ) : contents , labels = read_file ( data_dir ) data_id , label_id = [ ] , [ ] for i in range ( len ( contents ) ) : sent_ids = [ word_to_id . get ( w ) if w in word_to_id else word_to_id . get ( "<UNK>" ) for w in contents [ i ] ] # pad to the required length if len ( sent_ids ) > seq_length : sent_ids = sent_ids [ : seq_length ] else : padding = [ 0 ] * ( seq_length - len ( sent_ids ) ) sent_ids += padding data_id . append ( sent_ids ) y_pad = [ 0 ] * len ( cat_to_id ) y_pad [ cat_to_id [ labels [ i ] ] ] = 1 label_id . append ( y_pad ) return np . array ( data_id ) , np . array ( label_id )