(max_seq_len, embed_dim, word_level=True)
return (train_q1_indices,train_q2_indices, label, embed_matrix)
读取数据，对数据进行预处理，并生成embed_matrix
读取数据，对数据进行预处理，并生成embed_matrix
读取数据，对数据进行预处理，并生成embed_matrix
def load_dataset(max_seq_len, embed_dim, word_level=True):
    '''
    读取数据，对数据进行预处理，并生成embed_matrix
    '''
    #1、读取数据，数据预处理
    #数据路径
    question_path = os.path.join('datasets', 'question.csv')
    train_path = os.path.join('datasets', 'train.csv')
    if word_level:
        embed_path = os.path.join('datasets', 'word_embed.txt')
    else:
        embed_path = os.path.join('datasets', 'char_embed.txt')
    
    #读取数据
    question = pd.read_csv(question_path)
    
    train = pd.read_csv(train_path)
    # 把train里面的问题id匹配到句子
    train = pd.merge(train,question,left_on=['q1'],right_on=['qid'],how='left')
    train = pd.merge(train,question,left_on=['q2'],right_on=['qid'],how='left')
    
    if word_level:
        train = train[['label','words_x','words_y']]
    else:
        train = train[['label','chars_x','chars_y']]
    train.columns = ['label','q1','q2']
    
    # 读取word_to_vec_map，注意这里的index是word id
    word_to_vec_map = pd.read_csv(embed_path, sep=' ', header=None, index_col=0)
    
    # 先定义两个字典，实现wid与(positive) index的相互转换,注意index从1开始
    word = word_to_vec_map.index.values
    word_to_index = dict([(word[i],i+1) for i in range(len(word))])
    index_to_word = dict([(i+1, word[i]) for i in range(len(word))])
    
    # 把句子转换成int indices,并zero pad the sentance to max_seq_len
    train_q1_indices = sentences_to_indices(train.q1.values, word_to_index, max_seq_len)
    train_q2_indices = sentences_to_indices(train.q2.values, word_to_index, max_seq_len)
    label = train.label.values
    
    #3、生成embeding_matrix, index为整数，其中index=0,对应的是np.zeros(300),0向量，对应我们padding的值
    vocab_len = len(word_to_index) + 1                                   
    # Initialize the embedding matrix as numpy arrays of zeros
    embed_matrix = np.zeros((vocab_len, embed_dim))
    # Set each row "index" of the embedding matrix to be the word vector representation of the "index"th word of the vocabulary
    for word, index in word_to_index.items():
        embed_matrix[index, :] = word_to_vec_map.loc[word].values

    return (train_q1_indices,train_q2_indices, label, embed_matrix)
def load_dataset ( max_seq_len , embed_dim , word_level = True ) : #1、读取数据，数据预处理 #数据路径 question_path = os . path . join ( 'datasets' , 'question.csv' ) train_path = os . path . join ( 'datasets' , 'train.csv' ) if word_level : embed_path = os . path . join ( 'datasets' , 'word_embed.txt' ) else : embed_path = os . path . join ( 'datasets' , 'char_embed.txt' ) #读取数据 question = pd . read_csv ( question_path ) train = pd . read_csv ( train_path ) # 把train里面的问题id匹配到句子 train = pd . merge ( train , question , left_on = [ 'q1' ] , right_on = [ 'qid' ] , how = 'left' ) train = pd . merge ( train , question , left_on = [ 'q2' ] , right_on = [ 'qid' ] , how = 'left' ) if word_level : train = train [ [ 'label' , 'words_x' , 'words_y' ] ] else : train = train [ [ 'label' , 'chars_x' , 'chars_y' ] ] train . columns = [ 'label' , 'q1' , 'q2' ] # 读取word_to_vec_map，注意这里的index是word id word_to_vec_map = pd . read_csv ( embed_path , sep = ' ' , header = None , index_col = 0 ) # 先定义两个字典，实现wid与(positive) index的相互转换,注意index从1开始 word = word_to_vec_map . index . values word_to_index = dict ( [ ( word [ i ] , i + 1 ) for i in range ( len ( word ) ) ] ) index_to_word = dict ( [ ( i + 1 , word [ i ] ) for i in range ( len ( word ) ) ] ) # 把句子转换成int indices,并zero pad the sentance to max_seq_len train_q1_indices = sentences_to_indices ( train . q1 . values , word_to_index , max_seq_len ) train_q2_indices = sentences_to_indices ( train . q2 . values , word_to_index , max_seq_len ) label = train . label . values #3、生成embeding_matrix, index为整数，其中index=0,对应的是np.zeros(300),0向量，对应我们padding的值 vocab_len = len ( word_to_index ) + 1 # Initialize the embedding matrix as numpy arrays of zeros embed_matrix = np . zeros ( ( vocab_len , embed_dim ) ) # Set each row "index" of the embedding matrix to be the word vector representation of the "index"th word of the vocabulary for word , index in word_to_index . items ( ) : embed_matrix [ index , : ] = word_to_vec_map . loc [ word ] . values return ( train_q1_indices , train_q2_indices , label , embed_matrix )