(max_seq_len, word_level=True)
return test_q1_indices,test_q2_indices
读取测试数据
读取测试数据
读取测试数据
def load_test_data(max_seq_len, word_level=True):
    '''
    读取测试数据
    '''
    #1、读取数据，数据预处理
    #数据路径
    question_path = os.path.join('datasets', 'question.csv')
    test_path = os.path.join('datasets', 'test.csv')
    if word_level:
        embed_path = os.path.join('datasets', 'word_embed.txt')
    else:
        embed_path = os.path.join('datasets', 'char_embed.txt')
    
    #读取数据
    question = pd.read_csv(question_path)
    test = pd.read_csv(test_path)
    # 把train里面的问题id匹配到句子
    test = pd.merge(test,question,left_on=['q1'],right_on=['qid'],how='left')
    test = pd.merge(test,question,left_on=['q2'],right_on=['qid'],how='left')
    if word_level:
        test = test[['words_x','words_y']]
    else:
        test = test[['chars_x','chars_y']]
    test.columns = ['q1','q2']
    # 读取word_to_vec_map，注意这里的index是word id
    word_to_vec_map = pd.read_csv(embed_path, sep=' ', header=None, index_col=0)
    
    # 先定义两个字典，实现wid与(positive) index的相互转换,注意index从1开始
    word = word_to_vec_map.index.values
    word_to_index = dict([(word[i],i+1) for i in range(len(word))])
    index_to_word = dict([(i+1, word[i]) for i in range(len(word))])
    
    # 把句子转换成int indices,并zero pad the sentance to max_seq_len
    test_q1_indices = sentences_to_indices(test.q1.values, word_to_index, max_seq_len).astype(np.int32)
    test_q2_indices = sentences_to_indices(test.q2.values, word_to_index, max_seq_len).astype(np.int32)
    
    
    return test_q1_indices,test_q2_indices
def load_test_data ( max_seq_len , word_level = True ) : #1、读取数据，数据预处理 #数据路径 question_path = os . path . join ( 'datasets' , 'question.csv' ) test_path = os . path . join ( 'datasets' , 'test.csv' ) if word_level : embed_path = os . path . join ( 'datasets' , 'word_embed.txt' ) else : embed_path = os . path . join ( 'datasets' , 'char_embed.txt' ) #读取数据 question = pd . read_csv ( question_path ) test = pd . read_csv ( test_path ) # 把train里面的问题id匹配到句子 test = pd . merge ( test , question , left_on = [ 'q1' ] , right_on = [ 'qid' ] , how = 'left' ) test = pd . merge ( test , question , left_on = [ 'q2' ] , right_on = [ 'qid' ] , how = 'left' ) if word_level : test = test [ [ 'words_x' , 'words_y' ] ] else : test = test [ [ 'chars_x' , 'chars_y' ] ] test . columns = [ 'q1' , 'q2' ] # 读取word_to_vec_map，注意这里的index是word id word_to_vec_map = pd . read_csv ( embed_path , sep = ' ' , header = None , index_col = 0 ) # 先定义两个字典，实现wid与(positive) index的相互转换,注意index从1开始 word = word_to_vec_map . index . values word_to_index = dict ( [ ( word [ i ] , i + 1 ) for i in range ( len ( word ) ) ] ) index_to_word = dict ( [ ( i + 1 , word [ i ] ) for i in range ( len ( word ) ) ] ) # 把句子转换成int indices,并zero pad the sentance to max_seq_len test_q1_indices = sentences_to_indices ( test . q1 . values , word_to_index , max_seq_len ) . astype ( np . int32 ) test_q2_indices = sentences_to_indices ( test . q2 . values , word_to_index , max_seq_len ) . astype ( np . int32 ) return test_q1_indices , test_q2_indices