(input_file,out_file,max_len=52)
构建模型测试集输入
    :param input_file:
    :param out_file:
    :param max_len:
    :return:
构建模型测试集输入
    :param input_file:
    :param out_file:
    :param max_len:
    :return:
构建模型测试集输入 : param input_file : : param out_file : : param max_len : : return :
def get_input_bert_test(input_file,out_file,max_len=52):
    '''
    构建模型测试集输入
    :param input_file:
    :param out_file:
    :param max_len:
    :return:
    '''
    token_dict = get_token_dict()
    tokenizer = Tokenizer(token_dict)
    trie_obj = get_Trie()
    split_entity = pd.read_pickle('data/split_entity.pkl')
    entity_index = pd.read_pickle('data/ER_entity_index.pkl')
    inputs = {'ids': [], 'seg': [], 'begin': [], 'end': [], 'labels': [0], 'entity_id': [],
              'match_en': []}

    with open(input_file, 'r') as f:
        for line in f:
            temDict = json.loads(line)
            text = temDict['text']
            indices, segments = get_ids_seg(text, tokenizer)
            inputs['ids'].append(seq_padding(indices, max_len))
            inputs['seg'].append(seq_padding(segments, max_len))
            match_en = trie_obj.search_entity(text)
            match_en = get_match_entity(match_en, split_entity)
            en_list = [entity_index[en[0]] for en in match_en]
            begin = []
            end = []
            for i, en in enumerate(match_en):
                begin.append(en[1] + 1)
                end.append(en[1] + len(en[0]))
            inputs['begin'].append(seq_padding(begin, 13, value=-1))
            inputs['end'].append(seq_padding(end, 13, value=-1))
            inputs['entity_id'].append(seq_padding(en_list, 13))
            inputs['match_en'].append(match_en)
    for key in inputs:
        inputs[key] = np.array(inputs[key])
        print(key, inputs[key].shape)
        print(inputs[key][0])
    pd.to_pickle(inputs, out_file)
def get_input_bert_test ( input_file , out_file , max_len = 52 ) : token_dict = get_token_dict ( ) tokenizer = Tokenizer ( token_dict ) trie_obj = get_Trie ( ) split_entity = pd . read_pickle ( 'data/split_entity.pkl' ) entity_index = pd . read_pickle ( 'data/ER_entity_index.pkl' ) inputs = { 'ids' : [ ] , 'seg' : [ ] , 'begin' : [ ] , 'end' : [ ] , 'labels' : [ 0 ] , 'entity_id' : [ ] , 'match_en' : [ ] } with open ( input_file , 'r' ) as f : for line in f : temDict = json . loads ( line ) text = temDict [ 'text' ] indices , segments = get_ids_seg ( text , tokenizer ) inputs [ 'ids' ] . append ( seq_padding ( indices , max_len ) ) inputs [ 'seg' ] . append ( seq_padding ( segments , max_len ) ) match_en = trie_obj . search_entity ( text ) match_en = get_match_entity ( match_en , split_entity ) en_list = [ entity_index [ en [ 0 ] ] for en in match_en ] begin = [ ] end = [ ] for i , en in enumerate ( match_en ) : begin . append ( en [ 1 ] + 1 ) end . append ( en [ 1 ] + len ( en [ 0 ] ) ) inputs [ 'begin' ] . append ( seq_padding ( begin , 13 , value = - 1 ) ) inputs [ 'end' ] . append ( seq_padding ( end , 13 , value = - 1 ) ) inputs [ 'entity_id' ] . append ( seq_padding ( en_list , 13 ) ) inputs [ 'match_en' ] . append ( match_en ) for key in inputs : inputs [ key ] = np . array ( inputs [ key ] ) print ( key , inputs [ key ] . shape ) print ( inputs [ key ] [ 0 ] ) pd . to_pickle ( inputs , out_file )