(input_file,out_file,max_len=52)
构建模型的训练集输入
    :param input_file:
    :param out_file:
    :param max_len:
    :return:
构建模型的训练集输入
    :param input_file:
    :param out_file:
    :param max_len:
    :return:
构建模型的训练集输入 : param input_file : : param out_file : : param max_len : : return :
def get_input_bert(input_file,out_file,max_len=52):
    '''
    构建模型的训练集输入
    :param input_file:
    :param out_file:
    :param max_len:
    :return:
    '''
    token_dict = get_token_dict()
    tokenizer = Tokenizer(token_dict)
    split_entity = pd.read_pickle('data/split_entity.pkl')

    trie_obj = get_Trie()
    entity_index = pd.read_pickle('data/ER_entity_index.pkl')
    print(len(entity_index))
    inputs = {'ids': [], 'seg': [], 'begin': [], 'end': [],'labels': [], 'entity_id': [], 'match_en': []}
    match_len=0
    with open(input_file, 'r') as f:
        for line in f:
            temDict = json.loads(line)
            text = temDict['text']
            indices, segments = get_ids_seg(text, tokenizer)
            mention_data = temDict['mention_data']
            inputs['ids'].append(seq_padding(indices, max_len))
            inputs['seg'].append(seq_padding(segments, max_len))
            match_en = trie_obj.search_entity(text)
            if len(match_en)>match_len:
                match_len=len(match_en)
            match_en = get_match_entity(match_en,split_entity)
            label2 = [0] * len(match_en)

            en_list = []
            for en in match_en:
                if en[0] in entity_index:
                    en_list.append(entity_index[en[0]])
                else:
                    en_list.append(0)
                    print(en[0])
            begin = []
            end = []
            men_set = set()

            for men in mention_data:
                if men['kb_id'] != 'NIL':
                    men_set.add((men['mention'], int(men['offset'])))
            text = temDict['text']
            for i, en in enumerate(match_en):
                begin.append(en[1] + 1)
                end.append(en[1] + len(en[0]))
                if (text[en[1]:en[1] + len(en[0])], en[1]) in men_set:
                    label2[i] = 1
            inputs['labels'].append(seq_padding(label2, 13,value=0))
            inputs['begin'].append(seq_padding(begin, 13, value=-1))
            inputs['end'].append(seq_padding(end, 13, value=-1))
            inputs['entity_id'].append(seq_padding(en_list, 13))
            inputs['match_en'].append(match_en)

    for key in inputs:
        inputs[key] = np.array(inputs[key])
        print(key, inputs[key].shape)
        print(inputs[key][0])
    pd.to_pickle(inputs, out_file)

    print(match_len)
def get_input_bert ( input_file , out_file , max_len = 52 ) : token_dict = get_token_dict ( ) tokenizer = Tokenizer ( token_dict ) split_entity = pd . read_pickle ( 'data/split_entity.pkl' ) trie_obj = get_Trie ( ) entity_index = pd . read_pickle ( 'data/ER_entity_index.pkl' ) print ( len ( entity_index ) ) inputs = { 'ids' : [ ] , 'seg' : [ ] , 'begin' : [ ] , 'end' : [ ] , 'labels' : [ ] , 'entity_id' : [ ] , 'match_en' : [ ] } match_len = 0 with open ( input_file , 'r' ) as f : for line in f : temDict = json . loads ( line ) text = temDict [ 'text' ] indices , segments = get_ids_seg ( text , tokenizer ) mention_data = temDict [ 'mention_data' ] inputs [ 'ids' ] . append ( seq_padding ( indices , max_len ) ) inputs [ 'seg' ] . append ( seq_padding ( segments , max_len ) ) match_en = trie_obj . search_entity ( text ) if len ( match_en ) > match_len : match_len = len ( match_en ) match_en = get_match_entity ( match_en , split_entity ) label2 = [ 0 ] * len ( match_en ) en_list = [ ] for en in match_en : if en [ 0 ] in entity_index : en_list . append ( entity_index [ en [ 0 ] ] ) else : en_list . append ( 0 ) print ( en [ 0 ] ) begin = [ ] end = [ ] men_set = set ( ) for men in mention_data : if men [ 'kb_id' ] != 'NIL' : men_set . add ( ( men [ 'mention' ] , int ( men [ 'offset' ] ) ) ) text = temDict [ 'text' ] for i , en in enumerate ( match_en ) : begin . append ( en [ 1 ] + 1 ) end . append ( en [ 1 ] + len ( en [ 0 ] ) ) if ( text [ en [ 1 ] : en [ 1 ] + len ( en [ 0 ] ) ] , en [ 1 ] ) in men_set : label2 [ i ] = 1 inputs [ 'labels' ] . append ( seq_padding ( label2 , 13 , value = 0 ) ) inputs [ 'begin' ] . append ( seq_padding ( begin , 13 , value = - 1 ) ) inputs [ 'end' ] . append ( seq_padding ( end , 13 , value = - 1 ) ) inputs [ 'entity_id' ] . append ( seq_padding ( en_list , 13 ) ) inputs [ 'match_en' ] . append ( match_en ) for key in inputs : inputs [ key ] = np . array ( inputs [ key ] ) print ( key , inputs [ key ] . shape ) print ( inputs [ key ] [ 0 ] ) pd . to_pickle ( inputs , out_file ) print ( match_len )