()
消歧输入的构建
    :return:
消歧输入的构建
    :return:
消歧输入的构建 : return :
def get_input():
    '''
    消歧输入的构建
    :return:
    '''
    id_type=pd.read_pickle('data/id_type.pkl')
    type_index=pd.read_pickle('data/type_index.pkl')
    entity_id = pd.read_pickle('data/entity_id.pkl')
    id_entity=pd.read_pickle('data/id_entity.pkl')
    id_text=pd.read_pickle('data/id_text.pkl')
    inputs = {'ids': [], 'seg': [],'begin':[],'end':[],'en_type':[],'labels':[]}
    token_dict = get_token_dict()
    tokenizer = Tokenizer(token_dict)
    file_len=0
    trie_obj=get_Trie()
    with open('original_data/train.json') as f:
        for line in f:
            if file_len%100==0:
                print(file_len)
            file_len+=1
            temDict = json.loads(line)
            text=temDict['text']
            match_en = trie_obj.search_entity(text)
            mention_data=temDict['mention_data']
            for men in mention_data:
                mention=men['mention']
                kb_id=men['kb_id']
                offset = men['offset']
                begin=int(offset)+1
                end = begin+len(mention)
                if kb_id != 'NIL':
                    link_id=[kb_id]
                    link_id+=get_link_entity(mention,kb_id,entity_id,id_entity)

                    for id in link_id:
                        kb_text = id_text[id]
                        kb_type=type_index[id_type[id][0]]
                        indice, segment = tokenizer.encode(first=text,second=kb_text,max_len=256)
                        inputs['ids'].append(indice)
                        inputs['seg'].append(segment)
                        inputs['begin'].append([begin])
                        inputs['end'].append([end])
                        if id ==kb_id:
                            inputs['labels'].append(1)
                        else:
                            inputs['labels'].append(0)
                        inputs['en_type'].append([kb_type])
            mention_set=set()
            for men in mention_data:
                mention_set.add((men['mention'],int(men['offset'])))
            for en in match_en:
                if not en in mention_set:

                    link_id = get_link_entity_test(en[0],entity_id)
                    for id in link_id[:1]:
                        kb_text = id_text[id]
                        kb_type = type_index[id_type[id][0]]
                        indice, segment = tokenizer.encode(first=text, second=kb_text, max_len=256)
                        inputs['ids'].append(indice)
                        inputs['seg'].append(segment)
                        inputs['begin'].append([begin])
                        inputs['end'].append([end])
                        inputs['labels'].append(0)
                        inputs['en_type'].append([kb_type])
                    break

    for k in inputs:
        inputs[k]=np.array(inputs[k])
        print(k,inputs[k].shape)
        print(inputs[k][1])
    pd.to_pickle(inputs,'data/train_input_bert_final.pkl')
def get_input ( ) : id_type = pd . read_pickle ( 'data/id_type.pkl' ) type_index = pd . read_pickle ( 'data/type_index.pkl' ) entity_id = pd . read_pickle ( 'data/entity_id.pkl' ) id_entity = pd . read_pickle ( 'data/id_entity.pkl' ) id_text = pd . read_pickle ( 'data/id_text.pkl' ) inputs = { 'ids' : [ ] , 'seg' : [ ] , 'begin' : [ ] , 'end' : [ ] , 'en_type' : [ ] , 'labels' : [ ] } token_dict = get_token_dict ( ) tokenizer = Tokenizer ( token_dict ) file_len = 0 trie_obj = get_Trie ( ) with open ( 'original_data/train.json' ) as f : for line in f : if file_len % 100 == 0 : print ( file_len ) file_len += 1 temDict = json . loads ( line ) text = temDict [ 'text' ] match_en = trie_obj . search_entity ( text ) mention_data = temDict [ 'mention_data' ] for men in mention_data : mention = men [ 'mention' ] kb_id = men [ 'kb_id' ] offset = men [ 'offset' ] begin = int ( offset ) + 1 end = begin + len ( mention ) if kb_id != 'NIL' : link_id = [ kb_id ] link_id += get_link_entity ( mention , kb_id , entity_id , id_entity ) for id in link_id : kb_text = id_text [ id ] kb_type = type_index [ id_type [ id ] [ 0 ] ] indice , segment = tokenizer . encode ( first = text , second = kb_text , max_len = 256 ) inputs [ 'ids' ] . append ( indice ) inputs [ 'seg' ] . append ( segment ) inputs [ 'begin' ] . append ( [ begin ] ) inputs [ 'end' ] . append ( [ end ] ) if id == kb_id : inputs [ 'labels' ] . append ( 1 ) else : inputs [ 'labels' ] . append ( 0 ) inputs [ 'en_type' ] . append ( [ kb_type ] ) mention_set = set ( ) for men in mention_data : mention_set . add ( ( men [ 'mention' ] , int ( men [ 'offset' ] ) ) ) for en in match_en : if not en in mention_set : link_id = get_link_entity_test ( en [ 0 ] , entity_id ) for id in link_id [ : 1 ] : kb_text = id_text [ id ] kb_type = type_index [ id_type [ id ] [ 0 ] ] indice , segment = tokenizer . encode ( first = text , second = kb_text , max_len = 256 ) inputs [ 'ids' ] . append ( indice ) inputs [ 'seg' ] . append ( segment ) inputs [ 'begin' ] . append ( [ begin ] ) inputs [ 'end' ] . append ( [ end ] ) inputs [ 'labels' ] . append ( 0 ) inputs [ 'en_type' ] . append ( [ kb_type ] ) break for k in inputs : inputs [ k ] = np . array ( inputs [ k ] ) print ( k , inputs [ k ] . shape ) print ( inputs [ k ] [ 1 ] ) pd . to_pickle ( inputs , 'data/train_input_bert_final.pkl' )