()
return entity_alias
统计训练数据中不能链接到实体库的mentioin, 统计出现次数，将其添加到对应实体的别名中
    :return: 字典形式 key 为实体名字 value 为添加的新的别名字典
             如：'bilibili': {'b站', '哔哩哔哩', '哔哩哔哩弹幕视频网'}
统计训练数据中不能链接到实体库的mentioin, 统计出现次数，将其添加到对应实体的别名中
    :return: 字典形式 key 为实体名字 value 为添加的新的别名字典
             如：'bilibili': {'b站', '哔哩哔哩', '哔哩哔哩弹幕视频网'}
统计训练数据中不能链接到实体库的mentioin 统计出现次数，将其添加到对应实体的别名中 : return : 字典形式 key 为实体名字 value 为添加的新的别名字典 如： bilibili : { b站 哔哩哔哩 哔哩哔哩弹幕视频网 }
def new_alias():
    '''
    统计训练数据中不能链接到实体库的mentioin, 统计出现次数，将其添加到对应实体的别名中
    :return: 字典形式 key 为实体名字 value 为添加的新的别名字典
             如：'bilibili': {'b站', '哔哩哔哩', '哔哩哔哩弹幕视频网'}
    '''
    id_alias = {}
    entity_id={}
    id_entity={}
    with open('original_data/kb_data', 'r') as f:
        for line in f:
            temDict = json.loads(line)
            subject = temDict['subject']
            subject_id = temDict['subject_id']
            alias = set()
            for a in temDict['alias']:
                alias.add(a)
                alias.add(a.lower())
            alias.add(subject.lower())
            alias.add(entity_clear(subject))
            id_alias[subject_id] = alias
            subject_id = temDict['subject_id']
            entity_name = set(alias)
            entity_name.add(subject)
            entity_name.add(subject.lower())
            for a in alias:
                entity_name.add(a.lower())
            id_entity[subject_id] = subject
            for n in entity_name:
                if n in entity_id:
                    entity_id[n].add(subject_id)
                else:
                    entity_id[n] = set()
                    entity_id[n].add(subject_id)
    with open('original_data/train.json') as f:
        entity_alias_num={}
        for line in f:
            temDict = json.loads(line)
            mention_data=temDict['mention_data']
            for men in mention_data:
                mention=men['mention']
                kb_id = men['kb_id']
                if kb_id != 'NIL':
                    if id_entity[kb_id]!=mention:
                        if mention not in id_alias[kb_id]:
                            if id_entity[kb_id] in entity_alias_num:
                                entity_alias_num[id_entity[kb_id]]['count'] +=1
                                if mention in entity_alias_num[id_entity[kb_id]]:
                                    entity_alias_num[id_entity[kb_id]][mention] += 1
                                else:
                                    entity_alias_num[id_entity[kb_id]][mention] = 1
                            else:
                                entity_alias_num[id_entity[kb_id]]={}
                                entity_alias_num[id_entity[kb_id]]['count']=1
                                entity_alias_num[id_entity[kb_id]][mention]=1
    entity_alias={}
    for en in entity_alias_num:
        total_num=entity_alias_num[en]['count']
        if total_num>4:
            entity_alias[en]=set()
            for alias in entity_alias_num[en]:
                if alias=='count':
                    continue
                a_num=entity_alias_num[en][alias]
                if a_num>3:
                    entity_alias[en].add(alias)
            if len(entity_alias[en])==0:
                entity_alias.pop(en)
    return entity_alias
def new_alias ( ) : id_alias = { } entity_id = { } id_entity = { } with open ( 'original_data/kb_data' , 'r' ) as f : for line in f : temDict = json . loads ( line ) subject = temDict [ 'subject' ] subject_id = temDict [ 'subject_id' ] alias = set ( ) for a in temDict [ 'alias' ] : alias . add ( a ) alias . add ( a . lower ( ) ) alias . add ( subject . lower ( ) ) alias . add ( entity_clear ( subject ) ) id_alias [ subject_id ] = alias subject_id = temDict [ 'subject_id' ] entity_name = set ( alias ) entity_name . add ( subject ) entity_name . add ( subject . lower ( ) ) for a in alias : entity_name . add ( a . lower ( ) ) id_entity [ subject_id ] = subject for n in entity_name : if n in entity_id : entity_id [ n ] . add ( subject_id ) else : entity_id [ n ] = set ( ) entity_id [ n ] . add ( subject_id ) with open ( 'original_data/train.json' ) as f : entity_alias_num = { } for line in f : temDict = json . loads ( line ) mention_data = temDict [ 'mention_data' ] for men in mention_data : mention = men [ 'mention' ] kb_id = men [ 'kb_id' ] if kb_id != 'NIL' : if id_entity [ kb_id ] != mention : if mention not in id_alias [ kb_id ] : if id_entity [ kb_id ] in entity_alias_num : entity_alias_num [ id_entity [ kb_id ] ] [ 'count' ] += 1 if mention in entity_alias_num [ id_entity [ kb_id ] ] : entity_alias_num [ id_entity [ kb_id ] ] [ mention ] += 1 else : entity_alias_num [ id_entity [ kb_id ] ] [ mention ] = 1 else : entity_alias_num [ id_entity [ kb_id ] ] = { } entity_alias_num [ id_entity [ kb_id ] ] [ 'count' ] = 1 entity_alias_num [ id_entity [ kb_id ] ] [ mention ] = 1 entity_alias = { } for en in entity_alias_num : total_num = entity_alias_num [ en ] [ 'count' ] if total_num > 4 : entity_alias [ en ] = set ( ) for alias in entity_alias_num [ en ] : if alias == 'count' : continue a_num = entity_alias_num [ en ] [ alias ] if a_num > 3 : entity_alias [ en ] . add ( alias ) if len ( entity_alias [ en ] ) == 0 : entity_alias . pop ( en ) return entity_alias