()
知识库处理
    :return: 得到后续要用的一些文件具体为：
            entity_id字典  key:entity name  value: kb_id list
                        '胜利': ['10001', '19044', '37234', '38870', '40008', '85426', '86532', '140750']
            id_entity字典 key:kb_id value:subject(实体名字)
                         10001 胜利
            id_text字典 key：kb_id value:实体描述文本
                       '10001': '摘要:英雄联盟胜利系列皮肤是拳头公司制作的具有纪念意义限定系列皮肤之一。'
            id_type字典  key：kb_id value: entity type
                        '10001': ['Thing']

            type_index字典 key：type name value：index
                        ‘NAN’: 0
                        'Thing' :1
知识库处理
    :return: 得到后续要用的一些文件具体为：
            entity_id字典  key:entity name  value: kb_id list
                        '胜利': ['10001', '19044', '37234', '38870', '40008', '85426', '86532', '140750']
            id_entity字典 key:kb_id value:subject(实体名字)
                         10001 胜利
            id_text字典 key：kb_id value:实体描述文本
                       '10001': '摘要:英雄联盟胜利系列皮肤是拳头公司制作的具有纪念意义限定系列皮肤之一。'
            id_type字典  key：kb_id value: entity type
                        '10001': ['Thing']
知识库处理 : return : 得到后续要用的一些文件具体为： entity_id字典 key : entity name value : kb_id list 胜利 : [ 10001 19044 37234 38870 40008 85426 86532 140750 ] id_entity字典 key : kb_id value : subject ( 实体名字 ) 10001 胜利 id_text字典 key：kb_id value : 实体描述文本 10001 : 摘要 : 英雄联盟胜利系列皮肤是拳头公司制作的具有纪念意义限定系列皮肤之一。 id_type字典 key：kb_id value : entity type 10001 : [ Thing ]
def kb_processing():
    '''
    知识库处理
    :return: 得到后续要用的一些文件具体为：
            entity_id字典  key:entity name  value: kb_id list
                        '胜利': ['10001', '19044', '37234', '38870', '40008', '85426', '86532', '140750']
            id_entity字典 key:kb_id value:subject(实体名字)
                         10001 胜利
            id_text字典 key：kb_id value:实体描述文本
                       '10001': '摘要:英雄联盟胜利系列皮肤是拳头公司制作的具有纪念意义限定系列皮肤之一。'
            id_type字典  key：kb_id value: entity type
                        '10001': ['Thing']

            type_index字典 key：type name value：index
                        ‘NAN’: 0
                        'Thing' :1
    '''
    new_entity_alias=new_alias()
    id_text={}
    entity_id={}
    type_index={}
    type_index['NAN']=0
    type_i=1
    id_type={}
    id_entity={}

    with open('original_data/kb_data', 'r') as f:
        for line in f:
            temDict = json.loads(line)
            subject=temDict['subject']
            subject_id=temDict['subject_id']
            alias = set()
            for a in temDict['alias']:
                alias.add(a)
                alias.add(a.lower())
            alias.add(subject.lower())
            alias.add(entity_clear(subject))
            if subject in new_entity_alias:
                alias=alias|new_entity_alias[subject]
            en_data=temDict['data']
            en_type=temDict['type']
            entity_name=set(alias)
            entity_name.add(subject)
            for t in en_type:
                if not t in type_index:
                    type_index[t]=type_i
                    type_i+=1
            for n in entity_name:
                n=del_bookname(n)
                if n in entity_id:
                    entity_id[n].append(subject_id)
                else:
                    entity_id[n]=[]
                    entity_id[n].append(subject_id)
            id_type[subject_id]=en_type
            text=get_text(en_data)
            id_text[subject_id]=text
            id_entity[subject_id]=subject

    pd.to_pickle(entity_id,'data/entity_id.pkl')
    pd.to_pickle(id_entity, 'data/id_entity.pkl')
    pd.to_pickle(type_index, 'data/type_index.pkl')
    pd.to_pickle(id_type, 'data/id_type.pkl')
    pd.to_pickle(id_text, 'data/id_text.pkl')
def kb_processing ( ) : new_entity_alias = new_alias ( ) id_text = { } entity_id = { } type_index = { } type_index [ 'NAN' ] = 0 type_i = 1 id_type = { } id_entity = { } with open ( 'original_data/kb_data' , 'r' ) as f : for line in f : temDict = json . loads ( line ) subject = temDict [ 'subject' ] subject_id = temDict [ 'subject_id' ] alias = set ( ) for a in temDict [ 'alias' ] : alias . add ( a ) alias . add ( a . lower ( ) ) alias . add ( subject . lower ( ) ) alias . add ( entity_clear ( subject ) ) if subject in new_entity_alias : alias = alias | new_entity_alias [ subject ] en_data = temDict [ 'data' ] en_type = temDict [ 'type' ] entity_name = set ( alias ) entity_name . add ( subject ) for t in en_type : if not t in type_index : type_index [ t ] = type_i type_i += 1 for n in entity_name : n = del_bookname ( n ) if n in entity_id : entity_id [ n ] . append ( subject_id ) else : entity_id [ n ] = [ ] entity_id [ n ] . append ( subject_id ) id_type [ subject_id ] = en_type text = get_text ( en_data ) id_text [ subject_id ] = text id_entity [ subject_id ] = subject pd . to_pickle ( entity_id , 'data/entity_id.pkl' ) pd . to_pickle ( id_entity , 'data/id_entity.pkl' ) pd . to_pickle ( type_index , 'data/type_index.pkl' ) pd . to_pickle ( id_type , 'data/id_type.pkl' ) pd . to_pickle ( id_text , 'data/id_text.pkl' )