(self, file_name, graph_retriever_config, set_type)
return examples
Limit the number of examples used.
        This is mainly for sanity-chacking new settings.
Limit the number of examples used.
        This is mainly for sanity-chacking new settings.
Limit the number of examples used . This is mainly for sanity - chacking new settings .
def _create_examples(self, file_name, graph_retriever_config, set_type):

        task = graph_retriever_config.task
        jsn = json.load(open(file_name, 'r'))
        
        examples = []

        '''
        Limit the number of examples used.
        This is mainly for sanity-chacking new settings.
        '''
        if graph_retriever_config.example_limit is not None:
            random.shuffle(jsn)
            jsn = sorted(jsn, key = lambda x: x['q_id'])
            jsn = jsn[:graph_retriever_config.example_limit]

        '''
        Find the mximum size of the initial context (links are not included)
        '''
        graph_retriever_config.max_context_size = 0
            
        logger.info('#### Loading examples... from {} ####'.format(file_name))
        for (_, data) in enumerate(tqdm(jsn, desc='Example')):

            guid = data['q_id']
            question = data['question']
            context = data['context'] # {context title: paragraph}
            all_linked_paras_dic = data['all_linked_paras_dic'] # {context title: {linked title: paragraph}}
            short_gold = data['short_gold'] # [title 1, title 2] (Both are gold)
            redundant_gold = data['redundant_gold'] # [title 1, title 2, title 3] ("title 1" is not gold)
            all_redundant_gold = data['all_redundant_gold']

            '''
            Limit the number of redundant examples
            '''
            all_redundant_gold = all_redundant_gold[:graph_retriever_config.max_redundant_num]

            '''
            Control the size of the initial TF-IDF retrieved paragraphs
            *** Training time: to take a blalance between TF-IDF-based and link-based negative examples ***
            '''
            if graph_retriever_config.tfidf_limit is not None:
                new_context = {}
                for title in context:
                    if len(new_context) == graph_retriever_config.tfidf_limit:
                        break
                    new_context[title] = context[title]
                context = new_context

            '''
            Use TagMe-based context at test time.
            '''
            if set_type == 'dev' and task == 'nq' and graph_retriever_config.tagme:
                assert 'tagged_context' in data

                '''
                Reformat "tagged_context" if needed (c.f. the "context" case above)
                '''
                if type(data['tagged_context']) == list:
                    tagged_context = {c[0]: c[1] for c in data['tagged_context']}
                    data['tagged_context'] = tagged_context

                '''
                Append valid paragraphs from "tagged_context" to "context"
                '''
                for tagged_title in data['tagged_context']:
                    tagged_text = data['tagged_context'][tagged_title]
                    if tagged_title not in context and tagged_title is not None and tagged_title.strip() != '' and tagged_text is not None and tagged_text.strip() != '':
                        context[tagged_title] = tagged_text

            '''
            Clean "context" by removing invalid paragraphs
            '''
            removed_keys = []
            for title in context:
                if title is None or title.strip() == '' or context[title] is None or context[title].strip() == '':
                    removed_keys.append(title)
            for key in removed_keys:
                context.pop(key)

            if task in ['squad', 'nq'] and set_type == 'train':
                new_context = {}

                orig_title = list(context.keys())[0].split('_')[0]
                
                orig_titles = []
                other_titles = []

                for title in context:
                    title_ = title.split('_')[0]

                    if title_ == orig_title:
                        orig_titles.append(title)
                    else:
                        other_titles.append(title)

                orig_index = 0
                other_index = 0

                while orig_index < len(orig_titles) or other_index < len(other_titles):
                    if orig_index < len(orig_titles):
                        new_context[orig_titles[orig_index]] = context[orig_titles[orig_index]]
                        orig_index += 1

                    if other_index < len(other_titles):
                        new_context[other_titles[other_index]] = context[other_titles[other_index]]
                        other_index += 1

                context = new_context


            '''
            Convert link format
            '''
            new_all_linked_paras_dic = {} # {context title: {linked title: paragraph}}

            all_linked_paras_dic # {linked_title: paragraph} or mixed
            all_linked_para_title_dic = data['all_linked_para_title_dic'] # {context_title: [linked_title_1, linked_title_2, ...]}

            removed_keys = []
            tmp = {}
            for key in all_linked_paras_dic:
                if type(all_linked_paras_dic[key]) == dict:
                    removed_keys.append(key)

                    for linked_title in all_linked_paras_dic[key]:
                        if linked_title not in all_linked_paras_dic:
                            tmp[linked_title] = all_linked_paras_dic[key][linked_title]

                        if key in all_linked_para_title_dic:
                            all_linked_para_title_dic[key].append(linked_title)
                        else:
                            all_linked_para_title_dic[key] = [linked_title]

            for key in removed_keys:
                all_linked_paras_dic.pop(key)

            for key in tmp:
                if key not in all_linked_paras_dic:
                    all_linked_paras_dic[key] = tmp[key]

            for context_title in context:
                if context_title not in all_linked_para_title_dic:
                    continue

                new_entry = {}

                for linked_title in all_linked_para_title_dic[context_title]:
                    if linked_title not in all_linked_paras_dic:
                        continue

                    new_entry[linked_title] = all_linked_paras_dic[linked_title]

                if len(new_entry) > 0:
                    new_all_linked_paras_dic[context_title] = new_entry

            all_linked_paras_dic = new_all_linked_paras_dic

            if set_type == 'dev':
                '''
                Clean "all_linked_paras_dic" by removing invalid paragraphs
                '''
                for c in all_linked_paras_dic:
                    removed_keys = []
                    links = all_linked_paras_dic[c]
                    for title in links:
                        if title is None or title.strip() == '' or links[title] is None or type(links[title]) != str or links[title].strip() == '':
                            removed_keys.append(title)
                    for key in removed_keys:
                        links.pop(key)

                all_paras = {}
                for title in context:
                    all_paras[title] = context[title]

                    if not graph_retriever_config.open:
                        continue
                    
                    if title not in all_linked_paras_dic:
                        continue
                    for title_ in all_linked_paras_dic[title]:
                        if title_ not in all_paras:
                            all_paras[title_] = all_linked_paras_dic[title][title_]
            else:
                all_paras = None

            if set_type == 'dev' and graph_retriever_config.expand_links:
                expand_links(context, all_linked_paras_dic, all_paras)

            if set_type == 'dev' and graph_retriever_config.no_links:
                all_linked_paras_dic = {}
                            
            graph_retriever_config.max_context_size = max(graph_retriever_config.max_context_size, len(context))

            '''
            Ensure that all the gold paragraphs are included in "context"
            '''
            if set_type == 'train':
                for t in short_gold + redundant_gold:
                    assert t in context
            
            examples.append(InputExample(guid = guid,
                                         q = question,
                                         c = context,
                                         para_dic = all_linked_paras_dic,
                                         s_g = short_gold,
                                         r_g = redundant_gold,
                                         all_r_g = all_redundant_gold,
                                         all_paras = all_paras))

        if set_type == 'dev':
            examples = sorted(examples, key = lambda x: len(x.all_paras))
        logger.info('Done!')
        
        return examples
def _create_examples ( self , file_name , graph_retriever_config , set_type ) : task = graph_retriever_config . task jsn = json . load ( open ( file_name , 'r' ) ) examples = [ ] if graph_retriever_config . example_limit is not None : random . shuffle ( jsn ) jsn = sorted ( jsn , key = lambda x : x [ 'q_id' ] ) jsn = jsn [ : graph_retriever_config . example_limit ] '''
        Find the mximum size of the initial context (links are not included)
        ''' graph_retriever_config . max_context_size = 0 logger . info ( '#### Loading examples... from {} ####' . format ( file_name ) ) for ( _ , data ) in enumerate ( tqdm ( jsn , desc = 'Example' ) ) : guid = data [ 'q_id' ] question = data [ 'question' ] context = data [ 'context' ] # {context title: paragraph} all_linked_paras_dic = data [ 'all_linked_paras_dic' ] # {context title: {linked title: paragraph}} short_gold = data [ 'short_gold' ] # [title 1, title 2] (Both are gold) redundant_gold = data [ 'redundant_gold' ] # [title 1, title 2, title 3] ("title 1" is not gold) all_redundant_gold = data [ 'all_redundant_gold' ] '''
            Limit the number of redundant examples
            ''' all_redundant_gold = all_redundant_gold [ : graph_retriever_config . max_redundant_num ] '''
            Control the size of the initial TF-IDF retrieved paragraphs
            *** Training time: to take a blalance between TF-IDF-based and link-based negative examples ***
            ''' if graph_retriever_config . tfidf_limit is not None : new_context = { } for title in context : if len ( new_context ) == graph_retriever_config . tfidf_limit : break new_context [ title ] = context [ title ] context = new_context '''
            Use TagMe-based context at test time.
            ''' if set_type == 'dev' and task == 'nq' and graph_retriever_config . tagme : assert 'tagged_context' in data '''
                Reformat "tagged_context" if needed (c.f. the "context" case above)
                ''' if type ( data [ 'tagged_context' ] ) == list : tagged_context = { c [ 0 ] : c [ 1 ] for c in data [ 'tagged_context' ] } data [ 'tagged_context' ] = tagged_context '''
                Append valid paragraphs from "tagged_context" to "context"
                ''' for tagged_title in data [ 'tagged_context' ] : tagged_text = data [ 'tagged_context' ] [ tagged_title ] if tagged_title not in context and tagged_title is not None and tagged_title . strip ( ) != '' and tagged_text is not None and tagged_text . strip ( ) != '' : context [ tagged_title ] = tagged_text '''
            Clean "context" by removing invalid paragraphs
            ''' removed_keys = [ ] for title in context : if title is None or title . strip ( ) == '' or context [ title ] is None or context [ title ] . strip ( ) == '' : removed_keys . append ( title ) for key in removed_keys : context . pop ( key ) if task in [ 'squad' , 'nq' ] and set_type == 'train' : new_context = { } orig_title = list ( context . keys ( ) ) [ 0 ] . split ( '_' ) [ 0 ] orig_titles = [ ] other_titles = [ ] for title in context : title_ = title . split ( '_' ) [ 0 ] if title_ == orig_title : orig_titles . append ( title ) else : other_titles . append ( title ) orig_index = 0 other_index = 0 while orig_index < len ( orig_titles ) or other_index < len ( other_titles ) : if orig_index < len ( orig_titles ) : new_context [ orig_titles [ orig_index ] ] = context [ orig_titles [ orig_index ] ] orig_index += 1 if other_index < len ( other_titles ) : new_context [ other_titles [ other_index ] ] = context [ other_titles [ other_index ] ] other_index += 1 context = new_context '''
            Convert link format
            ''' new_all_linked_paras_dic = { } # {context title: {linked title: paragraph}} all_linked_paras_dic # {linked_title: paragraph} or mixed all_linked_para_title_dic = data [ 'all_linked_para_title_dic' ] # {context_title: [linked_title_1, linked_title_2, ...]} removed_keys = [ ] tmp = { } for key in all_linked_paras_dic : if type ( all_linked_paras_dic [ key ] ) == dict : removed_keys . append ( key ) for linked_title in all_linked_paras_dic [ key ] : if linked_title not in all_linked_paras_dic : tmp [ linked_title ] = all_linked_paras_dic [ key ] [ linked_title ] if key in all_linked_para_title_dic : all_linked_para_title_dic [ key ] . append ( linked_title ) else : all_linked_para_title_dic [ key ] = [ linked_title ] for key in removed_keys : all_linked_paras_dic . pop ( key ) for key in tmp : if key not in all_linked_paras_dic : all_linked_paras_dic [ key ] = tmp [ key ] for context_title in context : if context_title not in all_linked_para_title_dic : continue new_entry = { } for linked_title in all_linked_para_title_dic [ context_title ] : if linked_title not in all_linked_paras_dic : continue new_entry [ linked_title ] = all_linked_paras_dic [ linked_title ] if len ( new_entry ) > 0 : new_all_linked_paras_dic [ context_title ] = new_entry all_linked_paras_dic = new_all_linked_paras_dic if set_type == 'dev' : '''
                Clean "all_linked_paras_dic" by removing invalid paragraphs
                ''' for c in all_linked_paras_dic : removed_keys = [ ] links = all_linked_paras_dic [ c ] for title in links : if title is None or title . strip ( ) == '' or links [ title ] is None or type ( links [ title ] ) != str or links [ title ] . strip ( ) == '' : removed_keys . append ( title ) for key in removed_keys : links . pop ( key ) all_paras = { } for title in context : all_paras [ title ] = context [ title ] if not graph_retriever_config . open : continue if title not in all_linked_paras_dic : continue for title_ in all_linked_paras_dic [ title ] : if title_ not in all_paras : all_paras [ title_ ] = all_linked_paras_dic [ title ] [ title_ ] else : all_paras = None if set_type == 'dev' and graph_retriever_config . expand_links : expand_links ( context , all_linked_paras_dic , all_paras ) if set_type == 'dev' and graph_retriever_config . no_links : all_linked_paras_dic = { } graph_retriever_config . max_context_size = max ( graph_retriever_config . max_context_size , len ( context ) ) '''
            Ensure that all the gold paragraphs are included in "context"
            ''' if set_type == 'train' : for t in short_gold + redundant_gold : assert t in context examples . append ( InputExample ( guid = guid , q = question , c = context , para_dic = all_linked_paras_dic , s_g = short_gold , r_g = redundant_gold , all_r_g = all_redundant_gold , all_paras = all_paras ) ) if set_type == 'dev' : examples = sorted ( examples , key = lambda x : len ( x . all_paras ) ) logger . info ( 'Done!' ) return examples