(examples, max_seq_length, max_para_num, graph_retriever_config, tokenizer, train = False)
return features
Loads a data file into a list of `InputBatch`s.
Loads a data file into a list of `InputBatch`s.
Loads a data file into a list of InputBatch s .
def convert_examples_to_features(examples, max_seq_length, max_para_num, graph_retriever_config, tokenizer, train = False):
    """Loads a data file into a list of `InputBatch`s."""

    if not train and graph_retriever_config.db_save_path is not None:
        max_para_num = graph_retriever_config.max_context_size
        graph_retriever_config.max_para_num = max(graph_retriever_config.max_para_num, max_para_num)
    
    max_steps = graph_retriever_config.max_select_num
    
    DUMMY = [0] * max_seq_length
    features = []

    logger.info('#### Converting examples to features... ####')
    for (ex_index, example) in enumerate(tqdm(examples, desc='Example')):
        tokens_q = tokenize_question(example.question, tokenizer)
        
        ##############
        # Short gold #
        ##############
        title2index = {}
        input_ids = []
        input_masks = []
        segment_ids = []

        # Append gold and non-gold paragraphs from context
        if train and graph_retriever_config.use_redundant and len(example.redundant_gold) > 0:
            if graph_retriever_config.use_multiple_redundant:
                titles_list = example.short_gold + [redundant[0] for redundant in example.all_redundant_gold] + list(example.context.keys())
            else:
                titles_list = example.short_gold + [example.redundant_gold[0]] + list(example.context.keys())
        else:
            titles_list = example.short_gold + list(example.context.keys())
        for p in titles_list:

            if len(input_ids) == max_para_num:
                break

            # Avoid appending gold paragraphs as negative
            if p in title2index:
                continue

            # fullwiki eval
            # Gold paragraphs are not always in context
            if not train and graph_retriever_config.open and p not in example.context:
                continue
            
            title2index[p] = len(title2index)
            example.title_order.append(p)
            p = example.context[p]

            input_ids_, input_masks_, segment_ids_ = tokenize_paragraph(p, tokens_q, max_seq_length, tokenizer)
            input_ids.append(input_ids_)
            input_masks.append(input_masks_)
            segment_ids.append(segment_ids_)

        # Open-domain setting
        if graph_retriever_config.open:
            num_paragraphs_no_links = len(input_ids)
            
            for p_ in example.context:

                if not train and graph_retriever_config.db_save_path is not None:
                    break

                if len(input_ids) == max_para_num:
                    break

                if p_ not in example.all_linked_paras_dic:
                    continue
                
                for l in example.all_linked_paras_dic[p_]:

                    if len(input_ids) == max_para_num:
                        break
                    
                    if l in title2index:
                        continue

                    title2index[l] = len(title2index)
                    example.title_order.append(l)
                    p = example.all_linked_paras_dic[p_][l]

                    input_ids_, input_masks_, segment_ids_ = tokenize_paragraph(p, tokens_q, max_seq_length, tokenizer)
                    input_ids.append(input_ids_)
                    input_masks.append(input_masks_)
                    segment_ids.append(segment_ids_)
            
        assert len(input_ids) <= max_para_num
        
        num_paragraphs = len(input_ids)
        num_steps = len(example.short_gold)+1 # 1 for EOE

        if train:
            assert num_steps <= max_steps
        
        output_masks = [([1.0] * len(input_ids) + [0.0] * (max_para_num - len(input_ids) + 1)) for _ in range(max_para_num + 2)]

        if (not train) and graph_retriever_config.open:
            assert len(example.context) == num_paragraphs_no_links
            for i in range(len(output_masks[0])):
                if i >= num_paragraphs_no_links:
                    output_masks[0][i] = 0.0
        
        for i in range(len(input_ids)):
            output_masks[i+1][i] = 0.0            

        if train:
            size = num_steps-1

            for i in range(size):
                for j in range(size):
                    if i != j:
                        output_masks[i][j] = 0.0

            for i in range(size):
                output_masks[size][i] = 0.0
                        
            for i in range(max_steps):
                if i > size:
                    for j in range(len(output_masks[i])):
                        output_masks[i][j] = 0.0

            # Use REDUNDANT setting
            # Avoid treating the redundant paragraph as a negative example at the first step
            if graph_retriever_config.use_redundant and len(example.redundant_gold) > 0:
                if graph_retriever_config.use_multiple_redundant:
                    for redundant in example.all_redundant_gold:
                        output_masks[0][title2index[redundant[0]]] = 0.0
                else:
                    output_masks[0][title2index[example.redundant_gold[0]]] = 0.0
                    
        padding = [DUMMY] * (max_para_num - len(input_ids))
        input_ids += padding
        input_masks += padding
        segment_ids += padding

        features.append(
                InputFeatures(input_ids=input_ids,
                              input_masks=input_masks,
                              segment_ids=segment_ids,
                              output_masks = output_masks,
                              num_paragraphs = num_paragraphs,
                              num_steps = num_steps,
                              ex_index = ex_index))

        if not train or not graph_retriever_config.use_redundant or len(example.redundant_gold) == 0:
            continue


        ##################
        # Redundant gold #
        ##################
        for redundant_gold in example.all_redundant_gold:
            hist = set()
            input_ids_r = []
            input_masks_r = []
            segment_ids_r = []

            # Append gold and non-gold paragraphs from context
            for p in redundant_gold + list(example.context.keys()):

                if len(input_ids_r) == max_para_num:
                    break

                #assert p in title2index
                if p not in title2index:
                    assert p not in redundant_gold
                    continue

                if p in hist:
                    continue
                hist.add(p)

                index = title2index[p]
                input_ids_r.append(input_ids[index])
                input_masks_r.append(input_masks[index])
                segment_ids_r.append(segment_ids[index])

            # Open-domain setting (mainly for HotpotQA fullwiki)
            if graph_retriever_config.open:

                for p in title2index:

                    if len(input_ids_r) == max_para_num:
                        break

                    if p in hist:
                        continue
                    hist.add(p)

                    index = title2index[p]
                    input_ids_r.append(input_ids[index])
                    input_masks_r.append(input_masks[index])
                    segment_ids_r.append(segment_ids[index])

            assert len(input_ids_r) <= max_para_num

            num_paragraphs_r = len(input_ids_r)
            num_steps_r = len(redundant_gold)+1

            assert num_steps_r <= max_steps

            output_masks_r = [([1.0] * len(input_ids_r) + [0.0] * (max_para_num - len(input_ids_r) + 1)) for _ in range(max_para_num + 2)]

            size = num_steps_r-1

            for i in range(size):
                for j in range(size):
                    if i != j:
                        output_masks_r[i][j] = 0.0

                if i > 0:
                    output_masks_r[i][0] = 1.0

            for i in range(size): #size-1
                output_masks_r[size][i] = 0.0

            for i in range(max_steps):
                if i > size:
                    for j in range(len(output_masks_r[i])):
                        output_masks_r[i][j] = 0.0

            padding = [DUMMY] * (max_para_num - len(input_ids_r))
            input_ids_r += padding
            input_masks_r += padding
            segment_ids_r += padding

            features.append(
                    InputFeatures(input_ids=input_ids_r,
                                  input_masks=input_masks_r,
                                  segment_ids=segment_ids_r,
                                  output_masks = output_masks_r,
                                  num_paragraphs = num_paragraphs_r,
                                  num_steps = num_steps_r,
                                  ex_index = None))

            if not graph_retriever_config.use_multiple_redundant:
                break

    logger.info('Done!')
    return features
def convert_examples_to_features ( examples , max_seq_length , max_para_num , graph_retriever_config , tokenizer , train = False ) : if not train and graph_retriever_config . db_save_path is not None : max_para_num = graph_retriever_config . max_context_size graph_retriever_config . max_para_num = max ( graph_retriever_config . max_para_num , max_para_num ) max_steps = graph_retriever_config . max_select_num DUMMY = [ 0 ] * max_seq_length features = [ ] logger . info ( '#### Converting examples to features... ####' ) for ( ex_index , example ) in enumerate ( tqdm ( examples , desc = 'Example' ) ) : tokens_q = tokenize_question ( example . question , tokenizer ) ############## # Short gold # ############## title2index = { } input_ids = [ ] input_masks = [ ] segment_ids = [ ] # Append gold and non-gold paragraphs from context if train and graph_retriever_config . use_redundant and len ( example . redundant_gold ) > 0 : if graph_retriever_config . use_multiple_redundant : titles_list = example . short_gold + [ redundant [ 0 ] for redundant in example . all_redundant_gold ] + list ( example . context . keys ( ) ) else : titles_list = example . short_gold + [ example . redundant_gold [ 0 ] ] + list ( example . context . keys ( ) ) else : titles_list = example . short_gold + list ( example . context . keys ( ) ) for p in titles_list : if len ( input_ids ) == max_para_num : break # Avoid appending gold paragraphs as negative if p in title2index : continue # fullwiki eval # Gold paragraphs are not always in context if not train and graph_retriever_config . open and p not in example . context : continue title2index [ p ] = len ( title2index ) example . title_order . append ( p ) p = example . context [ p ] input_ids_ , input_masks_ , segment_ids_ = tokenize_paragraph ( p , tokens_q , max_seq_length , tokenizer ) input_ids . append ( input_ids_ ) input_masks . append ( input_masks_ ) segment_ids . append ( segment_ids_ ) # Open-domain setting if graph_retriever_config . open : num_paragraphs_no_links = len ( input_ids ) for p_ in example . context : if not train and graph_retriever_config . db_save_path is not None : break if len ( input_ids ) == max_para_num : break if p_ not in example . all_linked_paras_dic : continue for l in example . all_linked_paras_dic [ p_ ] : if len ( input_ids ) == max_para_num : break if l in title2index : continue title2index [ l ] = len ( title2index ) example . title_order . append ( l ) p = example . all_linked_paras_dic [ p_ ] [ l ] input_ids_ , input_masks_ , segment_ids_ = tokenize_paragraph ( p , tokens_q , max_seq_length , tokenizer ) input_ids . append ( input_ids_ ) input_masks . append ( input_masks_ ) segment_ids . append ( segment_ids_ ) assert len ( input_ids ) <= max_para_num num_paragraphs = len ( input_ids ) num_steps = len ( example . short_gold ) + 1 # 1 for EOE if train : assert num_steps <= max_steps output_masks = [ ( [ 1.0 ] * len ( input_ids ) + [ 0.0 ] * ( max_para_num - len ( input_ids ) + 1 ) ) for _ in range ( max_para_num + 2 ) ] if ( not train ) and graph_retriever_config . open : assert len ( example . context ) == num_paragraphs_no_links for i in range ( len ( output_masks [ 0 ] ) ) : if i >= num_paragraphs_no_links : output_masks [ 0 ] [ i ] = 0.0 for i in range ( len ( input_ids ) ) : output_masks [ i + 1 ] [ i ] = 0.0 if train : size = num_steps - 1 for i in range ( size ) : for j in range ( size ) : if i != j : output_masks [ i ] [ j ] = 0.0 for i in range ( size ) : output_masks [ size ] [ i ] = 0.0 for i in range ( max_steps ) : if i > size : for j in range ( len ( output_masks [ i ] ) ) : output_masks [ i ] [ j ] = 0.0 # Use REDUNDANT setting # Avoid treating the redundant paragraph as a negative example at the first step if graph_retriever_config . use_redundant and len ( example . redundant_gold ) > 0 : if graph_retriever_config . use_multiple_redundant : for redundant in example . all_redundant_gold : output_masks [ 0 ] [ title2index [ redundant [ 0 ] ] ] = 0.0 else : output_masks [ 0 ] [ title2index [ example . redundant_gold [ 0 ] ] ] = 0.0 padding = [ DUMMY ] * ( max_para_num - len ( input_ids ) ) input_ids += padding input_masks += padding segment_ids += padding features . append ( InputFeatures ( input_ids = input_ids , input_masks = input_masks , segment_ids = segment_ids , output_masks = output_masks , num_paragraphs = num_paragraphs , num_steps = num_steps , ex_index = ex_index ) ) if not train or not graph_retriever_config . use_redundant or len ( example . redundant_gold ) == 0 : continue ################## # Redundant gold # ################## for redundant_gold in example . all_redundant_gold : hist = set ( ) input_ids_r = [ ] input_masks_r = [ ] segment_ids_r = [ ] # Append gold and non-gold paragraphs from context for p in redundant_gold + list ( example . context . keys ( ) ) : if len ( input_ids_r ) == max_para_num : break #assert p in title2index if p not in title2index : assert p not in redundant_gold continue if p in hist : continue hist . add ( p ) index = title2index [ p ] input_ids_r . append ( input_ids [ index ] ) input_masks_r . append ( input_masks [ index ] ) segment_ids_r . append ( segment_ids [ index ] ) # Open-domain setting (mainly for HotpotQA fullwiki) if graph_retriever_config . open : for p in title2index : if len ( input_ids_r ) == max_para_num : break if p in hist : continue hist . add ( p ) index = title2index [ p ] input_ids_r . append ( input_ids [ index ] ) input_masks_r . append ( input_masks [ index ] ) segment_ids_r . append ( segment_ids [ index ] ) assert len ( input_ids_r ) <= max_para_num num_paragraphs_r = len ( input_ids_r ) num_steps_r = len ( redundant_gold ) + 1 assert num_steps_r <= max_steps output_masks_r = [ ( [ 1.0 ] * len ( input_ids_r ) + [ 0.0 ] * ( max_para_num - len ( input_ids_r ) + 1 ) ) for _ in range ( max_para_num + 2 ) ] size = num_steps_r - 1 for i in range ( size ) : for j in range ( size ) : if i != j : output_masks_r [ i ] [ j ] = 0.0 if i > 0 : output_masks_r [ i ] [ 0 ] = 1.0 for i in range ( size ) : #size-1 output_masks_r [ size ] [ i ] = 0.0 for i in range ( max_steps ) : if i > size : for j in range ( len ( output_masks_r [ i ] ) ) : output_masks_r [ i ] [ j ] = 0.0 padding = [ DUMMY ] * ( max_para_num - len ( input_ids_r ) ) input_ids_r += padding input_masks_r += padding segment_ids_r += padding features . append ( InputFeatures ( input_ids = input_ids_r , input_masks = input_masks_r , segment_ids = segment_ids_r , output_masks = output_masks_r , num_paragraphs = num_paragraphs_r , num_steps = num_steps_r , ex_index = None ) ) if not graph_retriever_config . use_multiple_redundant : break logger . info ( 'Done!' ) return features