(examples, max_seq_length, max_para_num, graph_retriever_config, tokenizer)
return features
Loads a data file into a list of `InputBatch`s.
Loads a data file into a list of `InputBatch`s.
Loads a data file into a list of InputBatch s .
def convert_examples_to_features(examples, max_seq_length, max_para_num, graph_retriever_config, tokenizer):
    """Loads a data file into a list of `InputBatch`s."""

    max_para_num = graph_retriever_config.max_context_size
    graph_retriever_config.max_para_num = max(graph_retriever_config.max_para_num, max_para_num)
    
    max_steps = graph_retriever_config.max_select_num
    
    DUMMY = [0] * max_seq_length
    features = []

    for (ex_index, example) in enumerate(examples):
        tokens_q = tokenize_question(example.question, tokenizer)

        title2index = {}
        input_ids = []
        input_masks = []
        segment_ids = []

        titles_list = list(example.context.keys())
        for p in titles_list:

            if len(input_ids) == max_para_num:
                break

            if p in title2index:
                continue

            title2index[p] = len(title2index)
            example.title_order.append(p)
            p = example.context[p]

            input_ids_, input_masks_, segment_ids_ = tokenize_paragraph(p, tokens_q, max_seq_length, tokenizer)
            input_ids.append(input_ids_)
            input_masks.append(input_masks_)
            segment_ids.append(segment_ids_)

        num_paragraphs_no_links = len(input_ids)
            
        assert len(input_ids) <= max_para_num
        
        num_paragraphs = len(input_ids)
        
        output_masks = [([1.0] * len(input_ids) + [0.0] * (max_para_num - len(input_ids) + 1)) for _ in range(max_para_num + 2)]

        assert len(example.context) == num_paragraphs_no_links
        for i in range(len(output_masks[0])):
            if i >= num_paragraphs_no_links:
                output_masks[0][i] = 0.0
        
        for i in range(len(input_ids)):
            output_masks[i+1][i] = 0.0            
                    
        padding = [DUMMY] * (max_para_num - len(input_ids))
        input_ids += padding
        input_masks += padding
        segment_ids += padding

        features.append(
                InputFeatures(input_ids=input_ids,
                              input_masks=input_masks,
                              segment_ids=segment_ids,
                              output_masks = output_masks,
                              num_paragraphs = num_paragraphs,
                              num_steps = -1,
                              ex_index = ex_index))

    return features
def convert_examples_to_features ( examples , max_seq_length , max_para_num , graph_retriever_config , tokenizer ) : max_para_num = graph_retriever_config . max_context_size graph_retriever_config . max_para_num = max ( graph_retriever_config . max_para_num , max_para_num ) max_steps = graph_retriever_config . max_select_num DUMMY = [ 0 ] * max_seq_length features = [ ] for ( ex_index , example ) in enumerate ( examples ) : tokens_q = tokenize_question ( example . question , tokenizer ) title2index = { } input_ids = [ ] input_masks = [ ] segment_ids = [ ] titles_list = list ( example . context . keys ( ) ) for p in titles_list : if len ( input_ids ) == max_para_num : break if p in title2index : continue title2index [ p ] = len ( title2index ) example . title_order . append ( p ) p = example . context [ p ] input_ids_ , input_masks_ , segment_ids_ = tokenize_paragraph ( p , tokens_q , max_seq_length , tokenizer ) input_ids . append ( input_ids_ ) input_masks . append ( input_masks_ ) segment_ids . append ( segment_ids_ ) num_paragraphs_no_links = len ( input_ids ) assert len ( input_ids ) <= max_para_num num_paragraphs = len ( input_ids ) output_masks = [ ( [ 1.0 ] * len ( input_ids ) + [ 0.0 ] * ( max_para_num - len ( input_ids ) + 1 ) ) for _ in range ( max_para_num + 2 ) ] assert len ( example . context ) == num_paragraphs_no_links for i in range ( len ( output_masks [ 0 ] ) ) : if i >= num_paragraphs_no_links : output_masks [ 0 ] [ i ] = 0.0 for i in range ( len ( input_ids ) ) : output_masks [ i + 1 ] [ i ] = 0.0 padding = [ DUMMY ] * ( max_para_num - len ( input_ids ) ) input_ids += padding input_masks += padding segment_ids += padding features . append ( InputFeatures ( input_ids = input_ids , input_masks = input_masks , segment_ids = segment_ids , output_masks = output_masks , num_paragraphs = num_paragraphs , num_steps = - 1 , ex_index = ex_index ) ) return features