(examples, max_seq_length, max_sent_num, max_sf_num, tokenizer, train=False)
return features
Loads a data file into a list of `InputBatch`s.
Loads a data file into a list of `InputBatch`s.
Loads a data file into a list of InputBatch s .
def convert_examples_to_features(examples, max_seq_length, max_sent_num, max_sf_num, tokenizer, train=False):
    """Loads a data file into a list of `InputBatch`s."""

    DUMMY = [0] * max_seq_length
    DUMMY_ = [0.0] * max_sent_num
    features = []
    logger.info('#### Constructing features... ####')
    for (ex_index, example) in enumerate(tqdm(examples, desc='Example')):

        tokens_q = tokenizer.tokenize(
            'Q: {} A: {}'.format(example.question, example.answer))
        tokens_q = ['[CLS]'] + tokens_q + ['[SEP]']

        input_ids = []
        input_masks = []
        segment_ids = []

        for title in example.titles:
            sents = example.context[title]
            for (i, s) in enumerate(sents):

                if len(input_ids) == max_sent_num:
                    break

                tokens_s = tokenizer.tokenize(
                    s)[:max_seq_length-len(tokens_q)-1]
                tokens_s = tokens_s + ['[SEP]']

                padding = [0] * (max_seq_length -
                                 len(tokens_s) - len(tokens_q))

                input_ids_ = tokenizer.convert_tokens_to_ids(
                    tokens_q + tokens_s)
                input_masks_ = [1] * len(input_ids_)
                segment_ids_ = [0] * len(tokens_q) + [1] * len(tokens_s)

                input_ids_ += padding
                input_ids.append(input_ids_)

                input_masks_ += padding
                input_masks.append(input_masks_)

                segment_ids_ += padding
                segment_ids.append(segment_ids_)

                assert len(input_ids_) == max_seq_length
                assert len(input_masks_) == max_seq_length
                assert len(segment_ids_) == max_seq_length

        target_ids = []
        target_offset = 0

        for title in example.titles:
            sfs = example.supporting_facts[title]
            for i in sfs:
                if i < len(example.context[title]) and i+target_offset < len(input_ids):
                    target_ids.append(i+target_offset)
                else:
                    logger.warning('')
                    logger.warning('Invalid annotation: {}'.format(sfs))
                    logger.warning('Invalid annotation: {}'.format(
                        example.context[title]))

            target_offset += len(example.context[title])

        assert len(input_ids) <= max_sent_num
        assert len(target_ids) <= max_sf_num

        num_sents = len(input_ids)
        num_sfs = len(target_ids)

        output_masks = [([1.0] * len(input_ids) + [0.0] * (max_sent_num -
                                                           len(input_ids) + 1)) for _ in range(max_sent_num + 2)]

        if train:

            for i in range(len(target_ids)):
                for j in range(len(target_ids)):
                    if i == j:
                        continue

                    output_masks[i][target_ids[j]] = 0.0

            for i in range(len(output_masks)):
                if i >= num_sfs+1:
                    for j in range(len(output_masks[i])):
                        output_masks[i][j] = 0.0

        else:
            for i in range(len(input_ids)):
                output_masks[i+1][i] = 0.0

        target_ids += [0] * (max_sf_num - len(target_ids))

        padding = [DUMMY] * (max_sent_num - len(input_ids))
        input_ids += padding
        input_masks += padding
        segment_ids += padding

        features.append(
            InputFeatures(input_ids=input_ids,
                          input_masks=input_masks,
                          segment_ids=segment_ids,
                          target_ids=target_ids,
                          output_masks=output_masks,
                          num_sents=num_sents,
                          num_sfs=num_sfs,
                          ex_index=ex_index))

    logger.info('Done!')

    return features
def convert_examples_to_features ( examples , max_seq_length , max_sent_num , max_sf_num , tokenizer , train = False ) : DUMMY = [ 0 ] * max_seq_length DUMMY_ = [ 0.0 ] * max_sent_num features = [ ] logger . info ( '#### Constructing features... ####' ) for ( ex_index , example ) in enumerate ( tqdm ( examples , desc = 'Example' ) ) : tokens_q = tokenizer . tokenize ( 'Q: {} A: {}' . format ( example . question , example . answer ) ) tokens_q = [ '[CLS]' ] + tokens_q + [ '[SEP]' ] input_ids = [ ] input_masks = [ ] segment_ids = [ ] for title in example . titles : sents = example . context [ title ] for ( i , s ) in enumerate ( sents ) : if len ( input_ids ) == max_sent_num : break tokens_s = tokenizer . tokenize ( s ) [ : max_seq_length - len ( tokens_q ) - 1 ] tokens_s = tokens_s + [ '[SEP]' ] padding = [ 0 ] * ( max_seq_length - len ( tokens_s ) - len ( tokens_q ) ) input_ids_ = tokenizer . convert_tokens_to_ids ( tokens_q + tokens_s ) input_masks_ = [ 1 ] * len ( input_ids_ ) segment_ids_ = [ 0 ] * len ( tokens_q ) + [ 1 ] * len ( tokens_s ) input_ids_ += padding input_ids . append ( input_ids_ ) input_masks_ += padding input_masks . append ( input_masks_ ) segment_ids_ += padding segment_ids . append ( segment_ids_ ) assert len ( input_ids_ ) == max_seq_length assert len ( input_masks_ ) == max_seq_length assert len ( segment_ids_ ) == max_seq_length target_ids = [ ] target_offset = 0 for title in example . titles : sfs = example . supporting_facts [ title ] for i in sfs : if i < len ( example . context [ title ] ) and i + target_offset < len ( input_ids ) : target_ids . append ( i + target_offset ) else : logger . warning ( '' ) logger . warning ( 'Invalid annotation: {}' . format ( sfs ) ) logger . warning ( 'Invalid annotation: {}' . format ( example . context [ title ] ) ) target_offset += len ( example . context [ title ] ) assert len ( input_ids ) <= max_sent_num assert len ( target_ids ) <= max_sf_num num_sents = len ( input_ids ) num_sfs = len ( target_ids ) output_masks = [ ( [ 1.0 ] * len ( input_ids ) + [ 0.0 ] * ( max_sent_num - len ( input_ids ) + 1 ) ) for _ in range ( max_sent_num + 2 ) ] if train : for i in range ( len ( target_ids ) ) : for j in range ( len ( target_ids ) ) : if i == j : continue output_masks [ i ] [ target_ids [ j ] ] = 0.0 for i in range ( len ( output_masks ) ) : if i >= num_sfs + 1 : for j in range ( len ( output_masks [ i ] ) ) : output_masks [ i ] [ j ] = 0.0 else : for i in range ( len ( input_ids ) ) : output_masks [ i + 1 ] [ i ] = 0.0 target_ids += [ 0 ] * ( max_sf_num - len ( target_ids ) ) padding = [ DUMMY ] * ( max_sent_num - len ( input_ids ) ) input_ids += padding input_masks += padding segment_ids += padding features . append ( InputFeatures ( input_ids = input_ids , input_masks = input_masks , segment_ids = segment_ids , target_ids = target_ids , output_masks = output_masks , num_sents = num_sents , num_sfs = num_sfs , ex_index = ex_index ) ) logger . info ( 'Done!' ) return features