(examples, tokenizer, max_seq_length=128, max_num_choices=10)
return features
将所有候选答案放置在片段开头
将所有候选答案放置在片段开头
将所有候选答案放置在片段开头
def convert_examples_to_features(examples, tokenizer, max_seq_length=128, max_num_choices=10):
    '''
    将所有候选答案放置在片段开头
    '''

    def _loop(example, unique_id, label):
        '''
        :param example:
        :param unique_id:
        :return:
            input_ids = (C, seq_len)
            token_type_ids = (C, seq_len) = segment_id
            input_mask = (C, seq_len)
            labels = int
            choices_mask = (C)
        '''
        input_ids = []
        input_masks = []
        segment_ids = []
        choice_masks = [1] * len(example.options)

        tag = example.tag
        all_doc_tokens = []
        for (i, token) in enumerate(example.doc_tokens):
            if '#idiom' in token:
                sub_tokens = [str(token)]
            else:
                sub_tokens = tokenizer.tokenize(token)
            for sub_token in sub_tokens:
                all_doc_tokens.append(sub_token)

        pos = all_doc_tokens.index(tag)
        num_tokens = max_tokens_for_doc - 5  # [unused1]和segA的成语
        tmp_l, tmp_r = add_tokens_for_around(all_doc_tokens, pos, num_tokens)
        num_l = len(tmp_l)
        num_r = len(tmp_r)

        tokens_l = []
        for token in tmp_l:
            if '#idiom' in token and token != tag:
                tokens_l.extend(['[MASK]'] * 4)
            else:
                tokens_l.append(token)
        tokens_l = tokens_l[-num_l:]
        del tmp_l

        tokens_r = []
        for token in tmp_r:
            if '#idiom' in token and token != tag:
                tokens_r.extend(['[MASK]'] * 4)
            else:
                tokens_r.append(token)
        tokens_r = tokens_r[: num_r]
        del tmp_r

        for i, elem in enumerate(example.options):
            option = tokenizer.tokenize(elem)
            tokens = ['[CLS]'] + option + ['[SEP]'] + tokens_l + ['[unused1]'] + tokens_r + ['[SEP]']

            input_id = tokenizer.convert_tokens_to_ids(tokens)
            input_mask = [1] * len(input_id)
            segment_id = [0] * len(input_id)

            while len(input_id) < max_seq_length:
                input_id.append(0)
                input_mask.append(0)
                segment_id.append(0)
            assert len(input_id) == max_seq_length
            assert len(input_mask) == max_seq_length
            assert len(segment_id) == max_seq_length

            input_ids.append(input_id)
            input_masks.append(input_mask)
            segment_ids.append(segment_id)

        if unique_id < 5:
            print("*** Example ***")
            print("unique_id: {}".format(unique_id))
            print("context_id: {}".format(tag))
            print("label: {}".format(label))
            print("tag_index: {}".format(pos))
            print("tokens: {}".format("".join(tokens)))
            print("choice_masks: {}".format(choice_masks))
        while len(input_ids) < max_num_choices:
            input_ids.append([0] * max_seq_length)
            input_masks.append([0] * max_seq_length)
            segment_ids.append([0] * max_seq_length)
            choice_masks.append(0)
        assert len(input_ids) == max_num_choices
        assert len(input_masks) == max_num_choices
        assert len(segment_ids) == max_num_choices
        assert len(choice_masks) == max_num_choices

        features.append(
            InputFeatures(
                unique_id=unique_id,
                example_id=example.example_id,
                tag=tag,
                tokens=tokens,
                input_ids=input_ids,
                input_masks=input_masks,
                segment_ids=segment_ids,
                choice_masks=choice_masks,
                label=label))

    max_tokens_for_doc = max_seq_length - 3  # [CLS] choice [SEP] document [SEP]
    features = []
    unique_id = 0

    for (example_index, example) in enumerate(tqdm(examples)):

        label = example.answer_index
        if label != None:
            _loop(example, unique_id, label)
        else:
            _loop(example, unique_id, None)
        unique_id += 1

        if unique_id % 12000 == 0:
            print("unique_id: %s" % (unique_id))
    print("unique_id: %s" % (unique_id))
    return features
def convert_examples_to_features ( examples , tokenizer , max_seq_length = 128 , max_num_choices = 10 ) : def _loop ( example , unique_id , label ) : '''
        :param example:
        :param unique_id:
        :return:
            input_ids = (C, seq_len)
            token_type_ids = (C, seq_len) = segment_id
            input_mask = (C, seq_len)
            labels = int
            choices_mask = (C)
        ''' input_ids = [ ] input_masks = [ ] segment_ids = [ ] choice_masks = [ 1 ] * len ( example . options ) tag = example . tag all_doc_tokens = [ ] for ( i , token ) in enumerate ( example . doc_tokens ) : if '#idiom' in token : sub_tokens = [ str ( token ) ] else : sub_tokens = tokenizer . tokenize ( token ) for sub_token in sub_tokens : all_doc_tokens . append ( sub_token ) pos = all_doc_tokens . index ( tag ) num_tokens = max_tokens_for_doc - 5 # [unused1]和segA的成语 tmp_l , tmp_r = add_tokens_for_around ( all_doc_tokens , pos , num_tokens ) num_l = len ( tmp_l ) num_r = len ( tmp_r ) tokens_l = [ ] for token in tmp_l : if '#idiom' in token and token != tag : tokens_l . extend ( [ '[MASK]' ] * 4 ) else : tokens_l . append ( token ) tokens_l = tokens_l [ - num_l : ] del tmp_l tokens_r = [ ] for token in tmp_r : if '#idiom' in token and token != tag : tokens_r . extend ( [ '[MASK]' ] * 4 ) else : tokens_r . append ( token ) tokens_r = tokens_r [ : num_r ] del tmp_r for i , elem in enumerate ( example . options ) : option = tokenizer . tokenize ( elem ) tokens = [ '[CLS]' ] + option + [ '[SEP]' ] + tokens_l + [ '[unused1]' ] + tokens_r + [ '[SEP]' ] input_id = tokenizer . convert_tokens_to_ids ( tokens ) input_mask = [ 1 ] * len ( input_id ) segment_id = [ 0 ] * len ( input_id ) while len ( input_id ) < max_seq_length : input_id . append ( 0 ) input_mask . append ( 0 ) segment_id . append ( 0 ) assert len ( input_id ) == max_seq_length assert len ( input_mask ) == max_seq_length assert len ( segment_id ) == max_seq_length input_ids . append ( input_id ) input_masks . append ( input_mask ) segment_ids . append ( segment_id ) if unique_id < 5 : print ( "*** Example ***" ) print ( "unique_id: {}" . format ( unique_id ) ) print ( "context_id: {}" . format ( tag ) ) print ( "label: {}" . format ( label ) ) print ( "tag_index: {}" . format ( pos ) ) print ( "tokens: {}" . format ( "" . join ( tokens ) ) ) print ( "choice_masks: {}" . format ( choice_masks ) ) while len ( input_ids ) < max_num_choices : input_ids . append ( [ 0 ] * max_seq_length ) input_masks . append ( [ 0 ] * max_seq_length ) segment_ids . append ( [ 0 ] * max_seq_length ) choice_masks . append ( 0 ) assert len ( input_ids ) == max_num_choices assert len ( input_masks ) == max_num_choices assert len ( segment_ids ) == max_num_choices assert len ( choice_masks ) == max_num_choices features . append ( InputFeatures ( unique_id = unique_id , example_id = example . example_id , tag = tag , tokens = tokens , input_ids = input_ids , input_masks = input_masks , segment_ids = segment_ids , choice_masks = choice_masks , label = label ) ) max_tokens_for_doc = max_seq_length - 3 # [CLS] choice [SEP] document [SEP] features = [ ] unique_id = 0 for ( example_index , example ) in enumerate ( tqdm ( examples ) ) : label = example . answer_index if label != None : _loop ( example , unique_id , label ) else : _loop ( example , unique_id , None ) unique_id += 1 if unique_id % 12000 == 0 : print ( "unique_id: %s" % ( unique_id ) ) print ( "unique_id: %s" % ( unique_id ) ) return features