(input_data_file, input_label_file, is_training=True)
return examples
将原始数据处理为如下形式：
    part_passage遍历每个blak的周围位置
    :param input_data:
    :param is_training:
    :return:
将原始数据处理为如下形式：
    part_passage遍历每个blak的周围位置
    :param input_data:
    :param is_training:
    :return:
将原始数据处理为如下形式： part_passage遍历每个blak的周围位置 : param input_data : : param is_training : : return :
def read_chid_examples(input_data_file, input_label_file, is_training=True):
    '''
    将原始数据处理为如下形式：
    part_passage遍历每个blak的周围位置
    :param input_data:
    :param is_training:
    :return:
    '''

    if is_training:
        input_label = json.load(open(input_label_file))
    input_data = open(input_data_file)

    def _is_chinese_char(cp):
        if ((cp >= 0x4E00 and cp <= 0x9FFF) or  #
                (cp >= 0x3400 and cp <= 0x4DBF) or  #
                (cp >= 0x20000 and cp <= 0x2A6DF) or  #
                (cp >= 0x2A700 and cp <= 0x2B73F) or  #
                (cp >= 0x2B740 and cp <= 0x2B81F) or  #
                (cp >= 0x2B820 and cp <= 0x2CEAF) or
                (cp >= 0xF900 and cp <= 0xFAFF) or  #
                (cp >= 0x2F800 and cp <= 0x2FA1F)):  #
            return True

        return False

    def is_fuhao(c):
        if c == '。' or c == '，' or c == '！' or c == '？' or c == '；' or c == '、' or c == '：' or c == '（' or c == '）' \
                or c == '－' or c == '~' or c == '「' or c == '《' or c == '》' or c == ',' or c == '」' or c == '"' or c == '“' or c == '”' \
                or c == '$' or c == '『' or c == '』' or c == '—' or c == ';' or c == '。' or c == '(' or c == ')' or c == '-' or c == '～' or c == '。' \
                or c == '‘' or c == '’':
            return True
        return False

    def _tokenize_chinese_chars(text):
        """Adds whitespace around any CJK character."""
        output = []
        is_blank = False
        for index, char in enumerate(text):
            cp = ord(char)
            if is_blank:
                output.append(char)
                if context[index - 12:index + 1].startswith("#idiom"):
                    is_blank = False
                    output.append(SPIECE_UNDERLINE)
            else:
                if text[index:index + 6] == "#idiom":
                    is_blank = True
                    if len(output) > 0 and output[-1] != SPIECE_UNDERLINE:
                        output.append(SPIECE_UNDERLINE)
                    output.append(char)
                elif _is_chinese_char(cp) or is_fuhao(char):
                    if len(output) > 0 and output[-1] != SPIECE_UNDERLINE:
                        output.append(SPIECE_UNDERLINE)
                    output.append(char)
                    output.append(SPIECE_UNDERLINE)
                else:
                    output.append(char)
        return "".join(output)

    def is_whitespace(c):
        if c == " " or c == "\t" or c == "\r" or c == "\n" or ord(c) == 0x202F or c == SPIECE_UNDERLINE:
            return True
        return False

    examples = []
    example_id = 0
    for data in tqdm(input_data):

        data = eval(data)
        options = data['candidates']

        for context in data['content']:

            context = context.replace("“", "\"").replace("”", "\"").replace("——", "--"). \
                replace("—", "-").replace("―", "-").replace("…", "...").replace("‘", "\'").replace("’", "\'")
            context = _tokenize_chinese_chars(context)

            paragraph_text = context.strip()
            doc_tokens = []
            prev_is_whitespace = True
            for c in paragraph_text:
                if is_whitespace(c):
                    prev_is_whitespace = True
                else:
                    if prev_is_whitespace:
                        doc_tokens.append(c)
                    else:
                        doc_tokens[-1] += c
                    prev_is_whitespace = False

            tags = [blank for blank in doc_tokens if '#idiom' in blank]

            if is_training:
                for tag_index, tag in enumerate(tags):
                    answer_index = input_label[tag]
                    example = ChidExample(
                        example_id=example_id,
                        tag=tag,
                        doc_tokens=doc_tokens,
                        options=options,
                        answer_index=answer_index)
                    examples.append(example)
            else:
                for tag_index, tag in enumerate(tags):
                    example = ChidExample(
                        example_id=example_id,
                        tag=tag,
                        doc_tokens=doc_tokens,
                        options=options)
                    examples.append(example)
        else:
            example_id += 1
    else:
        print('原始样本个数：{}'.format(example_id))

    print('实际生成总样例数：{}'.format(len(examples)))
    return examples
def read_chid_examples ( input_data_file , input_label_file , is_training = True ) : if is_training : input_label = json . load ( open ( input_label_file ) ) input_data = open ( input_data_file ) def _is_chinese_char ( cp ) : if ( ( cp >= 0x4E00 and cp <= 0x9FFF ) or # ( cp >= 0x3400 and cp <= 0x4DBF ) or # ( cp >= 0x20000 and cp <= 0x2A6DF ) or # ( cp >= 0x2A700 and cp <= 0x2B73F ) or # ( cp >= 0x2B740 and cp <= 0x2B81F ) or # ( cp >= 0x2B820 and cp <= 0x2CEAF ) or ( cp >= 0xF900 and cp <= 0xFAFF ) or # ( cp >= 0x2F800 and cp <= 0x2FA1F ) ) : # return True return False def is_fuhao ( c ) : if c == '。' o  c =  ' ' or   = ！' or c  =  ？  o  c == '； r   == ' '  r c  = '：' or c == '（' o  c =  ' ' \ or c == '－' o  c =  ' ' o  c =  ' ' or   = 《' or c  =  》  o  c == ', r   == '」 r   == ' '  r c  = ' '  r c  = '”' \ or c == '$' or c == '『' o  c =  ' ' or   = —' or c  =  ;  o  c  =  。  o  c == '( r   == ') r   == '- r   == '～ r   == ' '  or c == '‘' o  c =  ' ': return True return False def _tokenize_chinese_chars ( text ) : """Adds whitespace around any CJK character.""" output = [ ] is_blank = False for index , char in enumerate ( text ) : cp = ord ( char ) if is_blank : output . append ( char ) if context [ index - 12 : index + 1 ] . startswith ( "#idiom" ) : is_blank = False output . append ( SPIECE_UNDERLINE ) else : if text [ index : index + 6 ] == "#idiom" : is_blank = True if len ( output ) > 0 and output [ - 1 ] != SPIECE_UNDERLINE : output . append ( SPIECE_UNDERLINE ) output . append ( char ) elif _is_chinese_char ( cp ) or is_fuhao ( char ) : if len ( output ) > 0 and output [ - 1 ] != SPIECE_UNDERLINE : output . append ( SPIECE_UNDERLINE ) output . append ( char ) output . append ( SPIECE_UNDERLINE ) else : output . append ( char ) return "" . join ( output ) def is_whitespace ( c ) : if c == " " or c == "\t" or c == "\r" or c == "\n" or ord ( c ) == 0x202F or c == SPIECE_UNDERLINE : return True return False examples = [ ] example_id = 0 for data in tqdm ( input_data ) : data = eval ( data ) options = data [ 'candidates' ] for context in data [ 'content' ] : context = context . replace ( "“",  " ""). r e place(" ” ", "\ " ).re p l ace("—— " , "--"). replace ( "—",  " "). r e place(" ― ", "- " .re p l ace("…" ,  "... " .repl a c e("‘",  " \'"). r plac e ( "’", "\ ' ") context = _tokenize_chinese_chars ( context ) paragraph_text = context . strip ( ) doc_tokens = [ ] prev_is_whitespace = True for c in paragraph_text : if is_whitespace ( c ) : prev_is_whitespace = True else : if prev_is_whitespace : doc_tokens . append ( c ) else : doc_tokens [ - 1 ] += c prev_is_whitespace = False tags = [ blank for blank in doc_tokens if '#idiom' in blank ] if is_training : for tag_index , tag in enumerate ( tags ) : answer_index = input_label [ tag ] example = ChidExample ( example_id = example_id , tag = tag , doc_tokens = doc_tokens , options = options , answer_index = answer_index ) examples . append ( example ) else : for tag_index , tag in enumerate ( tags ) : example = ChidExample ( example_id = example_id , tag = tag , doc_tokens = doc_tokens , options = options ) examples . append ( example ) else : example_id += 1 else : print ( '原始样本个数：{}'.format(exampl e _id)) print ( '实际生成总样例数：{}'.format(len(exampl e s))) return examples