(input_file, is_training, version_2_with_negative, max_answer_len=100000, skip_negatives=False)
return examples
Read a SQuAD json file into a list of SquadExample.
Read a SQuAD json file into a list of SquadExample.
Read a SQuAD json file into a list of SquadExample .
def read_squad_examples(input_file, is_training, version_2_with_negative, max_answer_len=100000, skip_negatives=False):
    """Read a SQuAD json file into a list of SquadExample."""
    with open(input_file, "r", encoding='utf-8') as reader:
        input_data = json.load(reader)["data"]

    def is_whitespace(c):
        if c == " " or c == "\t" or c == "\r" or c == "\n" or ord(c) == 0x202F:
            return True
        return False

    examples = []
    for entry in input_data:
        for paragraph in entry["paragraphs"]:
            paragraph_text = paragraph["context"]
            doc_tokens = []
            char_to_word_offset = []
            prev_is_whitespace = True
            for c in paragraph_text:
                if is_whitespace(c):
                    prev_is_whitespace = True
                else:
                    if prev_is_whitespace:
                        doc_tokens.append(c)
                    else:
                        doc_tokens[-1] += c
                    prev_is_whitespace = False
                char_to_word_offset.append(len(doc_tokens) - 1)

            for qa in paragraph["qas"]:
                qas_id = str(qa["id"])
                # this is temporary added to see whether reducing the negatives
                # improves the performance.
                if skip_negatives is True and "_NEGATIVE_" in qas_id:
                    continue
                if "FAKE2" in qas_id:
                    continue
                question_text = qa["question"]
                start_position = None
                end_position = None
                orig_answer_text = None
                is_impossible = False
                if is_training:
                    if version_2_with_negative:
                        is_impossible = qa["is_impossible"]
                        if is_impossible is False:
                            switch = 0
                        else:
                            switch = 1
                    # if (len(qa["answers"]) != 1) and (not is_impossible):
                    #    raise ValueError(
                    #        "For training, each question should have exactly 1 answer.")
                    if not is_impossible:
                        answers = qa['answers']
                        if type(answers) == list:
                            answer = qa["answers"][0]
                        else:
                            answer = answers
                        orig_answer_text = answer["text"]
                        answer_offset = answer["answer_start"]
                        answer_length = len(orig_answer_text)
                        start_position = char_to_word_offset[answer_offset]
                        end_position = char_to_word_offset[answer_offset +
                                                           answer_length - 1]
                        # Only add answers where the text can be exactly recovered from the
                        # document. If this CAN'T happen it's likely due to weird Unicode
                        # stuff so we will just skip the example.
                        #
                        # Note that this means for training mode, every example is NOT
                        # guaranteed to be preserved.
                        actual_text = " ".join(
                            doc_tokens[start_position:(end_position + 1)])
                        cleaned_answer_text = " ".join(
                            whitespace_tokenize(orig_answer_text))
                        if actual_text.find(cleaned_answer_text) == -1:
                            logger.warning("Could not find answer: '%s' vs. '%s'",
                                           actual_text, cleaned_answer_text)
                            continue

                        if len(orig_answer_text.split()) > max_answer_len:
                            logger.info(
                                "Omitting a long answer: '%s'", orig_answer_text)
                            continue
                    else:
                        start_position = -1
                        end_position = -1
                        orig_answer_text = ""

                        if len(qa["answers"]) > 0:
                            answer = qa["answers"][0]
                            # Make sure that answer text will be preserved for
                            # yes/no.
                            if answer["text"] in ["yes", "no"]:
                                orig_answer_text = answer["text"]
                if not is_training:
                    switch = None
                example = SquadExample(
                    qas_id=qas_id,
                    question_text=question_text,
                    doc_tokens=doc_tokens,
                    orig_answer_text=orig_answer_text,
                    start_position=start_position,
                    end_position=end_position,
                    switch=switch,
                    is_impossible=is_impossible)
                examples.append(example)
    return examples
def read_squad_examples ( input_file , is_training , version_2_with_negative , max_answer_len = 100000 , skip_negatives = False ) : with open ( input_file , "r" , encoding = 'utf-8' ) as reader : input_data = json . load ( reader ) [ "data" ] def is_whitespace ( c ) : if c == " " or c == "\t" or c == "\r" or c == "\n" or ord ( c ) == 0x202F : return True return False examples = [ ] for entry in input_data : for paragraph in entry [ "paragraphs" ] : paragraph_text = paragraph [ "context" ] doc_tokens = [ ] char_to_word_offset = [ ] prev_is_whitespace = True for c in paragraph_text : if is_whitespace ( c ) : prev_is_whitespace = True else : if prev_is_whitespace : doc_tokens . append ( c ) else : doc_tokens [ - 1 ] += c prev_is_whitespace = False char_to_word_offset . append ( len ( doc_tokens ) - 1 ) for qa in paragraph [ "qas" ] : qas_id = str ( qa [ "id" ] ) # this is temporary added to see whether reducing the negatives # improves the performance. if skip_negatives is True and "_NEGATIVE_" in qas_id : continue if "FAKE2" in qas_id : continue question_text = qa [ "question" ] start_position = None end_position = None orig_answer_text = None is_impossible = False if is_training : if version_2_with_negative : is_impossible = qa [ "is_impossible" ] if is_impossible is False : switch = 0 else : switch = 1 # if (len(qa["answers"]) != 1) and (not is_impossible): #    raise ValueError( #        "For training, each question should have exactly 1 answer.") if not is_impossible : answers = qa [ 'answers' ] if type ( answers ) == list : answer = qa [ "answers" ] [ 0 ] else : answer = answers orig_answer_text = answer [ "text" ] answer_offset = answer [ "answer_start" ] answer_length = len ( orig_answer_text ) start_position = char_to_word_offset [ answer_offset ] end_position = char_to_word_offset [ answer_offset + answer_length - 1 ] # Only add answers where the text can be exactly recovered from the # document. If this CAN'T happen it's likely due to weird Unicode # stuff so we will just skip the example. # # Note that this means for training mode, every example is NOT # guaranteed to be preserved. actual_text = " " . join ( doc_tokens [ start_position : ( end_position + 1 ) ] ) cleaned_answer_text = " " . join ( whitespace_tokenize ( orig_answer_text ) ) if actual_text . find ( cleaned_answer_text ) == - 1 : logger . warning ( "Could not find answer: '%s' vs. '%s'" , actual_text , cleaned_answer_text ) continue if len ( orig_answer_text . split ( ) ) > max_answer_len : logger . info ( "Omitting a long answer: '%s'" , orig_answer_text ) continue else : start_position = - 1 end_position = - 1 orig_answer_text = "" if len ( qa [ "answers" ] ) > 0 : answer = qa [ "answers" ] [ 0 ] # Make sure that answer text will be preserved for # yes/no. if answer [ "text" ] in [ "yes" , "no" ] : orig_answer_text = answer [ "text" ] if not is_training : switch = None example = SquadExample ( qas_id = qas_id , question_text = question_text , doc_tokens = doc_tokens , orig_answer_text = orig_answer_text , start_position = start_position , end_position = end_position , switch = switch , is_impossible = is_impossible ) examples . append ( example ) return examples