(all_examples, all_features, all_results, n_best_size,
                                             max_answer_length, do_lower_case, output_prediction_file,
                                             output_nbest_file, output_null_log_odds_file, verbose_logging,
                                             version_2_with_negative, null_score_diff_threshold, no_masking=False)
Write final predictions to the json file and log-odds of null if needed.
Write final predictions to the json file and log-odds of null if needed.
Write final predictions to the json file and log - odds of null if needed .
def write_predictions_yes_no_no_empty_answer(all_examples, all_features, all_results, n_best_size,
                                             max_answer_length, do_lower_case, output_prediction_file,
                                             output_nbest_file, output_null_log_odds_file, verbose_logging,
                                             version_2_with_negative, null_score_diff_threshold, no_masking=False):
    """Write final predictions to the json file and log-odds of null if needed."""
    logger.info("Writing predictions to: %s" % (output_prediction_file))
    logger.info("Writing nbest to: %s" % (output_nbest_file))

    example_index_to_features = collections.defaultdict(list)
    for feature in all_features:
        example_index_to_features[feature.example_index].append(feature)

    unique_id_to_result = {}
    for result in all_results:
        unique_id_to_result[result.unique_id] = result

    _PrelimPrediction = collections.namedtuple(  # pylint: disable=invalid-name
        "PrelimPrediction",
        ["feature_index", "start_index", "end_index", "start_logit", "end_logit"])

    all_predictions = collections.OrderedDict()
    all_nbest_json = collections.OrderedDict()

    for (example_index, example) in enumerate(all_examples):
        features = example_index_to_features[example_index]

        prelim_predictions = []
        score_null = 1000000  # large and positive
        min_null_feature_index = 0  # the paragraph slice with min null score
        null_start_logit = 0  # the start logit at the slice with min null score
        null_end_logit = 0  # the end logit at the slice with min null score

        for (feature_index, feature) in enumerate(features):
            result = unique_id_to_result[feature.unique_id]
            start_indexes = _get_best_indexes(result.start_logits, n_best_size)
            end_indexes = _get_best_indexes(result.end_logits, n_best_size)
            if no_masking is True:
                feature_null_score = result.start_logits[0] + \
                    result.end_logits[0]
                if feature_null_score < score_null:
                    score_null = feature_null_score
                    min_null_feature_index = feature_index
                    null_start_logit = result.start_logits[0]
                    null_end_logit = result.end_logits[0]
            # if we could have irrelevant answers, get the min score of
            # irrelevant
            for start_index in start_indexes:
                for end_index in end_indexes:
                    # We could hypothetically create invalid predictions, e.g., predict
                    # that the start of the span is in the question. We throw out all
                    # invalid predictions.
                    if start_index >= len(feature.tokens):
                        continue
                    if end_index >= len(feature.tokens):
                        continue
                    if start_index not in feature.token_to_orig_map:
                        continue
                    if end_index not in feature.token_to_orig_map:
                        continue
                    if not feature.token_is_max_context.get(start_index, False):
                        continue
                    if end_index < start_index:
                        continue
                    length = end_index - start_index + 1
                    if length > max_answer_length:
                        continue
                    prelim_predictions.append(
                        _PrelimPrediction(
                            feature_index=feature_index,
                            start_index=start_index,
                            end_index=end_index,
                            start_logit=result.start_logits[start_index],
                            end_logit=result.end_logits[end_index]))
        if no_masking is True:
            prelim_predictions.append(
                _PrelimPrediction(
                    feature_index=min_null_feature_index,
                    start_index=0,
                    end_index=0,
                    start_logit=null_start_logit,
                    end_logit=null_end_logit))
        prelim_predictions = sorted(
            prelim_predictions,
            key=lambda x: (x.start_logit + x.end_logit),
            reverse=True)

        _NbestPrediction = collections.namedtuple(  # pylint: disable=invalid-name
            "NbestPrediction", ["text", "start_logit", "end_logit", "no_answer_logit", "switch", "switch_logits"])
        no_answer_logit = result.switch_logits[1]

        seen_predictions = {}
        nbest = []
        for pred in prelim_predictions:
            if len(nbest) >= n_best_size:
                break
            feature = features[pred.feature_index]
            if pred.start_index > 0:  # this is a non-null prediction
                tok_tokens = feature.tokens[pred.start_index:(
                    pred.end_index + 1)]
                orig_doc_start = feature.token_to_orig_map[pred.start_index]
                orig_doc_end = feature.token_to_orig_map[pred.end_index]
                orig_tokens = example.doc_tokens[orig_doc_start:(
                    orig_doc_end + 1)]
                tok_text = " ".join(tok_tokens)

                # De-tokenize WordPieces that have been split off.
                tok_text = tok_text.replace(" ##", "")
                tok_text = tok_text.replace("##", "")

                # Clean whitespace
                tok_text = tok_text.strip()
                tok_text = " ".join(tok_text.split())
                orig_text = " ".join(orig_tokens)

                final_text = get_final_text(
                    tok_text, orig_text, do_lower_case, verbose_logging)
                if final_text in seen_predictions:
                    continue

                seen_predictions[final_text] = True
            else:
                final_text = ""
                seen_predictions[final_text] = True

            nbest.append(
                _NbestPrediction(
                    text=final_text,
                    start_logit=pred.start_logit,
                    end_logit=pred.end_logit,
                    no_answer_logit=no_answer_logit,
                    switch=np.argmax(result.switch_logits),
                    switch_logits=result.switch_logits
                ))
        # if we didn't include the empty option in the n-best, include it
        if no_masking is True:
            if "" not in seen_predictions:
                nbest.append(
                    _NbestPrediction(
                        text="",
                        start_logit=null_start_logit,
                        end_logit=null_end_logit,
                        no_answer_logit=no_answer_logit,
                        switch=np.argmax(result.switch_logits),
                        switch_logits=result.switch_logits
                    ))

            # In very rare edge cases we could only have single null prediction.
            # So we just create a nonce prediction in this case to avoid
            # failure.
            if no_masking is True:
                if len(nbest) == 1:
                    nbest.insert(0,
                                 _NbestPrediction(text="", start_logit=0.0, end_logit=0.0, no_answer_logit=1.0, switch=1, switch_logits=[0.0, 0.0, 0.0, 0.0]))
            else:
                if len(nbest) == 0:
                    nbest.insert(0,
                                 _NbestPrediction(text="", start_logit=0.0, end_logit=0.0, no_answer_logit=1.0, switch=1, switch_logits=[0.0, 0.0, 0.0, 0.0]))

        # In very rare edge cases we could have no valid predictions. So we
        # just create a nonce prediction in this case to avoid failure.
        if not nbest:
            nbest.append(
                _NbestPrediction(text="", start_logit=0.0, end_logit=0.0, no_answer_logit=1.0, switch=1, switch_logits=[0.0, 0.0, 0.0, 0.0]))

        assert len(nbest) >= 1

        total_scores = []
        for entry in nbest:
            total_scores.append(entry.start_logit + entry.end_logit)

        probs = _compute_softmax(total_scores)

        nbest_json = []
        for (i, entry) in enumerate(nbest):
            output = collections.OrderedDict()
            output["text"] = entry.text
            output["probability"] = probs[i]
            output["start_logit"] = entry.start_logit
            output["end_logit"] = entry.end_logit
            output["no_answer_prob"] = entry.no_answer_logit
            output["switch"] = entry.switch
            output["switch_scores"] = entry.switch_logits
            nbest_json.append(output)

        assert len(nbest_json) >= 1

        # if the n-best is high enough, pick up no answer.
        possible_answers = np.argsort(
            nbest_json[0]["switch_scores"])[::-1]
        if possible_answers[0] == 1:
            all_predictions[example.qas_id] = switch_answers(
                possible_answers[1], nbest_json[0]["text"])
        else:
            all_predictions[example.qas_id] = switch_answers(
                possible_answers[0], nbest_json[0]["text"])

        all_nbest_json[example.qas_id] = nbest_json

    with open(output_prediction_file, "w") as writer:
        writer.write(json.dumps(all_predictions, indent=4) + "\n")
def write_predictions_yes_no_no_empty_answer ( all_examples , all_features , all_results , n_best_size , max_answer_length , do_lower_case , output_prediction_file , output_nbest_file , output_null_log_odds_file , verbose_logging , version_2_with_negative , null_score_diff_threshold , no_masking = False ) : logger . info ( "Writing predictions to: %s" % ( output_prediction_file ) ) logger . info ( "Writing nbest to: %s" % ( output_nbest_file ) ) example_index_to_features = collections . defaultdict ( list ) for feature in all_features : example_index_to_features [ feature . example_index ] . append ( feature ) unique_id_to_result = { } for result in all_results : unique_id_to_result [ result . unique_id ] = result _PrelimPrediction = collections . namedtuple ( # pylint: disable=invalid-name "PrelimPrediction" , [ "feature_index" , "start_index" , "end_index" , "start_logit" , "end_logit" ] ) all_predictions = collections . OrderedDict ( ) all_nbest_json = collections . OrderedDict ( ) for ( example_index , example ) in enumerate ( all_examples ) : features = example_index_to_features [ example_index ] prelim_predictions = [ ] score_null = 1000000 # large and positive min_null_feature_index = 0 # the paragraph slice with min null score null_start_logit = 0 # the start logit at the slice with min null score null_end_logit = 0 # the end logit at the slice with min null score for ( feature_index , feature ) in enumerate ( features ) : result = unique_id_to_result [ feature . unique_id ] start_indexes = _get_best_indexes ( result . start_logits , n_best_size ) end_indexes = _get_best_indexes ( result . end_logits , n_best_size ) if no_masking is True : feature_null_score = result . start_logits [ 0 ] + result . end_logits [ 0 ] if feature_null_score < score_null : score_null = feature_null_score min_null_feature_index = feature_index null_start_logit = result . start_logits [ 0 ] null_end_logit = result . end_logits [ 0 ] # if we could have irrelevant answers, get the min score of # irrelevant for start_index in start_indexes : for end_index in end_indexes : # We could hypothetically create invalid predictions, e.g., predict # that the start of the span is in the question. We throw out all # invalid predictions. if start_index >= len ( feature . tokens ) : continue if end_index >= len ( feature . tokens ) : continue if start_index not in feature . token_to_orig_map : continue if end_index not in feature . token_to_orig_map : continue if not feature . token_is_max_context . get ( start_index , False ) : continue if end_index < start_index : continue length = end_index - start_index + 1 if length > max_answer_length : continue prelim_predictions . append ( _PrelimPrediction ( feature_index = feature_index , start_index = start_index , end_index = end_index , start_logit = result . start_logits [ start_index ] , end_logit = result . end_logits [ end_index ] ) ) if no_masking is True : prelim_predictions . append ( _PrelimPrediction ( feature_index = min_null_feature_index , start_index = 0 , end_index = 0 , start_logit = null_start_logit , end_logit = null_end_logit ) ) prelim_predictions = sorted ( prelim_predictions , key = lambda x : ( x . start_logit + x . end_logit ) , reverse = True ) _NbestPrediction = collections . namedtuple ( # pylint: disable=invalid-name "NbestPrediction" , [ "text" , "start_logit" , "end_logit" , "no_answer_logit" , "switch" , "switch_logits" ] ) no_answer_logit = result . switch_logits [ 1 ] seen_predictions = { } nbest = [ ] for pred in prelim_predictions : if len ( nbest ) >= n_best_size : break feature = features [ pred . feature_index ] if pred . start_index > 0 : # this is a non-null prediction tok_tokens = feature . tokens [ pred . start_index : ( pred . end_index + 1 ) ] orig_doc_start = feature . token_to_orig_map [ pred . start_index ] orig_doc_end = feature . token_to_orig_map [ pred . end_index ] orig_tokens = example . doc_tokens [ orig_doc_start : ( orig_doc_end + 1 ) ] tok_text = " " . join ( tok_tokens ) # De-tokenize WordPieces that have been split off. tok_text = tok_text . replace ( " ##" , "" ) tok_text = tok_text . replace ( "##" , "" ) # Clean whitespace tok_text = tok_text . strip ( ) tok_text = " " . join ( tok_text . split ( ) ) orig_text = " " . join ( orig_tokens ) final_text = get_final_text ( tok_text , orig_text , do_lower_case , verbose_logging ) if final_text in seen_predictions : continue seen_predictions [ final_text ] = True else : final_text = "" seen_predictions [ final_text ] = True nbest . append ( _NbestPrediction ( text = final_text , start_logit = pred . start_logit , end_logit = pred . end_logit , no_answer_logit = no_answer_logit , switch = np . argmax ( result . switch_logits ) , switch_logits = result . switch_logits ) ) # if we didn't include the empty option in the n-best, include it if no_masking is True : if "" not in seen_predictions : nbest . append ( _NbestPrediction ( text = "" , start_logit = null_start_logit , end_logit = null_end_logit , no_answer_logit = no_answer_logit , switch = np . argmax ( result . switch_logits ) , switch_logits = result . switch_logits ) ) # In very rare edge cases we could only have single null prediction. # So we just create a nonce prediction in this case to avoid # failure. if no_masking is True : if len ( nbest ) == 1 : nbest . insert ( 0 , _NbestPrediction ( text = "" , start_logit = 0.0 , end_logit = 0.0 , no_answer_logit = 1.0 , switch = 1 , switch_logits = [ 0.0 , 0.0 , 0.0 , 0.0 ] ) ) else : if len ( nbest ) == 0 : nbest . insert ( 0 , _NbestPrediction ( text = "" , start_logit = 0.0 , end_logit = 0.0 , no_answer_logit = 1.0 , switch = 1 , switch_logits = [ 0.0 , 0.0 , 0.0 , 0.0 ] ) ) # In very rare edge cases we could have no valid predictions. So we # just create a nonce prediction in this case to avoid failure. if not nbest : nbest . append ( _NbestPrediction ( text = "" , start_logit = 0.0 , end_logit = 0.0 , no_answer_logit = 1.0 , switch = 1 , switch_logits = [ 0.0 , 0.0 , 0.0 , 0.0 ] ) ) assert len ( nbest ) >= 1 total_scores = [ ] for entry in nbest : total_scores . append ( entry . start_logit + entry . end_logit ) probs = _compute_softmax ( total_scores ) nbest_json = [ ] for ( i , entry ) in enumerate ( nbest ) : output = collections . OrderedDict ( ) output [ "text" ] = entry . text output [ "probability" ] = probs [ i ] output [ "start_logit" ] = entry . start_logit output [ "end_logit" ] = entry . end_logit output [ "no_answer_prob" ] = entry . no_answer_logit output [ "switch" ] = entry . switch output [ "switch_scores" ] = entry . switch_logits nbest_json . append ( output ) assert len ( nbest_json ) >= 1 # if the n-best is high enough, pick up no answer. possible_answers = np . argsort ( nbest_json [ 0 ] [ "switch_scores" ] ) [ : : - 1 ] if possible_answers [ 0 ] == 1 : all_predictions [ example . qas_id ] = switch_answers ( possible_answers [ 1 ] , nbest_json [ 0 ] [ "text" ] ) else : all_predictions [ example . qas_id ] = switch_answers ( possible_answers [ 0 ] , nbest_json [ 0 ] [ "text" ] ) all_nbest_json [ example . qas_id ] = nbest_json with open ( output_prediction_file , "w" ) as writer : writer . write ( json . dumps ( all_predictions , indent = 4 ) + "\n" )