(config, all_examples, all_features, all_results, n_best_size,
                           max_answer_length, do_lower_case, output_prediction_file,
                           output_nbest_file)
Write final predictions to the json file and log-odds of null if needed.
Write final predictions to the json file and log-odds of null if needed.
Write final predictions to the json file and log - odds of null if needed .
def write_predictions_topk(config, all_examples, all_features, all_results, n_best_size,
                           max_answer_length, do_lower_case, output_prediction_file,
                           output_nbest_file):
    """Write final predictions to the json file and log-odds of null if needed."""
    print("Writing predictions to: %s" % (output_prediction_file))
    print("Writing nbest to: %s" % (output_nbest_file))

    example_index_to_features = collections.defaultdict(list)
    for feature in all_features:
        example_index_to_features[feature['example_index']].append(feature)

    unique_id_to_result = {}
    for result in all_results:
        unique_id_to_result[result.unique_id] = result

    _PrelimPrediction = collections.namedtuple(  # pylint: disable=invalid-name
        "PrelimPrediction",
        ["feature_index", "start_index", "end_index", "start_logit", "end_logit"])

    all_predictions = collections.OrderedDict()
    all_nbest_json = collections.OrderedDict()

    for (example_index, example) in enumerate(tqdm(all_examples)):
        features = example_index_to_features[example_index]
        prelim_predictions = []
        # keep track of the minimum score of null start+end of position 0
        for (feature_index, feature) in enumerate(features):
            result = unique_id_to_result[feature['unique_id']]
            for i in range(config.start_n_top):
                for j in range(config.end_n_top):
                    start_logit = result.start_top_logits[i]
                    start_index = result.start_top_index[i]

                    j_index = i * config.end_n_top + j

                    end_logit = result.end_top_logits[j_index]
                    end_index = result.end_top_index[j_index]

                    # We could hypothetically create invalid predictions, e.g., predict
                    # that the start of the span is in the question. We throw out all
                    # invalid predictions.
                    if start_index >= len(feature['tokens']):
                        continue
                    if end_index >= len(feature['tokens']):
                        continue
                    if not feature['token_is_max_context'].get(str(start_index), False):
                        continue
                    if end_index < start_index:
                        continue
                    length = end_index - start_index + 1
                    if length > max_answer_length:
                        continue

                    prelim_predictions.append(
                        _PrelimPrediction(
                            feature_index=feature_index,
                            start_index=start_index,
                            end_index=end_index,
                            start_logit=start_logit,
                            end_logit=end_logit))

        prelim_predictions = sorted(
            prelim_predictions,
            key=lambda x: (x.start_logit + x.end_logit),
            reverse=True)

        _NbestPrediction = collections.namedtuple(  # pylint: disable=invalid-name
            "NbestPrediction", ["text", "start_logit", "end_logit"])

        seen_predictions = {}
        nbest = []
        # ipdb.set_trace()
        for pred in prelim_predictions:
            if len(nbest) >= n_best_size:
                break
            feature = features[pred.feature_index]
            if pred.start_index > 0:  # this is a non-null prediction
                tok_tokens = feature['tokens'][pred.start_index:(pred.end_index + 1)]
                orig_doc_start = feature['token_to_orig_map'][str(pred.start_index)]
                orig_doc_end = feature['token_to_orig_map'][str(pred.end_index)]
                orig_tokens = example['ori_doc_tokens'][orig_doc_start:(orig_doc_end + 1)]
                tok_text = "".join(tok_tokens)

                # De-tokenize WordPieces that have been split off.
                tok_text = tok_text.replace(" ##", "")
                tok_text = tok_text.replace("##", "")

                # Clean whitespace
                tok_text = tok_text.strip()
                tok_text = " ".join(tok_text.split())
                orig_text = "".join(orig_tokens)

                final_text = get_final_text(tok_text, orig_text, do_lower_case)
                if final_text in seen_predictions:
                    continue

                seen_predictions[final_text] = True
            else:
                final_text = ""
                seen_predictions[final_text] = True

            nbest.append(
                _NbestPrediction(
                    text=final_text,
                    start_logit=pred.start_logit,
                    end_logit=pred.end_logit))

        # In very rare edge cases we could have no valid predictions. So we
        # just create a nonce prediction in this case to avoid failure.
        if not nbest:
            nbest.append(_NbestPrediction(text="empty", start_logit=0.0, end_logit=0.0))

        assert len(nbest) >= 1
        # ipdb.set_trace()

        total_scores = []
        for entry in nbest:
            total_scores.append(entry.start_logit + entry.end_logit)

        probs = _compute_softmax(total_scores)

        # ipdb.set_trace()

        nbest_json = []
        # ipdb.set_trace()
        for (i, entry) in enumerate(nbest):
            output = collections.OrderedDict()
            output["text"] = entry.text
            output["probability"] = float(probs[i])
            output["start_logit"] = float(entry.start_logit)
            output["end_logit"] = float(entry.end_logit)
            nbest_json.append(output)

        assert len(nbest_json) >= 1

        # ipdb.set_trace()

        all_predictions[example['qid']] = nbest_json[0]["text"]
        all_nbest_json[example['qid']] = nbest_json

    with open(output_prediction_file, "w") as writer:
        writer.write(json.dumps(all_predictions, indent=4, ensure_ascii=False) + "\n")

    with open(output_nbest_file, "w") as writer:
        writer.write(json.dumps(all_nbest_json, indent=4, ensure_ascii=False) + "\n")
def write_predictions_topk ( config , all_examples , all_features , all_results , n_best_size , max_answer_length , do_lower_case , output_prediction_file , output_nbest_file ) : print ( "Writing predictions to: %s" % ( output_prediction_file ) ) print ( "Writing nbest to: %s" % ( output_nbest_file ) ) example_index_to_features = collections . defaultdict ( list ) for feature in all_features : example_index_to_features [ feature [ 'example_index' ] ] . append ( feature ) unique_id_to_result = { } for result in all_results : unique_id_to_result [ result . unique_id ] = result _PrelimPrediction = collections . namedtuple ( # pylint: disable=invalid-name "PrelimPrediction" , [ "feature_index" , "start_index" , "end_index" , "start_logit" , "end_logit" ] ) all_predictions = collections . OrderedDict ( ) all_nbest_json = collections . OrderedDict ( ) for ( example_index , example ) in enumerate ( tqdm ( all_examples ) ) : features = example_index_to_features [ example_index ] prelim_predictions = [ ] # keep track of the minimum score of null start+end of position 0 for ( feature_index , feature ) in enumerate ( features ) : result = unique_id_to_result [ feature [ 'unique_id' ] ] for i in range ( config . start_n_top ) : for j in range ( config . end_n_top ) : start_logit = result . start_top_logits [ i ] start_index = result . start_top_index [ i ] j_index = i * config . end_n_top + j end_logit = result . end_top_logits [ j_index ] end_index = result . end_top_index [ j_index ] # We could hypothetically create invalid predictions, e.g., predict # that the start of the span is in the question. We throw out all # invalid predictions. if start_index >= len ( feature [ 'tokens' ] ) : continue if end_index >= len ( feature [ 'tokens' ] ) : continue if not feature [ 'token_is_max_context' ] . get ( str ( start_index ) , False ) : continue if end_index < start_index : continue length = end_index - start_index + 1 if length > max_answer_length : continue prelim_predictions . append ( _PrelimPrediction ( feature_index = feature_index , start_index = start_index , end_index = end_index , start_logit = start_logit , end_logit = end_logit ) ) prelim_predictions = sorted ( prelim_predictions , key = lambda x : ( x . start_logit + x . end_logit ) , reverse = True ) _NbestPrediction = collections . namedtuple ( # pylint: disable=invalid-name "NbestPrediction" , [ "text" , "start_logit" , "end_logit" ] ) seen_predictions = { } nbest = [ ] # ipdb.set_trace() for pred in prelim_predictions : if len ( nbest ) >= n_best_size : break feature = features [ pred . feature_index ] if pred . start_index > 0 : # this is a non-null prediction tok_tokens = feature [ 'tokens' ] [ pred . start_index : ( pred . end_index + 1 ) ] orig_doc_start = feature [ 'token_to_orig_map' ] [ str ( pred . start_index ) ] orig_doc_end = feature [ 'token_to_orig_map' ] [ str ( pred . end_index ) ] orig_tokens = example [ 'ori_doc_tokens' ] [ orig_doc_start : ( orig_doc_end + 1 ) ] tok_text = "" . join ( tok_tokens ) # De-tokenize WordPieces that have been split off. tok_text = tok_text . replace ( " ##" , "" ) tok_text = tok_text . replace ( "##" , "" ) # Clean whitespace tok_text = tok_text . strip ( ) tok_text = " " . join ( tok_text . split ( ) ) orig_text = "" . join ( orig_tokens ) final_text = get_final_text ( tok_text , orig_text , do_lower_case ) if final_text in seen_predictions : continue seen_predictions [ final_text ] = True else : final_text = "" seen_predictions [ final_text ] = True nbest . append ( _NbestPrediction ( text = final_text , start_logit = pred . start_logit , end_logit = pred . end_logit ) ) # In very rare edge cases we could have no valid predictions. So we # just create a nonce prediction in this case to avoid failure. if not nbest : nbest . append ( _NbestPrediction ( text = "empty" , start_logit = 0.0 , end_logit = 0.0 ) ) assert len ( nbest ) >= 1 # ipdb.set_trace() total_scores = [ ] for entry in nbest : total_scores . append ( entry . start_logit + entry . end_logit ) probs = _compute_softmax ( total_scores ) # ipdb.set_trace() nbest_json = [ ] # ipdb.set_trace() for ( i , entry ) in enumerate ( nbest ) : output = collections . OrderedDict ( ) output [ "text" ] = entry . text output [ "probability" ] = float ( probs [ i ] ) output [ "start_logit" ] = float ( entry . start_logit ) output [ "end_logit" ] = float ( entry . end_logit ) nbest_json . append ( output ) assert len ( nbest_json ) >= 1 # ipdb.set_trace() all_predictions [ example [ 'qid' ] ] = nbest_json [ 0 ] [ "text" ] all_nbest_json [ example [ 'qid' ] ] = nbest_json with open ( output_prediction_file , "w" ) as writer : writer . write ( json . dumps ( all_predictions , indent = 4 , ensure_ascii = False ) + "\n" ) with open ( output_nbest_file , "w" ) as writer : writer . write ( json . dumps ( all_nbest_json , indent = 4 , ensure_ascii = False ) + "\n" )