(pred_text, orig_text, do_lower_case, verbose_logging=False)
return output_text
Project the tokenized prediction back to the original text.
Project the tokenized prediction back to the original text.
Project the tokenized prediction back to the original text .
def get_final_text(pred_text, orig_text, do_lower_case, verbose_logging=False):
    """Project the tokenized prediction back to the original text."""

    # When we created the data, we kept track of the alignment between original
    # (whitespace tokenized) tokens and our WordPiece tokenized tokens. So
    # now `orig_text` contains the span of our original text corresponding to the
    # span that we predicted.
    #
    # However, `orig_text` may contain extra characters that we don't want in
    # our prediction.
    #
    # For example, let's say:
    #   pred_text = steve smith
    #   orig_text = Steve Smith's
    #
    # We don't want to return `orig_text` because it contains the extra "'s".
    #
    # We don't want to return `pred_text` because it's already been normalized
    # (the SQuAD eval script also does punctuation stripping/lower casing but
    # our tokenizer does additional normalization like stripping accent
    # characters).
    #
    # What we really want to return is "Steve Smith".
    #
    # Therefore, we have to apply a semi-complicated alignment heuristic between
    # `pred_text` and `orig_text` to get a character-to-character alignment. This
    # can fail in certain cases in which case we just return `orig_text`.

    def _strip_spaces(text):
        ns_chars = []
        ns_to_s_map = collections.OrderedDict()
        for (i, c) in enumerate(text):
            if c == " ":
                continue
            ns_to_s_map[len(ns_chars)] = i
            ns_chars.append(c)
        ns_text = "".join(ns_chars)
        return (ns_text, ns_to_s_map)

    # We first tokenize `orig_text`, strip whitespace from the result
    # and `pred_text`, and check if they are the same length. If they are
    # NOT the same length, the heuristic has failed. If they are the same
    # length, we assume the characters are one-to-one aligned.
    tokenizer = BasicTokenizer(do_lower_case=do_lower_case)

    tok_text = "".join(tokenizer.tokenize(orig_text))

    start_position = tok_text.find(pred_text)
    if start_position == -1:
        if verbose_logging:
            print("Unable to find text: '%s' in '%s'" % (pred_text, orig_text))
        return orig_text
    end_position = start_position + len(pred_text) - 1

    (orig_ns_text, orig_ns_to_s_map) = _strip_spaces(orig_text)
    (tok_ns_text, tok_ns_to_s_map) = _strip_spaces(tok_text)

    if len(orig_ns_text) != len(tok_ns_text):
        if verbose_logging:
            print("Length not equal after stripping spaces: '%s' vs '%s'" % (orig_ns_text, tok_ns_text))
        return orig_text

    # We then project the characters in `pred_text` back to `orig_text` using
    # the character-to-character alignment.
    tok_s_to_ns_map = {}
    for (i, tok_index) in tok_ns_to_s_map.items():
        tok_s_to_ns_map[tok_index] = i

    orig_start_position = None
    if start_position in tok_s_to_ns_map:
        ns_start_position = tok_s_to_ns_map[start_position]
        if ns_start_position in orig_ns_to_s_map:
            orig_start_position = orig_ns_to_s_map[ns_start_position]

    if orig_start_position is None:
        if verbose_logging:
            print("Couldn't map start position")
        return orig_text

    orig_end_position = None
    if end_position in tok_s_to_ns_map:
        ns_end_position = tok_s_to_ns_map[end_position]
        if ns_end_position in orig_ns_to_s_map:
            orig_end_position = orig_ns_to_s_map[ns_end_position]

    if orig_end_position is None:
        if verbose_logging:
            print("Couldn't map end position")
        return orig_text

    output_text = orig_text[orig_start_position:(orig_end_position + 1)]
    return output_text
def get_final_text ( pred_text , orig_text , do_lower_case , verbose_logging = False ) : # When we created the data, we kept track of the alignment between original # (whitespace tokenized) tokens and our WordPiece tokenized tokens. So # now `orig_text` contains the span of our original text corresponding to the # span that we predicted. # # However, `orig_text` may contain extra characters that we don't want in # our prediction. # # For example, let's say: #   pred_text = steve smith #   orig_text = Steve Smith's # # We don't want to return `orig_text` because it contains the extra "'s". # # We don't want to return `pred_text` because it's already been normalized # (the SQuAD eval script also does punctuation stripping/lower casing but # our tokenizer does additional normalization like stripping accent # characters). # # What we really want to return is "Steve Smith". # # Therefore, we have to apply a semi-complicated alignment heuristic between # `pred_text` and `orig_text` to get a character-to-character alignment. This # can fail in certain cases in which case we just return `orig_text`. def _strip_spaces ( text ) : ns_chars = [ ] ns_to_s_map = collections . OrderedDict ( ) for ( i , c ) in enumerate ( text ) : if c == " " : continue ns_to_s_map [ len ( ns_chars ) ] = i ns_chars . append ( c ) ns_text = "" . join ( ns_chars ) return ( ns_text , ns_to_s_map ) # We first tokenize `orig_text`, strip whitespace from the result # and `pred_text`, and check if they are the same length. If they are # NOT the same length, the heuristic has failed. If they are the same # length, we assume the characters are one-to-one aligned. tokenizer = BasicTokenizer ( do_lower_case = do_lower_case ) tok_text = "" . join ( tokenizer . tokenize ( orig_text ) ) start_position = tok_text . find ( pred_text ) if start_position == - 1 : if verbose_logging : print ( "Unable to find text: '%s' in '%s'" % ( pred_text , orig_text ) ) return orig_text end_position = start_position + len ( pred_text ) - 1 ( orig_ns_text , orig_ns_to_s_map ) = _strip_spaces ( orig_text ) ( tok_ns_text , tok_ns_to_s_map ) = _strip_spaces ( tok_text ) if len ( orig_ns_text ) != len ( tok_ns_text ) : if verbose_logging : print ( "Length not equal after stripping spaces: '%s' vs '%s'" % ( orig_ns_text , tok_ns_text ) ) return orig_text # We then project the characters in `pred_text` back to `orig_text` using # the character-to-character alignment. tok_s_to_ns_map = { } for ( i , tok_index ) in tok_ns_to_s_map . items ( ) : tok_s_to_ns_map [ tok_index ] = i orig_start_position = None if start_position in tok_s_to_ns_map : ns_start_position = tok_s_to_ns_map [ start_position ] if ns_start_position in orig_ns_to_s_map : orig_start_position = orig_ns_to_s_map [ ns_start_position ] if orig_start_position is None : if verbose_logging : print ( "Couldn't map start position" ) return orig_text orig_end_position = None if end_position in tok_s_to_ns_map : ns_end_position = tok_s_to_ns_map [ end_position ] if ns_end_position in orig_ns_to_s_map : orig_end_position = orig_ns_to_s_map [ ns_end_position ] if orig_end_position is None : if verbose_logging : print ( "Couldn't map end position" ) return orig_text output_text = orig_text [ orig_start_position : ( orig_end_position + 1 ) ] return output_text