(doc_tokens, input_start, input_end, tokenizer,
                         orig_answer_text)
return (input_start, input_end)
Returns tokenized answer spans that better match the annotated answer.
Returns tokenized answer spans that better match the annotated answer.
Returns tokenized answer spans that better match the annotated answer .
def _improve_answer_span(doc_tokens, input_start, input_end, tokenizer,
                         orig_answer_text):
  """Returns tokenized answer spans that better match the annotated answer."""

  # The SQuAD annotations are character based. We first project them to
  # whitespace-tokenized words. But then after WordPiece tokenization, we can
  # often find a "better match". For example:
  #
  #   Question: What year was John Smith born?
  #   Context: The leader was John Smith (1895-1943).
  #   Answer: 1895
  #
  # The original whitespace-tokenized answer will be "(1895-1943).". However
  # after tokenization, our tokens will be "( 1895 - 1943 ) .". So we can match
  # the exact answer, 1895.
  #
  # However, this is not always possible. Consider the following:
  #
  #   Question: What country is the top exporter of electornics?
  #   Context: The Japanese electronics industry is the lagest in the world.
  #   Answer: Japan
  #
  # In this case, the annotator chose "Japan" as a character sub-span of
  # the word "Japanese". Since our WordPiece tokenizer does not split
  # "Japanese", we just use "Japanese" as the annotation. This is fairly rare
  # in SQuAD, but does happen.
  tok_answer_text = " ".join(tokenizer.tokenize(orig_answer_text))

  for new_start in range(input_start, input_end + 1):
    for new_end in range(input_end, new_start - 1, -1):
      text_span = " ".join(doc_tokens[new_start:(new_end + 1)])
      if text_span == tok_answer_text:
        return (new_start, new_end)

  return (input_start, input_end)
def _improve_answer_span ( doc_tokens , input_start , input_end , tokenizer , orig_answer_text ) : # The SQuAD annotations are character based. We first project them to # whitespace-tokenized words. But then after WordPiece tokenization, we can # often find a "better match". For example: # #   Question: What year was John Smith born? #   Context: The leader was John Smith (1895-1943). #   Answer: 1895 # # The original whitespace-tokenized answer will be "(1895-1943).". However # after tokenization, our tokens will be "( 1895 - 1943 ) .". So we can match # the exact answer, 1895. # # However, this is not always possible. Consider the following: # #   Question: What country is the top exporter of electornics? #   Context: The Japanese electronics industry is the lagest in the world. #   Answer: Japan # # In this case, the annotator chose "Japan" as a character sub-span of # the word "Japanese". Since our WordPiece tokenizer does not split # "Japanese", we just use "Japanese" as the annotation. This is fairly rare # in SQuAD, but does happen. tok_answer_text = " " . join ( tokenizer . tokenize ( orig_answer_text ) ) for new_start in range ( input_start , input_end + 1 ) : for new_end in range ( input_end , new_start - 1 , - 1 ) : text_span = " " . join ( doc_tokens [ new_start : ( new_end + 1 ) ] ) if text_span == tok_answer_text : return ( new_start , new_end ) return ( input_start , input_end )