(examples, tokenizer, max_seq_length,
                                 doc_stride, max_query_length, is_training,
                                 output_fn)
Loads a data file into a list of `InputBatch`s.
Loads a data file into a list of `InputBatch`s.
Loads a data file into a list of InputBatch s .
def convert_examples_to_features(examples, tokenizer, max_seq_length,
                                 doc_stride, max_query_length, is_training,
                                 output_fn):
  """Loads a data file into a list of `InputBatch`s."""

  unique_id = 1000000000

  for (example_index, example) in enumerate(examples):
    query_tokens = tokenizer.tokenize(example.question_text)

    if len(query_tokens) > max_query_length:
      query_tokens = query_tokens[0:max_query_length]

    tok_to_orig_index = []
    orig_to_tok_index = []
    all_doc_tokens = []
    for (i, token) in enumerate(example.doc_tokens):
      orig_to_tok_index.append(len(all_doc_tokens))
      sub_tokens = tokenizer.tokenize(token)
      for sub_token in sub_tokens:
        tok_to_orig_index.append(i)
        all_doc_tokens.append(sub_token)

    tok_start_position = None
    tok_end_position = None
    if is_training and example.is_impossible:
      tok_start_position = -1
      tok_end_position = -1
    if is_training and not example.is_impossible:
      tok_start_position = orig_to_tok_index[example.start_position]
      if example.end_position < len(example.doc_tokens) - 1:
        tok_end_position = orig_to_tok_index[example.end_position + 1] - 1
      else:
        tok_end_position = len(all_doc_tokens) - 1
      (tok_start_position, tok_end_position) = _improve_answer_span(
          all_doc_tokens, tok_start_position, tok_end_position, tokenizer,
          example.orig_answer_text)

    # The -3 accounts for [CLS], [SEP] and [SEP]
    max_tokens_for_doc = max_seq_length - len(query_tokens) - 3

    # We can have documents that are longer than the maximum sequence length.
    # To deal with this we do a sliding window approach, where we take chunks
    # of the up to our max length with a stride of `doc_stride`.
    _DocSpan = collections.namedtuple(  # pylint: disable=invalid-name
        "DocSpan", ["start", "length"])
    doc_spans = []
    start_offset = 0
    while start_offset < len(all_doc_tokens):
      length = len(all_doc_tokens) - start_offset
      if length > max_tokens_for_doc:
        length = max_tokens_for_doc
      doc_spans.append(_DocSpan(start=start_offset, length=length))
      if start_offset + length == len(all_doc_tokens):
        break
      start_offset += min(length, doc_stride)

    for (doc_span_index, doc_span) in enumerate(doc_spans):
      tokens = []
      token_to_orig_map = {}
      token_is_max_context = {}
      segment_ids = []
      tokens.append("[CLS]")
      segment_ids.append(0)
      for token in query_tokens:
        tokens.append(token)
        segment_ids.append(0)
      tokens.append("[SEP]")
      segment_ids.append(0)

      for i in range(doc_span.length):
        split_token_index = doc_span.start + i
        token_to_orig_map[len(tokens)] = tok_to_orig_index[split_token_index]

        is_max_context = _check_is_max_context(doc_spans, doc_span_index,
                                               split_token_index)
        token_is_max_context[len(tokens)] = is_max_context
        tokens.append(all_doc_tokens[split_token_index])
        segment_ids.append(1)
      tokens.append("[SEP]")
      segment_ids.append(1)

      input_ids = tokenizer.convert_tokens_to_ids(tokens)

      # The mask has 1 for real tokens and 0 for padding tokens. Only real
      # tokens are attended to.
      input_mask = [1] * len(input_ids)

      # Zero-pad up to the sequence length.
      while len(input_ids) < max_seq_length:
        input_ids.append(0)
        input_mask.append(0)
        segment_ids.append(0)

      assert len(input_ids) == max_seq_length
      assert len(input_mask) == max_seq_length
      assert len(segment_ids) == max_seq_length

      start_position = None
      end_position = None
      if is_training and not example.is_impossible:
        # For training, if our document chunk does not contain an annotation
        # we throw it out, since there is nothing to predict.
        doc_start = doc_span.start
        doc_end = doc_span.start + doc_span.length - 1
        out_of_span = False
        if not (tok_start_position >= doc_start and
                tok_end_position <= doc_end):
          out_of_span = True
        if out_of_span:
          start_position = 0
          end_position = 0
        else:
          doc_offset = len(query_tokens) + 2
          start_position = tok_start_position - doc_start + doc_offset
          end_position = tok_end_position - doc_start + doc_offset

      if is_training and example.is_impossible:
        start_position = 0
        end_position = 0

      if example_index < 20:
        tf.logging.info("*** Example ***")
        tf.logging.info("unique_id: %s" % (unique_id))
        tf.logging.info("example_index: %s" % (example_index))
        tf.logging.info("doc_span_index: %s" % (doc_span_index))
        tf.logging.info("tokens: %s" % " ".join(
            [tokenization.printable_text(x) for x in tokens]))
        tf.logging.info("token_to_orig_map: %s" % " ".join(
            ["%d:%d" % (x, y) for (x, y) in six.iteritems(token_to_orig_map)]))
        tf.logging.info("token_is_max_context: %s" % " ".join([
            "%d:%s" % (x, y) for (x, y) in six.iteritems(token_is_max_context)
        ]))
        tf.logging.info("input_ids: %s" % " ".join([str(x) for x in input_ids]))
        tf.logging.info(
            "input_mask: %s" % " ".join([str(x) for x in input_mask]))
        tf.logging.info(
            "segment_ids: %s" % " ".join([str(x) for x in segment_ids]))
        if is_training and example.is_impossible:
          tf.logging.info("impossible example")
        if is_training and not example.is_impossible:
          answer_text = " ".join(tokens[start_position:(end_position + 1)])
          tf.logging.info("start_position: %d" % (start_position))
          tf.logging.info("end_position: %d" % (end_position))
          tf.logging.info(
              "answer: %s" % (tokenization.printable_text(answer_text)))

      feature = InputFeatures(
          unique_id=unique_id,
          example_index=example_index,
          doc_span_index=doc_span_index,
          tokens=tokens,
          token_to_orig_map=token_to_orig_map,
          token_is_max_context=token_is_max_context,
          input_ids=input_ids,
          input_mask=input_mask,
          segment_ids=segment_ids,
          start_position=start_position,
          end_position=end_position,
          is_impossible=example.is_impossible)

      # Run callback
      output_fn(feature)

      unique_id += 1
def convert_examples_to_features ( examples , tokenizer , max_seq_length , doc_stride , max_query_length , is_training , output_fn ) : unique_id = 1000000000 for ( example_index , example ) in enumerate ( examples ) : query_tokens = tokenizer . tokenize ( example . question_text ) if len ( query_tokens ) > max_query_length : query_tokens = query_tokens [ 0 : max_query_length ] tok_to_orig_index = [ ] orig_to_tok_index = [ ] all_doc_tokens = [ ] for ( i , token ) in enumerate ( example . doc_tokens ) : orig_to_tok_index . append ( len ( all_doc_tokens ) ) sub_tokens = tokenizer . tokenize ( token ) for sub_token in sub_tokens : tok_to_orig_index . append ( i ) all_doc_tokens . append ( sub_token ) tok_start_position = None tok_end_position = None if is_training and example . is_impossible : tok_start_position = - 1 tok_end_position = - 1 if is_training and not example . is_impossible : tok_start_position = orig_to_tok_index [ example . start_position ] if example . end_position < len ( example . doc_tokens ) - 1 : tok_end_position = orig_to_tok_index [ example . end_position + 1 ] - 1 else : tok_end_position = len ( all_doc_tokens ) - 1 ( tok_start_position , tok_end_position ) = _improve_answer_span ( all_doc_tokens , tok_start_position , tok_end_position , tokenizer , example . orig_answer_text ) # The -3 accounts for [CLS], [SEP] and [SEP] max_tokens_for_doc = max_seq_length - len ( query_tokens ) - 3 # We can have documents that are longer than the maximum sequence length. # To deal with this we do a sliding window approach, where we take chunks # of the up to our max length with a stride of `doc_stride`. _DocSpan = collections . namedtuple ( # pylint: disable=invalid-name "DocSpan" , [ "start" , "length" ] ) doc_spans = [ ] start_offset = 0 while start_offset < len ( all_doc_tokens ) : length = len ( all_doc_tokens ) - start_offset if length > max_tokens_for_doc : length = max_tokens_for_doc doc_spans . append ( _DocSpan ( start = start_offset , length = length ) ) if start_offset + length == len ( all_doc_tokens ) : break start_offset += min ( length , doc_stride ) for ( doc_span_index , doc_span ) in enumerate ( doc_spans ) : tokens = [ ] token_to_orig_map = { } token_is_max_context = { } segment_ids = [ ] tokens . append ( "[CLS]" ) segment_ids . append ( 0 ) for token in query_tokens : tokens . append ( token ) segment_ids . append ( 0 ) tokens . append ( "[SEP]" ) segment_ids . append ( 0 ) for i in range ( doc_span . length ) : split_token_index = doc_span . start + i token_to_orig_map [ len ( tokens ) ] = tok_to_orig_index [ split_token_index ] is_max_context = _check_is_max_context ( doc_spans , doc_span_index , split_token_index ) token_is_max_context [ len ( tokens ) ] = is_max_context tokens . append ( all_doc_tokens [ split_token_index ] ) segment_ids . append ( 1 ) tokens . append ( "[SEP]" ) segment_ids . append ( 1 ) input_ids = tokenizer . convert_tokens_to_ids ( tokens ) # The mask has 1 for real tokens and 0 for padding tokens. Only real # tokens are attended to. input_mask = [ 1 ] * len ( input_ids ) # Zero-pad up to the sequence length. while len ( input_ids ) < max_seq_length : input_ids . append ( 0 ) input_mask . append ( 0 ) segment_ids . append ( 0 ) assert len ( input_ids ) == max_seq_length assert len ( input_mask ) == max_seq_length assert len ( segment_ids ) == max_seq_length start_position = None end_position = None if is_training and not example . is_impossible : # For training, if our document chunk does not contain an annotation # we throw it out, since there is nothing to predict. doc_start = doc_span . start doc_end = doc_span . start + doc_span . length - 1 out_of_span = False if not ( tok_start_position >= doc_start and tok_end_position <= doc_end ) : out_of_span = True if out_of_span : start_position = 0 end_position = 0 else : doc_offset = len ( query_tokens ) + 2 start_position = tok_start_position - doc_start + doc_offset end_position = tok_end_position - doc_start + doc_offset if is_training and example . is_impossible : start_position = 0 end_position = 0 if example_index < 20 : tf . logging . info ( "*** Example ***" ) tf . logging . info ( "unique_id: %s" % ( unique_id ) ) tf . logging . info ( "example_index: %s" % ( example_index ) ) tf . logging . info ( "doc_span_index: %s" % ( doc_span_index ) ) tf . logging . info ( "tokens: %s" % " " . join ( [ tokenization . printable_text ( x ) for x in tokens ] ) ) tf . logging . info ( "token_to_orig_map: %s" % " " . join ( [ "%d:%d" % ( x , y ) for ( x , y ) in six . iteritems ( token_to_orig_map ) ] ) ) tf . logging . info ( "token_is_max_context: %s" % " " . join ( [ "%d:%s" % ( x , y ) for ( x , y ) in six . iteritems ( token_is_max_context ) ] ) ) tf . logging . info ( "input_ids: %s" % " " . join ( [ str ( x ) for x in input_ids ] ) ) tf . logging . info ( "input_mask: %s" % " " . join ( [ str ( x ) for x in input_mask ] ) ) tf . logging . info ( "segment_ids: %s" % " " . join ( [ str ( x ) for x in segment_ids ] ) ) if is_training and example . is_impossible : tf . logging . info ( "impossible example" ) if is_training and not example . is_impossible : answer_text = " " . join ( tokens [ start_position : ( end_position + 1 ) ] ) tf . logging . info ( "start_position: %d" % ( start_position ) ) tf . logging . info ( "end_position: %d" % ( end_position ) ) tf . logging . info ( "answer: %s" % ( tokenization . printable_text ( answer_text ) ) ) feature = InputFeatures ( unique_id = unique_id , example_index = example_index , doc_span_index = doc_span_index , tokens = tokens , token_to_orig_map = token_to_orig_map , token_is_max_context = token_is_max_context , input_ids = input_ids , input_mask = input_mask , segment_ids = segment_ids , start_position = start_position , end_position = end_position , is_impossible = example . is_impossible ) # Run callback output_fn ( feature ) unique_id += 1