(tokens, masked_lm_prob,
                                 max_predictions_per_seq, vocab_words, rng)
return (output_tokens, masked_lm_positions, masked_lm_labels)
Creates the predictions for the masked LM objective.
Creates the predictions for the masked LM objective.
Creates the predictions for the masked LM objective .
def create_masked_lm_predictions_original(tokens, masked_lm_prob,
                                 max_predictions_per_seq, vocab_words, rng):
  """Creates the predictions for the masked LM objective."""

  cand_indexes = []
  for (i, token) in enumerate(tokens):
    if token == "[CLS]" or token == "[SEP]":
      continue
    # Whole Word Masking means that if we mask all of the wordpieces
    # corresponding to an original word. When a word has been split into
    # WordPieces, the first token does not have any marker and any subsequence
    # tokens are prefixed with ##. So whenever we see the ## token, we
    # append it to the previous set of word indexes.
    #
    # Note that Whole Word Masking does *not* change the training code
    # at all -- we still predict each WordPiece independently, softmaxed
    # over the entire vocabulary.
    if (FLAGS.do_whole_word_mask and len(cand_indexes) >= 1 and
        token.startswith("##")):
      cand_indexes[-1].append(i)
    else:
      cand_indexes.append([i])

  rng.shuffle(cand_indexes)

  output_tokens = list(tokens)

  num_to_predict = min(max_predictions_per_seq,
                       max(1, int(round(len(tokens) * masked_lm_prob))))

  masked_lms = []
  covered_indexes = set()
  for index_set in cand_indexes:
    if len(masked_lms) >= num_to_predict:
      break
    # If adding a whole-word mask would exceed the maximum number of
    # predictions, then just skip this candidate.
    if len(masked_lms) + len(index_set) > num_to_predict:
      continue
    is_any_index_covered = False
    for index in index_set:
      if index in covered_indexes:
        is_any_index_covered = True
        break
    if is_any_index_covered:
      continue
    for index in index_set:
      covered_indexes.add(index)

      masked_token = None
      # 80% of the time, replace with [MASK]
      if rng.random() < 0.8:
        masked_token = "[MASK]"
      else:
        # 10% of the time, keep original
        if rng.random() < 0.5:
          masked_token = tokens[index]
        # 10% of the time, replace with random word
        else:
          masked_token = vocab_words[rng.randint(0, len(vocab_words) - 1)]

      output_tokens[index] = masked_token

      masked_lms.append(MaskedLmInstance(index=index, label=tokens[index]))
  assert len(masked_lms) <= num_to_predict
  masked_lms = sorted(masked_lms, key=lambda x: x.index)

  masked_lm_positions = []
  masked_lm_labels = []
  for p in masked_lms:
    masked_lm_positions.append(p.index)
    masked_lm_labels.append(p.label)

  return (output_tokens, masked_lm_positions, masked_lm_labels)
def create_masked_lm_predictions_original ( tokens , masked_lm_prob , max_predictions_per_seq , vocab_words , rng ) : cand_indexes = [ ] for ( i , token ) in enumerate ( tokens ) : if token == "[CLS]" or token == "[SEP]" : continue # Whole Word Masking means that if we mask all of the wordpieces # corresponding to an original word. When a word has been split into # WordPieces, the first token does not have any marker and any subsequence # tokens are prefixed with ##. So whenever we see the ## token, we # append it to the previous set of word indexes. # # Note that Whole Word Masking does *not* change the training code # at all -- we still predict each WordPiece independently, softmaxed # over the entire vocabulary. if ( FLAGS . do_whole_word_mask and len ( cand_indexes ) >= 1 and token . startswith ( "##" ) ) : cand_indexes [ - 1 ] . append ( i ) else : cand_indexes . append ( [ i ] ) rng . shuffle ( cand_indexes ) output_tokens = list ( tokens ) num_to_predict = min ( max_predictions_per_seq , max ( 1 , int ( round ( len ( tokens ) * masked_lm_prob ) ) ) ) masked_lms = [ ] covered_indexes = set ( ) for index_set in cand_indexes : if len ( masked_lms ) >= num_to_predict : break # If adding a whole-word mask would exceed the maximum number of # predictions, then just skip this candidate. if len ( masked_lms ) + len ( index_set ) > num_to_predict : continue is_any_index_covered = False for index in index_set : if index in covered_indexes : is_any_index_covered = True break if is_any_index_covered : continue for index in index_set : covered_indexes . add ( index ) masked_token = None # 80% of the time, replace with [MASK] if rng . random ( ) < 0.8 : masked_token = "[MASK]" else : # 10% of the time, keep original if rng . random ( ) < 0.5 : masked_token = tokens [ index ] # 10% of the time, replace with random word else : masked_token = vocab_words [ rng . randint ( 0 , len ( vocab_words ) - 1 ) ] output_tokens [ index ] = masked_token masked_lms . append ( MaskedLmInstance ( index = index , label = tokens [ index ] ) ) assert len ( masked_lms ) <= num_to_predict masked_lms = sorted ( masked_lms , key = lambda x : x . index ) masked_lm_positions = [ ] masked_lm_labels = [ ] for p in masked_lms : masked_lm_positions . append ( p . index ) masked_lm_labels . append ( p . label ) return ( output_tokens , masked_lm_positions , masked_lm_labels )