(inputs, tokenizer, args)
return inputs, labels
Prepare masked tokens inputs/labels for masked language modeling: 80% MASK, 10% random, 10% original.
Prepare masked tokens inputs/labels for masked language modeling: 80% MASK, 10% random, 10% original.
Prepare masked tokens inputs / labels for masked language modeling : 80% MASK 10% random 10% original .
def mask_tokens(inputs, tokenizer, args):
    """ Prepare masked tokens inputs/labels for masked language modeling: 80% MASK, 10% random, 10% original. """
    labels = inputs.clone()
    # We sample a few tokens in each sequence for masked-LM training (with probability args.mlm_probability defaults to 0.15 in Bert/RoBERTa)
    probability_matrix = torch.full(labels.shape, args.mlm_probability)
    special_tokens_mask = [tokenizer.get_special_tokens_mask(val, already_has_special_tokens=True) for val in labels.tolist()]
    probability_matrix.masked_fill_(torch.tensor(special_tokens_mask, dtype=torch.bool), value=0.0)
    masked_indices = torch.bernoulli(probability_matrix).bool()
    labels[~masked_indices] = -1  # We only compute loss on masked tokens

    # 80% of the time, we replace masked input tokens with tokenizer.mask_token ([MASK])
    indices_replaced = torch.bernoulli(torch.full(labels.shape, 0.8)).bool() & masked_indices
    inputs[indices_replaced] = tokenizer.convert_tokens_to_ids(tokenizer.mask_token)

    # 10% of the time, we replace masked input tokens with random word
    indices_random = torch.bernoulli(torch.full(labels.shape, 0.5)).bool() & masked_indices & ~indices_replaced
    random_words = torch.randint(len(tokenizer), labels.shape, dtype=torch.long)
    inputs[indices_random] = random_words[indices_random]

    # The rest of the time (10% of the time) we keep the masked input tokens unchanged
    return inputs, labels
def mask_tokens ( inputs , tokenizer , args ) : labels = inputs . clone ( ) # We sample a few tokens in each sequence for masked-LM training (with probability args.mlm_probability defaults to 0.15 in Bert/RoBERTa) probability_matrix = torch . full ( labels . shape , args . mlm_probability ) special_tokens_mask = [ tokenizer . get_special_tokens_mask ( val , already_has_special_tokens = True ) for val in labels . tolist ( ) ] probability_matrix . masked_fill_ ( torch . tensor ( special_tokens_mask , dtype = torch . bool ) , value = 0.0 ) masked_indices = torch . bernoulli ( probability_matrix ) . bool ( ) labels [ ~ masked_indices ] = - 1 # We only compute loss on masked tokens # 80% of the time, we replace masked input tokens with tokenizer.mask_token ([MASK]) indices_replaced = torch . bernoulli ( torch . full ( labels . shape , 0.8 ) ) . bool ( ) & masked_indices inputs [ indices_replaced ] = tokenizer . convert_tokens_to_ids ( tokenizer . mask_token ) # 10% of the time, we replace masked input tokens with random word indices_random = torch . bernoulli ( torch . full ( labels . shape , 0.5 ) ) . bool ( ) & masked_indices & ~ indices_replaced random_words = torch . randint ( len ( tokenizer ) , labels . shape , dtype = torch . long ) inputs [ indices_random ] = random_words [ indices_random ] # The rest of the time (10% of the time) we keep the masked input tokens unchanged return inputs , labels