(model, tokenizer)
Add special tokens to the tokenizer and the model if they have not already been added.
Add special tokens to the tokenizer and the model if they have not already been added.
Add special tokens to the tokenizer and the model if they have not already been added .
def add_special_tokens_(model, tokenizer):
    """ Add special tokens to the tokenizer and the model if they have not already been added. """
    orig_num_tokens = len(tokenizer.encoder)
    num_added_tokens = tokenizer.add_special_tokens(ATTR_TO_SPECIAL_TOKEN) # doesn't add if they are already there
    if num_added_tokens > 0:
        model.resize_token_embeddings(new_num_tokens=orig_num_tokens + num_added_tokens)
def add_special_tokens_ ( model , tokenizer ) : orig_num_tokens = len ( tokenizer . encoder ) num_added_tokens = tokenizer . add_special_tokens ( ATTR_TO_SPECIAL_TOKEN ) # doesn't add if they are already there if num_added_tokens > 0 : model . resize_token_embeddings ( new_num_tokens = orig_num_tokens + num_added_tokens )