

from transformers import T5Tokenizer, T5ForConditionalGeneration
tokenizer = T5Tokenizer.from_pretrained("google-t5/t5-small")
model = T5ForConditionalGeneration.from_pretrained("google-t5/t5-small")
task_prefix = "translate English to German: "
use different length sentences to test batching
sentences = ["The house is wonderful.", "I like to work in NYC."]
inputs = tokenizer([task_prefix + sentence for sentence in sentences], return_tensors="pt", padding=True)
output_sequences = model.generate(
     input_ids=inputs["input_ids"],
     attention_mask=inputs["attention_mask"],
     do_sample=False,  # disable sampling to test if batching affects output
 )
print(tokenizer.batch_decode(output_sequences, skip_special_tokens=True))
['Das Haus ist wunderbar.', 'Ich arbeite gerne in NYC.']