

from PIL import Image
import requests
from transformers import AutoProcessor, Kosmos2ForConditionalGeneration
model = Kosmos2ForConditionalGeneration.from_pretrained("microsoft/kosmos-2-patch14-224")
processor = AutoProcessor.from_pretrained("microsoft/kosmos-2-patch14-224")
url = "https://huggingface.co/microsoft/kosmos-2-patch14-224/resolve/main/snowman.jpg"
image = Image.open(requests.get(url, stream=True).raw)
prompt = " An image of"
inputs = processor(text=prompt, images=image, return_tensors="pt")
generated_ids = model.generate(
     pixel_values=inputs["pixel_values"],
     input_ids=inputs["input_ids"],
     attention_mask=inputs["attention_mask"],
     image_embeds=None,
     image_embeds_position_mask=inputs["image_embeds_position_mask"],
     use_cache=True,
     max_new_tokens=64,
 )
generated_text = processor.batch_decode(generated_ids, skip_special_tokens=True)[0]
processed_text = processor.post_process_generation(generated_text, cleanup_and_extract=False)
processed_text
' An image of a snowman warming himself by a fire.'
caption, entities = processor.post_process_generation(generated_text)
caption
'An image of a snowman warming himself by a fire.'
entities
[('a snowman', (12, 21), [(0.390625, 0.046875, 0.984375, 0.828125)]), ('a fire', (41, 47), [(0.171875, 0.015625, 0.484375, 0.890625)])]