inpaint_zoom/app/zoom_out_app.py

from diffusers import DiffusionPipeline, DPMSolverMultistepScheduler
from inpaint_zoom.utils.zoom_out_utils import preprocess_image, preprocess_mask_image, write_video, dummy
from PIL import Image
import gradio as gr
import torch
import os
os.environ["CUDA_VISIBLE_DEVICES"]="0"


stable_paint_model_list = [
  "stabilityai/stable-diffusion-2-inpainting", 
  "runwayml/stable-diffusion-inpainting"
]

stable_paint_prompt_list = [
        "children running in the forest , sunny, bright, by studio ghibli painting, superior quality, masterpiece,  traditional Japanese colors, by Grzegorz Rutkowski, concept art",
        "A beautiful landscape of a mountain range with a lake in the foreground",
]

stable_paint_negative_prompt_list = [
        "lurry, bad art, blurred, text, watermark",
    ]


def stable_diffusion_zoom_out(
  model_id,
  original_prompt,
  negative_prompt,
  guidance_scale,
  num_inference_steps,
  step_size,
  num_frames,
    ):
    
    pipe = DiffusionPipeline.from_pretrained(model_id, torch_dtype=torch.float16)
    pipe.set_use_memory_efficient_attention_xformers(True)
    pipe.scheduler = DPMSolverMultistepScheduler.from_config(pipe.scheduler.config)
    pipe = pipe.to("cuda")
    pipe.safety_checker = dummy

    new_image = Image.new(mode="RGBA", size=(512,512)) 
    current_image, mask_image = preprocess_mask_image(new_image)
    
    current_image = pipe(
      prompt=[original_prompt], 
      negative_prompt=[negative_prompt], 
      image=current_image, 
      mask_image=mask_image, 
      num_inference_steps=num_inference_steps,
      guidance_scale=guidance_scale
    ).images[0]

    
    all_frames = []
    all_frames.append(current_image)

    for i in range(num_frames):
        prev_image = preprocess_image(current_image, step_size, 512)
        current_image = prev_image
        current_image, mask_image = preprocess_mask_image(current_image)
        current_image = pipe(prompt=[original_prompt], negative_prompt=[negative_prompt], image=current_image, mask_image=mask_image, num_inference_steps=num_inference_steps).images[0]
        
        current_image.paste(prev_image, mask=prev_image)
        all_frames.append(current_image)

    save_path = "output.mp4"  
    write_video(save_path, all_frames, fps=30)
    return save_path
  
  
def stable_diffusion_zoom_out_app():
    with gr.Blocks():
        with gr.Row():
            with gr.Column():
                text2image_out_model_path = gr.Dropdown(
                    choices=stable_paint_model_list, 
                    value=stable_paint_model_list[0], 
                    label='Text-Image Model Id'
                )

                text2image_out_prompt = gr.Textbox(
                    lines=2, 
                    value=stable_paint_prompt_list[0], 
                    label='Prompt'
                )

                text2image_out_negative_prompt = gr.Textbox(
                    lines=1, 
                    value=stable_paint_negative_prompt_list[0], 
                    label='Negative Prompt'
                )
                
                with gr.Row():
                    with gr.Column():
                        text2image_out_guidance_scale = gr.Slider(
                            minimum=0.1, 
                            maximum=15, 
                            step=0.1, 
                            value=7.5, 
                            label='Guidance Scale'
                        )

                        text2image_out_num_inference_step = gr.Slider(
                            minimum=1, 
                            maximum=100, 
                            step=1, 
                            value=50, 
                            label='Num Inference Step'
                        )
                    with gr.Row():
                        with gr.Column():
                            text2image_out_step_size = gr.Slider(
                                minimum=1,
                                maximum=100,
                                step=1,
                                value=10,
                                label='Step Size'
                            )
                            
                            text2image_out_num_frames = gr.Slider(
                                minimum=1,
                                maximum=100,
                                step=1,
                                value=10,
                                label='Frames'
                            )

                text2image_out_predict = gr.Button(value='Generator')
        
            with gr.Column():
                output_image = gr.Video(label="Output Video")
                    
        
        text2image_out_predict.click(
            fn=stable_diffusion_zoom_out,
            inputs=[
                text2image_out_model_path,
                text2image_out_prompt,
                text2image_out_negative_prompt,
                text2image_out_guidance_scale,
                text2image_out_num_inference_step,
                text2image_out_step_size,
                text2image_out_num_frames,
            ],
            outputs=output_image
        )