hf_demo_test.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "initial_id",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-12-09T09:44:30.641366Z",
     "start_time": "2024-12-09T09:44:11.789050Z"
    }
   },
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "import gradio as gr\n",
    "from diffusers import DiffusionPipeline\n",
    "import matplotlib.pyplot as plt\n",
    "import torch\n",
    "from PIL import Image\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "ddf33e0d3abacc2c",
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "#append current path\n",
    "sys.path.extend(\"/afs/csail.mit.edu/u/h/huiren/code/diffusion/stable_diffusion/release/hf_demo\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "643e49fd601daf8f",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-12-09T09:44:35.790962Z",
     "start_time": "2024-12-09T09:44:35.779496Z"
    }
   },
   "outputs": [],
   "source": [
    "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"1\"\n",
    "device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n",
    "dtype=torch.bfloat16 if torch.cuda.is_available() else torch.float16"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "e03aae2a4e5676dd",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-12-09T09:44:44.157412Z",
     "start_time": "2024-12-09T09:44:37.138452Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/data/vision/torralba/selfmanaged/torralba/scratch/jomat/sam_dataset/miniforge3/envs/diffusion/lib/python3.9/site-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "21523c1459824498a59b1d237927a48e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading pipeline components...:   0%|          | 0/7 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "pipe = DiffusionPipeline.from_pretrained(\"rhfeiyang/art-free-diffusion-v1\",\n",
    "                                         torch_dtype=dtype).to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "83916bc68ff5d914",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-12-09T09:44:52.694399Z",
     "start_time": "2024-12-09T09:44:44.210695Z"
    }
   },
   "outputs": [],
   "source": [
    "from inference import get_lora_network, inference, get_validation_dataloader\n",
    "lora_map = {\n",
    "    \"None\": \"None\",\n",
    "    \"Andre Derain (fauvism)\": \"andre-derain_subset1\",\n",
    "    \"Vincent van Gogh (post impressionism)\": \"van_gogh_subset1\",\n",
    "    \"Andy Warhol (pop art)\": \"andy_subset1\",\n",
    "    \"Walter Battiss\": \"walter-battiss_subset2\",\n",
    "    \"Camille Corot (realism)\": \"camille-corot_subset1\",\n",
    "    \"Claude Monet (impressionism)\": \"monet_subset2\",\n",
    "    \"Pablo Picasso (cubism)\": \"picasso_subset1\",\n",
    "    \"Jackson Pollock\": \"jackson-pollock_subset1\",\n",
    "    \"Gerhard Richter (abstract expressionism)\": \"gerhard-richter_subset1\",\n",
    "    \"M.C. Escher\": \"m.c.-escher_subset1\",\n",
    "    \"Albert Gleizes\": \"albert-gleizes_subset1\",\n",
    "    \"Hokusai (ukiyo-e)\": \"katsushika-hokusai_subset1\",\n",
    "    \"Wassily Kandinsky\": \"kandinsky_subset1\",\n",
    "    \"Gustav Klimt (art nouveau)\": \"klimt_subset3\",\n",
    "    \"Roy Lichtenstein\": \"roy-lichtenstein_subset1\",\n",
    "    \"Henri Matisse (abstract expressionism)\": \"henri-matisse_subset1\",\n",
    "    \"Joan Miro\": \"joan-miro_subset2\",\n",
    "}\n",
    "\n",
    "\n",
    "\n",
    "def demo_inference_gen_artistic(adapter_choice:str, prompt:str, seed:int=0, steps=50, guidance_scale=7.5, adapter_scale=1.0):\n",
    "    adapter_path = lora_map[adapter_choice]\n",
    "    if adapter_path not in [None, \"None\"]:\n",
    "        adapter_path = f\"data/Art_adapters/{adapter_path}/adapter_alpha1.0_rank1_all_up_1000steps.pt\"\n",
    "        style_prompt=\"sks art\"\n",
    "    else:\n",
    "        style_prompt=None\n",
    "    prompts = [prompt]\n",
    "    infer_loader = get_validation_dataloader(prompts,num_workers=0)\n",
    "    network = get_lora_network(pipe.unet, adapter_path, weight_dtype=dtype, device=device)[\"network\"]\n",
    "\n",
    "    pred_images = inference(network, pipe.tokenizer, pipe.text_encoder, pipe.vae, pipe.unet, pipe.scheduler, infer_loader,\n",
    "                            height=512, width=512, scales=[adapter_scale],\n",
    "                            save_dir=None, seed=seed,steps=steps, guidance_scale=guidance_scale,\n",
    "                            start_noise=-1, show=False, style_prompt=style_prompt, no_load=True,\n",
    "                            from_scratch=True, device=device, weight_dtype=dtype)[0][1.0][0]\n",
    "    return pred_images\n",
    "\n",
    "\n",
    "def demo_inference_gen_ori( prompt:str, seed:int=0, steps=50, guidance_scale=7.5):\n",
    "    style_prompt=None\n",
    "    prompts = [prompt]\n",
    "    infer_loader = get_validation_dataloader(prompts,num_workers=0)\n",
    "    network = get_lora_network(pipe.unet, \"None\", weight_dtype=dtype)[\"network\"]\n",
    "\n",
    "    pred_images = inference(network, pipe.tokenizer, pipe.text_encoder, pipe.vae, pipe.unet, pipe.scheduler, infer_loader,\n",
    "                            height=512, width=512, scales=[0.0],\n",
    "                            save_dir=None, seed=seed,steps=steps, guidance_scale=guidance_scale,\n",
    "                            start_noise=-1, show=False, style_prompt=style_prompt, no_load=True,\n",
    "                            from_scratch=True, device=device, weight_dtype=dtype)[0][0.0][0]\n",
    "    return pred_images\n",
    "\n",
    "\n",
    "\n",
    "def demo_inference_stylization_ori(ref_image, prompt:str, seed:int=0, steps=50, guidance_scale=7.5, start_noise=800):\n",
    "    style_prompt=None\n",
    "    prompts = [prompt]\n",
    "    # convert np to pil\n",
    "    ref_image = [Image.fromarray(ref_image)]\n",
    "    network = get_lora_network(pipe.unet, \"None\", weight_dtype=dtype)[\"network\"]\n",
    "    infer_loader = get_validation_dataloader(prompts, ref_image,num_workers=0)\n",
    "    pred_images = inference(network, pipe.tokenizer, pipe.text_encoder, pipe.vae, pipe.unet, pipe.scheduler, infer_loader,\n",
    "                            height=512, width=512, scales=[0.0],\n",
    "                            save_dir=None, seed=seed,steps=steps, guidance_scale=guidance_scale,\n",
    "                            start_noise=start_noise, show=False, style_prompt=style_prompt, no_load=True,\n",
    "                            from_scratch=False, device=device, weight_dtype=dtype)[0][0.0][0]\n",
    "    return pred_images\n",
    "\n",
    "\n",
    "def demo_inference_stylization_artistic(ref_image, adapter_choice:str, prompt:str, seed:int=0, steps=50, guidance_scale=7.5, adapter_scale=1.0,start_noise=800):\n",
    "    adapter_path = lora_map[adapter_choice]\n",
    "    if adapter_path not in [None, \"None\"]:\n",
    "        adapter_path = f\"data/Art_adapters/{adapter_path}/adapter_alpha1.0_rank1_all_up_1000steps.pt\"\n",
    "        style_prompt=\"sks art\"\n",
    "    else:\n",
    "        style_prompt=None\n",
    "    prompts = [prompt]\n",
    "    # convert np to pil\n",
    "    ref_image = [Image.fromarray(ref_image)]\n",
    "    network = get_lora_network(pipe.unet, adapter_path, weight_dtype=dtype)[\"network\"]\n",
    "    infer_loader = get_validation_dataloader(prompts, ref_image,num_workers=0)\n",
    "    pred_images = inference(network, pipe.tokenizer, pipe.text_encoder, pipe.vae, pipe.unet, pipe.scheduler, infer_loader,\n",
    "                            height=512, width=512, scales=[adapter_scale],\n",
    "                            save_dir=None, seed=seed,steps=steps, guidance_scale=guidance_scale,\n",
    "                            start_noise=start_noise, show=False, style_prompt=style_prompt, no_load=True,\n",
    "                            from_scratch=False, device=device, weight_dtype=dtype)[0][1.0][0]\n",
    "    return pred_images\n",
    "\n",
    "\n",
    "def demo_inference_all(prompt:str, ref_image, adapter_choice=\"Andre Derain (fauvism)\", seed:int=0, steps=20, guidance_scale=7.5, adapter_scale=1.0,start_noise=800):\n",
    "    results = []\n",
    "    results.append(demo_inference_gen_ori(prompt, seed, steps, guidance_scale))\n",
    "    results.append(demo_inference_gen_artistic(adapter_choice, prompt, seed, steps, guidance_scale, adapter_scale))\n",
    "    results.append(demo_inference_stylization_ori(ref_image, prompt, seed, steps, guidance_scale, start_noise))\n",
    "    results.append(demo_inference_stylization_artistic(ref_image, adapter_choice, prompt, seed, steps, guidance_scale, adapter_scale, start_noise))\n",
    "    return results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "aa33e9d104023847",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-12-10T02:56:13.419303Z",
     "start_time": "2024-12-10T02:56:13.002796Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/data/vision/torralba/selfmanaged/torralba/scratch/jomat/sam_dataset/miniforge3/envs/diffusion/lib/python3.9/site-packages/gradio/helpers.py:370: UserWarning: Examples will be cached but not all input components have example values. This may result in an exception being thrown by your function. If you do get an error while caching examples, make sure all of your inputs have example values for all of your examples or you provide default values for those particular parameters in your function.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Caching examples at: '/data/vision/torralba/selfmanaged/torralba/projects/jomat/hui/stable_diffusion/release/hf_demo/gradio_cached_examples/589'\n",
      "Caching example 1/4\n",
      "Train method: None\n",
      "Rank: 1, Alpha: 1\n",
      "create LoRA for U-Net: 0 modules.\n",
      "current time: 2024-12-14 01:19:01\n",
      "save dir: None\n",
      "['Snow-covered trees with sunlight shining through'], seed=0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████| 21/21 [00:02<00:00,  8.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Time taken for one batch, Art Adapter scale=0.0: 2.58335280418396\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train method: all_up\n",
      "Rank: 1, Alpha: 1.0\n",
      "create LoRA for U-Net: 123 modules.\n",
      "Missing: <All keys matched successfully>\n",
      "current time: 2024-12-14 01:19:04\n",
      "save dir: None\n",
      "['Snow-covered trees with sunlight shining through in the style of sks art'], seed=0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████| 21/21 [00:02<00:00,  8.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Time taken for one batch, Art Adapter scale=1.0: 2.6419241428375244\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train method: None\n",
      "Rank: 1, Alpha: 1\n",
      "create LoRA for U-Net: 0 modules.\n",
      "current time: 2024-12-14 01:19:07\n",
      "save dir: None\n",
      "['Snow-covered trees with sunlight shining through'], seed=0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████| 21/21 [00:01<00:00, 10.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Time taken for one batch, Art Adapter scale=0.0: 2.0403149127960205\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train method: all_up\n",
      "Rank: 1, Alpha: 1.0\n",
      "create LoRA for U-Net: 123 modules.\n",
      "Missing: <All keys matched successfully>\n",
      "current time: 2024-12-14 01:19:09\n",
      "save dir: None\n",
      "['Snow-covered trees with sunlight shining through in the style of sks art'], seed=0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "00%|██████████████████████████████████████████████████████████████████████████████████████████████████| 21/21 [00:02<00:00,  9.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Time taken for one batch, Art Adapter scale=1.0: 2.1876306533813477\n",
      "Caching example 2/4\n",
      "Train method: None\n",
      "Rank: 1, Alpha: 1\n",
      "create LoRA for U-Net: 0 modules.\n",
      "current time: 2024-12-14 01:19:12\n",
      "save dir: None\n",
      "[\"A picturesque landscape showcasing a winding river cutting through a lush green valley, surrounded by rugged mountains under a clear blue sky. The mix of red and brown tones in the rocky hills adds to the region's natural beauty and diversity.\"], seed=0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "00%|██████████████████████████████████████████████████████████████████████████████████████████████████| 21/21 [00:02<00:00,  7.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Time taken for one batch, Art Adapter scale=0.0: 2.8535258769989014\n",
      "Train method: all_up\n",
      "Rank: 1, Alpha: 1.0\n",
      "create LoRA for U-Net: 123 modules.\n",
      "Missing: <All keys matched successfully>\n",
      "current time: 2024-12-14 01:19:15\n",
      "save dir: None\n",
      "[\"A picturesque landscape showcasing a winding river cutting through a lush green valley, surrounded by rugged mountains under a clear blue sky. The mix of red and brown tones in the rocky hills adds to the region's natural beauty and diversity in the style of sks art\"], seed=0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "00%|██████████████████████████████████████████████████████████████████████████████████████████████████| 21/21 [00:02<00:00,  7.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Time taken for one batch, Art Adapter scale=1.0: 3.0402302742004395\n",
      "Train method: None\n",
      "Rank: 1, Alpha: 1\n",
      "create LoRA for U-Net: 0 modules.\n",
      "current time: 2024-12-14 01:19:18\n",
      "save dir: None\n",
      "[\"A picturesque landscape showcasing a winding river cutting through a lush green valley, surrounded by rugged mountains under a clear blue sky. The mix of red and brown tones in the rocky hills adds to the region's natural beauty and diversity.\"], seed=0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "00%|██████████████████████████████████████████████████████████████████████████████████████████████████| 21/21 [00:02<00:00,  9.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Time taken for one batch, Art Adapter scale=0.0: 2.33781099319458\n",
      "Train method: all_up\n",
      "Rank: 1, Alpha: 1.0\n",
      "create LoRA for U-Net: 123 modules.\n",
      "Missing: <All keys matched successfully>\n",
      "current time: 2024-12-14 01:19:21\n",
      "save dir: None\n",
      "[\"A picturesque landscape showcasing a winding river cutting through a lush green valley, surrounded by rugged mountains under a clear blue sky. The mix of red and brown tones in the rocky hills adds to the region's natural beauty and diversity in the style of sks art\"], seed=0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "00%|██████████████████████████████████████████████████████████████████████████████████████████████████| 21/21 [00:02<00:00,  8.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Time taken for one batch, Art Adapter scale=1.0: 2.490582227706909\n",
      "Caching example 3/4\n",
      "Train method: None\n",
      "Rank: 1, Alpha: 1\n",
      "create LoRA for U-Net: 0 modules.\n",
      "current time: 2024-12-14 01:19:24\n",
      "save dir: None\n",
      "['a black SUV driving down a highway with a scenic view of mountains and water in the background. The SUV is the main focus of the image, and it appears to be traveling at a moderate speed. The road is well-maintained and provides a smooth driving experience. The mountains and water create a picturesque backdrop, adding to the overall beauty of the scene. The image captures the essence of a leisurely road trip, with the SUV as the primary subject, highlighting the sense of adventure and exploration that comes with such journeys.'], seed=0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "00%|██████████████████████████████████████████████████████████████████████████████████████████████████| 21/21 [00:03<00:00,  6.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Time taken for one batch, Art Adapter scale=0.0: 3.2783615589141846\n",
      "Train method: all_up\n",
      "Rank: 1, Alpha: 1.0\n",
      "create LoRA for U-Net: 123 modules.\n",
      "Missing: <All keys matched successfully>\n",
      "current time: 2024-12-14 01:19:27\n",
      "save dir: None\n",
      "['a black SUV driving down a highway with a scenic view of mountains and water in the background. The SUV is the main focus of the image, and it appears to be traveling at a moderate speed. The road is well-maintained and provides a smooth driving experience. The mountains and water create a picturesque backdrop, adding to the overall beauty of the scene. The image captures the essence of a leisurely road trip, with the SUV as the primary subject, highlighting the sense of adventure and exploration that comes with such journeys in the style of sks art'], seed=0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "00%|██████████████████████████████████████████████████████████████████████████████████████████████████| 21/21 [00:03<00:00,  6.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Time taken for one batch, Art Adapter scale=1.0: 3.4552266597747803\n",
      "Train method: None\n",
      "Rank: 1, Alpha: 1\n",
      "create LoRA for U-Net: 0 modules.\n",
      "current time: 2024-12-14 01:19:31\n",
      "save dir: None\n",
      "['a black SUV driving down a highway with a scenic view of mountains and water in the background. The SUV is the main focus of the image, and it appears to be traveling at a moderate speed. The road is well-maintained and provides a smooth driving experience. The mountains and water create a picturesque backdrop, adding to the overall beauty of the scene. The image captures the essence of a leisurely road trip, with the SUV as the primary subject, highlighting the sense of adventure and exploration that comes with such journeys.'], seed=0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "00%|██████████████████████████████████████████████████████████████████████████████████████████████████| 21/21 [00:02<00:00,  8.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Time taken for one batch, Art Adapter scale=0.0: 2.6474156379699707\n",
      "Train method: all_up\n",
      "Rank: 1, Alpha: 1.0\n",
      "create LoRA for U-Net: 123 modules.\n",
      "Missing: <All keys matched successfully>\n",
      "current time: 2024-12-14 01:19:34\n",
      "save dir: None\n",
      "['a black SUV driving down a highway with a scenic view of mountains and water in the background. The SUV is the main focus of the image, and it appears to be traveling at a moderate speed. The road is well-maintained and provides a smooth driving experience. The mountains and water create a picturesque backdrop, adding to the overall beauty of the scene. The image captures the essence of a leisurely road trip, with the SUV as the primary subject, highlighting the sense of adventure and exploration that comes with such journeys in the style of sks art'], seed=0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "00%|██████████████████████████████████████████████████████████████████████████████████████████████████| 21/21 [00:02<00:00,  7.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Time taken for one batch, Art Adapter scale=1.0: 2.803696870803833\n",
      "Caching example 4/4\n",
      "Train method: None\n",
      "Rank: 1, Alpha: 1\n",
      "create LoRA for U-Net: 0 modules.\n",
      "current time: 2024-12-14 01:19:37\n",
      "save dir: None\n",
      "[\"A blue bench situated in a park, surrounded by trees and leaves. The bench is positioned under a tree, providing shade and a peaceful atmosphere. There are several benches in the park, with one being closer to the foreground and the others further in the background. A person can be seen in the distance, possibly enjoying the park or taking a walk. The overall scene is serene and inviting, with the bench serving as a focal point in the park's landscape.\"], seed=0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "00%|██████████████████████████████████████████████████████████████████████████████████████████████████| 21/21 [00:03<00:00,  5.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Time taken for one batch, Art Adapter scale=0.0: 3.648933172225952\n",
      "Train method: all_up\n",
      "Rank: 1, Alpha: 1.0\n",
      "create LoRA for U-Net: 123 modules.\n",
      "Missing: <All keys matched successfully>\n",
      "current time: 2024-12-14 01:19:41\n",
      "save dir: None\n",
      "[\"A blue bench situated in a park, surrounded by trees and leaves. The bench is positioned under a tree, providing shade and a peaceful atmosphere. There are several benches in the park, with one being closer to the foreground and the others further in the background. A person can be seen in the distance, possibly enjoying the park or taking a walk. The overall scene is serene and inviting, with the bench serving as a focal point in the park's landscape in the style of sks art\"], seed=0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "00%|██████████████████████████████████████████████████████████████████████████████████████████████████| 21/21 [00:03<00:00,  5.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Time taken for one batch, Art Adapter scale=1.0: 3.8492958545684814\n",
      "Train method: None\n",
      "Rank: 1, Alpha: 1\n",
      "create LoRA for U-Net: 0 modules.\n",
      "current time: 2024-12-14 01:19:45\n",
      "save dir: None\n",
      "[\"A blue bench situated in a park, surrounded by trees and leaves. The bench is positioned under a tree, providing shade and a peaceful atmosphere. There are several benches in the park, with one being closer to the foreground and the others further in the background. A person can be seen in the distance, possibly enjoying the park or taking a walk. The overall scene is serene and inviting, with the bench serving as a focal point in the park's landscape.\"], seed=0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "00%|██████████████████████████████████████████████████████████████████████████████████████████████████| 21/21 [00:02<00:00,  7.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Time taken for one batch, Art Adapter scale=0.0: 2.9497482776641846\n",
      "Train method: all_up\n",
      "Rank: 1, Alpha: 1.0\n",
      "create LoRA for U-Net: 123 modules.\n",
      "Missing: <All keys matched successfully>\n",
      "current time: 2024-12-14 01:19:49\n",
      "save dir: None\n",
      "[\"A blue bench situated in a park, surrounded by trees and leaves. The bench is positioned under a tree, providing shade and a peaceful atmosphere. There are several benches in the park, with one being closer to the foreground and the others further in the background. A person can be seen in the distance, possibly enjoying the park or taking a walk. The overall scene is serene and inviting, with the bench serving as a focal point in the park's landscape in the style of sks art\"], seed=0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "00%|██████████████████████████████████████████████████████████████████████████████████████████████████| 21/21 [00:03<00:00,  6.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Time taken for one batch, Art Adapter scale=1.0: 3.107145309448242\n",
      "Running on local URL:  http://127.0.0.1:7865\n",
      "Running on public URL: https://26432817d4a57fa05b.gradio.live\n",
      "\n",
      "This share link expires in 72 hours. For free permanent hosting and GPU upgrades, run `gradio deploy` from Terminal to deploy to Spaces (https://huggingface.co/spaces)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div><iframe src=\"https://26432817d4a57fa05b.gradio.live\" width=\"100%\" height=\"500\" allow=\"autoplay; camera; microphone; clipboard-read; clipboard-write;\" frameborder=\"0\" allowfullscreen></iframe></div>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": []
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "block = gr.Blocks()\n",
    "# Direct infer\n",
    "# Direct infer\n",
    "with block:\n",
    "    with gr.Group():\n",
    "        gr.Markdown(\" # Art-Free Diffusion Demo\")\n",
    "        gr.Markdown(\"(More features in development...)\")\n",
    "        with gr.Row():\n",
    "            text = gr.Textbox(\n",
    "                label=\"Prompt (long and detailed would be better):\",\n",
    "                max_lines=10,\n",
    "                placeholder=\"Enter your prompt (long and detailed would be better)\",\n",
    "                container=True,\n",
    "                value=\"A blue bench situated in a park, surrounded by trees and leaves. The bench is positioned under a tree, providing shade and a peaceful atmosphere. There are several benches in the park, with one being closer to the foreground and the others further in the background. A person can be seen in the distance, possibly enjoying the park or taking a walk. The overall scene is serene and inviting, with the bench serving as a focal point in the park's landscape.\",\n",
    "            )\n",
    "\n",
    "        with gr.Tab('Generation'):\n",
    "            with gr.Row():\n",
    "                with gr.Column():\n",
    "                    # gr.Markdown(\"## Art-Free Generation\")\n",
    "                    # gr.Markdown(\"Generate images from text prompts.\")\n",
    "\n",
    "                    gallery_gen_ori = gr.Image(\n",
    "                        label=\"W/O Adapter\",\n",
    "                        show_label=True,\n",
    "                        elem_id=\"gallery\",\n",
    "                        height=\"auto\"\n",
    "                    )\n",
    "\n",
    "\n",
    "                with gr.Column():\n",
    "                    # gr.Markdown(\"## Art-Free Generation\")\n",
    "                    # gr.Markdown(\"Generate images from text prompts.\")\n",
    "                    gallery_gen_art = gr.Image(\n",
    "                        label=\"W/ Adapter\",\n",
    "                        show_label=True,\n",
    "                        elem_id=\"gallery\",\n",
    "                        height=\"auto\"\n",
    "                    )\n",
    "\n",
    "\n",
    "            with gr.Row():\n",
    "                btn_gen_ori = gr.Button(\"Art-Free Generate\", scale=1)\n",
    "                btn_gen_art = gr.Button(\"Artistic Generate\", scale=1)\n",
    "\n",
    "\n",
    "        with gr.Tab('Stylization'):\n",
    "            with gr.Row():\n",
    "\n",
    "                with gr.Column():\n",
    "                    # gr.Markdown(\"## Art-Free Generation\")\n",
    "                    # gr.Markdown(\"Generate images from text prompts.\")\n",
    "\n",
    "                    gallery_stylization_ref = gr.Image(\n",
    "                        label=\"Ref Image\",\n",
    "                        show_label=True,\n",
    "                        elem_id=\"gallery\",\n",
    "                        height=\"auto\",\n",
    "                        scale=1,\n",
    "                        value=\"data/003904765.jpg\"\n",
    "                    )\n",
    "                with gr.Column(scale=2):\n",
    "                    with gr.Row():\n",
    "                        with gr.Column():\n",
    "                            # gr.Markdown(\"## Art-Free Generation\")\n",
    "                            # gr.Markdown(\"Generate images from text prompts.\")\n",
    "\n",
    "                            gallery_stylization_ori = gr.Image(\n",
    "                                label=\"W/O Adapter\",\n",
    "                                show_label=True,\n",
    "                                elem_id=\"gallery\",\n",
    "                                height=\"auto\",\n",
    "                                scale=1,\n",
    "                            )\n",
    "\n",
    "\n",
    "                        with gr.Column():\n",
    "                            # gr.Markdown(\"## Art-Free Generation\")\n",
    "                            # gr.Markdown(\"Generate images from text prompts.\")\n",
    "                            gallery_stylization_art = gr.Image(\n",
    "                                label=\"W/ Adapter\",\n",
    "                                show_label=True,\n",
    "                                elem_id=\"gallery\",\n",
    "                                height=\"auto\",\n",
    "                                scale=1,\n",
    "                            )\n",
    "                    start_timestep = gr.Slider(label=\"Timestep start from:\", minimum=0, maximum=1000, value=800, step=1)\n",
    "            with gr.Row():\n",
    "                btn_style_ori = gr.Button(\"Art-Free Stylize\", scale=1)\n",
    "                btn_style_art = gr.Button(\"Artistic Stylize\", scale=1)\n",
    "\n",
    "\n",
    "        with gr.Row():\n",
    "            # with gr.Column():\n",
    "            # samples = gr.Slider(label=\"Images\", minimum=1, maximum=4, value=1, step=1, scale=1)\n",
    "            scale = gr.Slider(\n",
    "                label=\"Guidance Scale\", minimum=0, maximum=20, value=7.5, step=0.1\n",
    "            )\n",
    "            # with gr.Column():\n",
    "            adapter_choice = gr.Dropdown(\n",
    "                label=\"Select Art Adapter\",\n",
    "                choices=[ \"Andre Derain (fauvism)\",\"Vincent van Gogh (post impressionism)\",\"Andy Warhol (pop art)\",\n",
    "                          \"Camille Corot (realism)\", \"Claude Monet (impressionism)\", \"Pablo Picasso (cubism)\", \"Gerhard Richter (abstract expressionism)\",\n",
    "                          \"Hokusai (ukiyo-e)\", \"Gustav Klimt (art nouveau)\", \"Henri Matisse (abstract expressionism)\",\n",
    "                          \"Walter Battiss\", \"Jackson Pollock\",  \"M.C. Escher\", \"Albert Gleizes\",  \"Wassily Kandinsky\",\n",
    "                          \"Roy Lichtenstein\", \"Joan Miro\"\n",
    "                          ],\n",
    "                value=\"Andre Derain (fauvism)\",\n",
    "                scale=1\n",
    "            )\n",
    "\n",
    "        with gr.Row():\n",
    "            steps = gr.Slider(label=\"Steps\", minimum=1, maximum=50, value=20, step=1)\n",
    "            adapter_scale = gr.Slider(label=\"Adapter Scale\", minimum=0, maximum=1.5, value=1., step=0.1, scale=1)\n",
    "\n",
    "        with gr.Row():\n",
    "            seed = gr.Slider(label=\"Seed\",minimum=0,maximum=2147483647,step=1,randomize=True,scale=1)\n",
    "\n",
    "\n",
    "        gr.on([btn_gen_ori.click], demo_inference_gen_ori, inputs=[text, seed, steps, scale], outputs=gallery_gen_ori)\n",
    "        gr.on([btn_gen_art.click], demo_inference_gen_artistic, inputs=[adapter_choice, text, seed, steps, scale, adapter_scale], outputs=gallery_gen_art)\n",
    "\n",
    "        gr.on([btn_style_ori.click], demo_inference_stylization_ori, inputs=[gallery_stylization_ref, text, seed, steps, scale, start_timestep], outputs=gallery_stylization_ori)\n",
    "        gr.on([btn_style_art.click], demo_inference_stylization_artistic, inputs=[gallery_stylization_ref, adapter_choice, text, seed, steps, scale, adapter_scale, start_timestep], outputs=gallery_stylization_art)\n",
    "\n",
    "    examples = gr.Examples(\n",
    "        examples=[\n",
    "            [\"Snow-covered trees with sunlight shining through\",\n",
    "             \"data/Snow-covered_trees_with_sunlight_shining_through.jpg\",\n",
    "             ],\n",
    "            [\"A picturesque landscape showcasing a winding river cutting through a lush green valley, surrounded by rugged mountains under a clear blue sky. The mix of red and brown tones in the rocky hills adds to the region's natural beauty and diversity.\",\n",
    "             \"data/0011772.jpg\",\n",
    "            ],\n",
    "            [\"a black SUV driving down a highway with a scenic view of mountains and water in the background. The SUV is the main focus of the image, and it appears to be traveling at a moderate speed. The road is well-maintained and provides a smooth driving experience. The mountains and water create a picturesque backdrop, adding to the overall beauty of the scene. The image captures the essence of a leisurely road trip, with the SUV as the primary subject, highlighting the sense of adventure and exploration that comes with such journeys.\",\n",
    "             \"data/a_black_SUV_driving_down_a_highway_with_a_scenic_view_of_mountains_and_water_in_the_background._The_.jpg\",\n",
    "             ],\n",
    "            [\n",
    "                \"A blue bench situated in a park, surrounded by trees and leaves. The bench is positioned under a tree, providing shade and a peaceful atmosphere. There are several benches in the park, with one being closer to the foreground and the others further in the background. A person can be seen in the distance, possibly enjoying the park or taking a walk. The overall scene is serene and inviting, with the bench serving as a focal point in the park's landscape.\",\n",
    "                \"data/003904765.jpg\",\n",
    "            ]\n",
    "\n",
    "        ],\n",
    "        inputs=[\n",
    "            text,\n",
    "            gallery_stylization_ref,\n",
    "            adapter_choice,\n",
    "            seed,\n",
    "            steps,\n",
    "            scale,\n",
    "            adapter_scale,\n",
    "            start_timestep,            \n",
    "        ],\n",
    "        fn=demo_inference_all,\n",
    "        outputs=[gallery_gen_ori, gallery_gen_art, gallery_stylization_ori, gallery_stylization_art],\n",
    "        cache_examples=True,\n",
    "    )\n",
    "block.launch(share=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3239c12167a5f2cd",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.18"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}