gradio_app.py

import os
import random
import tempfile
import time
from contextlib import nullcontext
from functools import lru_cache
from typing import Any

import gradio as gr
import numpy as np
import torch
import trimesh
from gradio_litmodel3d import LitModel3D
from gradio_pointcloudeditor import PointCloudEditor
from PIL import Image
from transparent_background import Remover

os.system("USE_CUDA=1 pip install -vv --no-build-isolation ./texture_baker ./uv_unwrapper")
os.system("pip install ./deps/pynim-0.0.3-cp310-cp310-linux_x86_64.whl")

import spar3d.utils as spar3d_utils
from spar3d.models.mesh import QUAD_REMESH_AVAILABLE, TRIANGLE_REMESH_AVAILABLE
from spar3d.system import SPAR3D

os.environ["GRADIO_TEMP_DIR"] = os.path.join(os.environ.get("TMPDIR", "/tmp"), "gradio")

bg_remover = Remover()  # default setting

COND_WIDTH = 512
COND_HEIGHT = 512
COND_DISTANCE = 2.2
COND_FOVY = 0.591627
BACKGROUND_COLOR = [0.5, 0.5, 0.5]

# Cached. Doesn't change
c2w_cond = spar3d_utils.default_cond_c2w(COND_DISTANCE)
intrinsic, intrinsic_normed_cond = spar3d_utils.create_intrinsic_from_fov_rad(
    COND_FOVY, COND_HEIGHT, COND_WIDTH
)

generated_files = []

# Delete previous gradio temp dir folder
if os.path.exists(os.environ["GRADIO_TEMP_DIR"]):
    print(f"Deleting {os.environ['GRADIO_TEMP_DIR']}")
    import shutil

    shutil.rmtree(os.environ["GRADIO_TEMP_DIR"])

device = spar3d_utils.get_device()

model = SPAR3D.from_pretrained(
    "stabilityai/stable-point-aware-3d",
    config_name="config.yaml",
    weight_name="model.safetensors",
)
model.eval()
model = model.to(device)

example_files = [
    os.path.join("demo_files/examples", f) for f in os.listdir("demo_files/examples")
]


def forward_model(
    batch,
    system,
    guidance_scale=3.0,
    seed=0,
    device="cuda",
    remesh_option="none",
    vertex_count=-1,
    texture_resolution=1024,
):
    batch_size = batch["rgb_cond"].shape[0]

    # prepare the condition for point cloud generation
    # set seed
    random.seed(seed)
    torch.manual_seed(seed)
    np.random.seed(seed)
    cond_tokens = system.forward_pdiff_cond(batch)

    if "pc_cond" not in batch:
        sample_iter = system.sampler.sample_batch_progressive(
            batch_size,
            cond_tokens,
            guidance_scale=guidance_scale,
            device=device,
        )
        for x in sample_iter:
            samples = x["xstart"]
        batch["pc_cond"] = samples.permute(0, 2, 1).float()
        batch["pc_cond"] = spar3d_utils.normalize_pc_bbox(batch["pc_cond"])

    # subsample to the 512 points
    batch["pc_cond"] = batch["pc_cond"][
        :, torch.randperm(batch["pc_cond"].shape[1])[:512]
    ]

    # get the point cloud
    xyz = batch["pc_cond"][0, :, :3].cpu().numpy()
    color_rgb = (batch["pc_cond"][0, :, 3:6] * 255).cpu().numpy().astype(np.uint8)
    pc_rgb_trimesh = trimesh.PointCloud(vertices=xyz, colors=color_rgb)

    # forward for the final mesh
    trimesh_mesh, _glob_dict = model.generate_mesh(
        batch, texture_resolution, remesh=remesh_option, vertex_count=vertex_count
    )
    trimesh_mesh = trimesh_mesh[0]

    return trimesh_mesh, pc_rgb_trimesh


def run_model(
    input_image,
    guidance_scale,
    random_seed,
    pc_cond,
    remesh_option,
    vertex_count,
    texture_resolution,
):
    start = time.time()
    with torch.no_grad():
        with (
            torch.autocast(device_type=device, dtype=torch.float16)
            if "cuda" in device
            else nullcontext()
        ):
            model_batch = create_batch(input_image)
            model_batch = {k: v.to(device) for k, v in model_batch.items()}

            if pc_cond is not None:
                # Check if pc_cond is a list
                if isinstance(pc_cond, list):
                    cond_tensor = torch.tensor(pc_cond).float().cuda().view(-1, 6)
                    xyz = cond_tensor[:, :3]
                    color_rgb = cond_tensor[:, 3:]
                elif isinstance(pc_cond, dict):
                    xyz = torch.tensor(pc_cond["positions"]).float().cuda()
                    color_rgb = torch.tensor(pc_cond["colors"]).float().cuda()
                else:
                    xyz = torch.tensor(pc_cond.vertices).float().cuda()
                    color_rgb = (
                        torch.tensor(pc_cond.colors[:, :3]).float().cuda() / 255.0
                    )
                model_batch["pc_cond"] = torch.cat([xyz, color_rgb], dim=-1).unsqueeze(
                    0
                )
                # sub-sample the point cloud to the target number of points
                if model_batch["pc_cond"].shape[1] > 512:
                    idx = torch.randperm(model_batch["pc_cond"].shape[1])[:512]
                    model_batch["pc_cond"] = model_batch["pc_cond"][:, idx]
                elif model_batch["pc_cond"].shape[1] < 512:
                    num_points = model_batch["pc_cond"].shape[1]
                    gr.Warning(
                        f"The uploaded point cloud should have at least 512 points. This point cloud only has {num_points}. Results may be worse."
                    )
                    pad = 512 - num_points
                    sampled_idx = torch.randint(
                        0, model_batch["pc_cond"].shape[1], (pad,)
                    )
                    model_batch["pc_cond"] = torch.cat(
                        [
                            model_batch["pc_cond"],
                            model_batch["pc_cond"][:, sampled_idx],
                        ],
                        dim=1,
                    )

            trimesh_mesh, trimesh_pc = forward_model(
                model_batch,
                model,
                guidance_scale=guidance_scale,
                seed=random_seed,
                device="cuda",
                remesh_option=remesh_option.lower(),
                vertex_count=vertex_count,
                texture_resolution=texture_resolution,
            )

    # Create new tmp file
    temp_dir = tempfile.mkdtemp()
    tmp_file = os.path.join(temp_dir, "mesh.glb")

    trimesh_mesh.export(tmp_file, file_type="glb", include_normals=True)
    generated_files.append(tmp_file)

    tmp_file_pc = os.path.join(temp_dir, "points.ply")
    trimesh_pc.export(tmp_file_pc)
    generated_files.append(tmp_file_pc)

    print("Generation took:", time.time() - start, "s")

    return tmp_file, tmp_file_pc, trimesh_pc


def create_batch(input_image: Image) -> dict[str, Any]:
    img_cond = (
        torch.from_numpy(
            np.asarray(input_image.resize((COND_WIDTH, COND_HEIGHT))).astype(np.float32)
            / 255.0
        )
        .float()
        .clip(0, 1)
    )
    mask_cond = img_cond[:, :, -1:]
    rgb_cond = torch.lerp(
        torch.tensor(BACKGROUND_COLOR)[None, None, :], img_cond[:, :, :3], mask_cond
    )

    batch_elem = {
        "rgb_cond": rgb_cond,
        "mask_cond": mask_cond,
        "c2w_cond": c2w_cond.unsqueeze(0),
        "intrinsic_cond": intrinsic.unsqueeze(0),
        "intrinsic_normed_cond": intrinsic_normed_cond.unsqueeze(0),
    }
    # Add batch dim
    batched = {k: v.unsqueeze(0) for k, v in batch_elem.items()}
    return batched


@lru_cache
def checkerboard(squares: int, size: int, min_value: float = 0.5):
    base = np.zeros((squares, squares)) + min_value
    base[1::2, ::2] = 1
    base[::2, 1::2] = 1

    repeat_mult = size // squares
    return (
        base.repeat(repeat_mult, axis=0)
        .repeat(repeat_mult, axis=1)[:, :, None]
        .repeat(3, axis=-1)
    )


def remove_background(input_image: Image) -> Image:
    return bg_remover.process(input_image.convert("RGB"))


def show_mask_img(input_image: Image) -> Image:
    img_numpy = np.array(input_image)
    alpha = img_numpy[:, :, 3] / 255.0
    chkb = checkerboard(32, 512) * 255
    new_img = img_numpy[..., :3] * alpha[:, :, None] + chkb * (1 - alpha[:, :, None])
    return Image.fromarray(new_img.astype(np.uint8), mode="RGB")


def process_model_run(
    background_state,
    guidance_scale,
    random_seed,
    pc_cond,
    remesh_option,
    vertex_count_type,
    vertex_count,
    texture_resolution,
):
    # Adjust vertex count based on selection
    final_vertex_count = (
        -1
        if vertex_count_type == "Keep Vertex Count"
        else (
            vertex_count // 2
            if vertex_count_type == "Target Face Count"
            else vertex_count
        )
    )
    print(
        f"Final vertex count: {final_vertex_count} with type {vertex_count_type} and vertex count {vertex_count}"
    )

    glb_file, pc_file, pc_plot = run_model(
        background_state,
        guidance_scale,
        random_seed,
        pc_cond,
        remesh_option,
        final_vertex_count,
        texture_resolution,
    )
    # Create a single float list of x y z r g b
    point_list = []
    for i in range(pc_plot.vertices.shape[0]):
        point_list.extend(
            [
                pc_plot.vertices[i, 0],
                pc_plot.vertices[i, 1],
                pc_plot.vertices[i, 2],
                pc_plot.colors[i, 0] / 255.0,
                pc_plot.colors[i, 1] / 255.0,
                pc_plot.colors[i, 2] / 255.0,
            ]
        )

    return glb_file, pc_file, point_list


def regenerate_run(
    background_state,
    guidance_scale,
    random_seed,
    pc_cond,
    remesh_option,
    vertex_count_type,
    vertex_count,
    texture_resolution,
):
    glb_file, pc_file, point_list = process_model_run(
        background_state,
        guidance_scale,
        random_seed,
        pc_cond,
        remesh_option,
        vertex_count_type,
        vertex_count,
        texture_resolution,
    )
    return (
        gr.update(),  # run_btn
        gr.update(),  # img_proc_state
        gr.update(),  # background_remove_state
        gr.update(),  # preview_removal
        gr.update(value=glb_file, visible=True),  # output_3d
        gr.update(visible=True),  # hdr_row
        gr.update(visible=True),  # point_cloud_row
        gr.update(value=point_list),  # point_cloud_editor
        gr.update(value=pc_file),  # pc_download
        gr.update(visible=False),  # regenerate_btn
    )


def run_button(
    run_btn,
    input_image,
    background_state,
    foreground_ratio,
    no_crop,
    guidance_scale,
    random_seed,
    pc_upload,
    pc_cond_file,
    remesh_option,
    vertex_count_type,
    vertex_count,
    texture_resolution,
):
    if run_btn == "Run":
        if torch.cuda.is_available():
            torch.cuda.reset_peak_memory_stats()

        if pc_upload:
            # make sure the pc_cond_file has been uploaded
            try:
                pc_cond = trimesh.load(pc_cond_file.name)
            except Exception:
                raise gr.Error(
                    "Please upload a valid point cloud ply file as condition."
                )
        else:
            pc_cond = None

        glb_file, pc_file, pc_list = process_model_run(
            background_state,
            guidance_scale,
            random_seed,
            pc_cond,
            remesh_option,
            vertex_count_type,
            vertex_count,
            texture_resolution,
        )

        if torch.cuda.is_available():
            print("Peak Memory:", torch.cuda.max_memory_allocated() / 1024 / 1024, "MB")
        elif torch.backends.mps.is_available():
            print(
                "Peak Memory:", torch.mps.driver_allocated_memory() / 1024 / 1024, "MB"
            )

        return (
            gr.update(),  # run_btn
            gr.update(),  # img_proc_state
            gr.update(),  # background_remove_state
            gr.update(),  # preview_removal
            gr.update(value=glb_file, visible=True),  # output_3d
            gr.update(visible=True),  # hdr_row
            gr.update(visible=True),  # point_cloud_row
            gr.update(value=pc_list),  # point_cloud_editor
            gr.update(value=pc_file),  # pc_download
            gr.update(visible=False),  # regenerate_btn
        )

    elif run_btn == "Remove Background":
        rem_removed = remove_background(input_image)

        fr_res = spar3d_utils.foreground_crop(
            rem_removed,
            crop_ratio=foreground_ratio,
            newsize=(COND_WIDTH, COND_HEIGHT),
            no_crop=no_crop,
        )

        return (
            gr.update(value="Run", visible=True),  # run_btn
            rem_removed,  # img_proc_state,
            fr_res,  # background_remove_state
            gr.update(value=show_mask_img(fr_res), visible=True),  # preview_removal
            gr.update(value=None, visible=False),  # output_3d
            gr.update(visible=False),  # hdr_row
            gr.update(visible=False),  # point_cloud_row
            gr.update(value=None),  # point_cloud_editor
            gr.update(value=None),  # pc_download
            gr.update(visible=False),  # regenerate_btn
        )


def requires_bg_remove(image, fr, no_crop):
    if image is None:
        return (
            gr.update(visible=False, value="Run"),  # run_Btn
            None,  # img_proc_state
            None,  # background_remove_state
            gr.update(value=None, visible=False),  # preview_removal
            gr.update(value=None, visible=False),  # output_3d
            gr.update(visible=False),  # hdr_row
            gr.update(visible=False),  # point_cloud_row
            gr.update(value=None),  # point_cloud_editor
            gr.update(value=None),  # pc_download
            gr.update(visible=False),  # regenerate_btn
        )
    alpha_channel = np.array(image.getchannel("A"))
    min_alpha = alpha_channel.min()

    if min_alpha == 0:
        print("Already has alpha")
        fr_res = spar3d_utils.foreground_crop(
            image, fr, newsize=(COND_WIDTH, COND_HEIGHT), no_crop=no_crop
        )
        return (
            gr.update(value="Run", visible=True),  # run_Btn
            image,  # img_proc_state
            fr_res,  # background_remove_state
            gr.update(value=show_mask_img(fr_res), visible=True),  # preview_removal
            gr.update(value=None, visible=False),  # output_3d
            gr.update(visible=False),  # hdr_row
            gr.update(visible=False),  # point_cloud_row
            gr.update(value=None),  # point_cloud_editor
            gr.update(value=None),  # pc_download
            gr.update(visible=False),  # regenerate_btn
        )
    return (
        gr.update(value="Remove Background", visible=True),  # run_Btn
        None,  # img_proc_state
        None,  # background_remove_state
        gr.update(value=None, visible=False),  # preview_removal
        gr.update(value=None, visible=False),  # output_3d
        gr.update(visible=False),  # hdr_row
        gr.update(visible=False),  # point_cloud_row
        gr.update(value=None),  # point_cloud_editor
        gr.update(value=None),  # pc_download
        gr.update(visible=False),  # regenerate_btn
    )


def update_foreground_ratio(img_proc, fr, no_crop):
    foreground_res = spar3d_utils.foreground_crop(
        img_proc, fr, newsize=(COND_WIDTH, COND_HEIGHT), no_crop=no_crop
    )
    return (
        foreground_res,
        gr.update(value=show_mask_img(foreground_res)),
    )


def update_resolution_controls(remesh_choice, vertex_count_type):
    show_controls = remesh_choice.lower() != "none"
    show_vertex_count = vertex_count_type != "Keep Vertex Count"
    return (
        gr.update(visible=show_controls),  # vertex_count_type
        gr.update(visible=show_controls and show_vertex_count),  # vertex_count_slider
    )


with gr.Blocks() as demo:
    img_proc_state = gr.State()
    background_remove_state = gr.State()
    gr.Markdown(
        """
    # SPAR3D: Stable Point-Aware Reconstruction of 3D Objects from Single Images

    SPAR3D is a state-of-the-art method for 3D mesh reconstruction from a single image. This demo allows you to upload an image and generate a 3D mesh model from it. A feature of SPAR3D is it generates point clouds as intermediate representation before producing the mesh. You can edit the point cloud to adjust the final mesh. We provide a simple point cloud editor in this demo, where you can drag, recolor and rescale the point clouds. If you have more advanced editing needs (e.g. box selection, duplication, local streching, etc.), you can download the point cloud and edit it in softwares such as MeshLab or Blender. The edited point cloud can then be uploaded to this demo to generate a new 3D model by checking the "Point cloud upload" box.

    **Tips**

    1. If the image does not have a valid alpha channel, it will go through the background removal step. Our built-in background removal can be inaccurate sometimes, which will result in poor mesh quality. In such cases, you can use external background removal tools to obtain a RGBA image before uploading here.
    2. You can adjust the foreground ratio to control the size of the foreground object. This may have major impact on the final mesh.
    3. Guidance scale controls the strength of the image condition in the point cloud generation process. A higher value may result in higher mesh fidelity, but the variability by changing the random seed will be lower. Note that the guidance scale and the seed are not effective when the point cloud is manually uploaded.
    4. Our online editor supports multi-selection by holding down the shift key. This allows you to recolor multiple points at once.
    5. The editing should mainly alter the unseen parts of the object. Visible parts can be edited, but the edits should be consistent with the image. Editing the visible parts in a way that contradicts the image may result in poor mesh quality.
    6. You can upload your own HDR environment map to light the 3D model.
    """
    )
    with gr.Row(variant="panel"):
        with gr.Column():
            with gr.Row():
                input_img = gr.Image(
                    type="pil", label="Input Image", sources="upload", image_mode="RGBA"
                )
                preview_removal = gr.Image(
                    label="Preview Background Removal",
                    type="pil",
                    image_mode="RGB",
                    interactive=False,
                    visible=False,
                )

            gr.Markdown("### Input Controls")
            with gr.Group():
                with gr.Row():
                    no_crop = gr.Checkbox(label="No cropping", value=False)
                    pc_upload = gr.Checkbox(label="Point cloud upload", value=False)

                pc_cond_file = gr.File(
                    label="Point Cloud Upload",
                    file_types=[".ply"],
                    file_count="single",
                    visible=False,
                )

                foreground_ratio = gr.Slider(
                    label="Padding Ratio",
                    minimum=1.0,
                    maximum=2.0,
                    value=1.3,
                    step=0.05,
                )

            pc_upload.change(
                lambda x: gr.update(visible=x),
                inputs=pc_upload,
                outputs=[pc_cond_file],
            )

            no_crop.change(
                update_foreground_ratio,
                inputs=[img_proc_state, foreground_ratio, no_crop],
                outputs=[background_remove_state, preview_removal],
            )

            foreground_ratio.change(
                update_foreground_ratio,
                inputs=[img_proc_state, foreground_ratio, no_crop],
                outputs=[background_remove_state, preview_removal],
            )

            gr.Markdown("### Point Diffusion Controls")
            with gr.Group():
                guidance_scale = gr.Slider(
                    label="Guidance Scale",
                    minimum=1.0,
                    maximum=10.0,
                    value=3.0,
                    step=1.0,
                )

                random_seed = gr.Slider(
                    label="Seed",
                    minimum=0,
                    maximum=10000,
                    value=0,
                    step=1,
                )

            no_remesh = not TRIANGLE_REMESH_AVAILABLE and not QUAD_REMESH_AVAILABLE
            gr.Markdown(
                "### Texture Controls"
                if no_remesh
                else "### Meshing and Texture Controls"
            )
            with gr.Group():
                remesh_choices = ["None"]
                if TRIANGLE_REMESH_AVAILABLE:
                    remesh_choices.append("Triangle")
                if QUAD_REMESH_AVAILABLE:
                    remesh_choices.append("Quad")

                remesh_option = gr.Radio(
                    choices=remesh_choices,
                    label="Remeshing",
                    value="None",
                    visible=not no_remesh,
                )

                vertex_count_type = gr.Radio(
                    choices=[
                        "Keep Vertex Count",
                        "Target Vertex Count",
                        "Target Face Count",
                    ],
                    label="Mesh Resolution Control",
                    value="Keep Vertex Count",
                    visible=False,
                )

                vertex_count_slider = gr.Slider(
                    label="Target Count",
                    minimum=0,
                    maximum=20000,
                    value=2000,
                    visible=False,
                )

                texture_size = gr.Slider(
                    label="Texture Size",
                    minimum=512,
                    maximum=2048,
                    value=1024,
                    step=256,
                    visible=True,
                )

            remesh_option.change(
                update_resolution_controls,
                inputs=[remesh_option, vertex_count_type],
                outputs=[vertex_count_type, vertex_count_slider],
            )

            vertex_count_type.change(
                update_resolution_controls,
                inputs=[remesh_option, vertex_count_type],
                outputs=[vertex_count_type, vertex_count_slider],
            )

            run_btn = gr.Button("Run", variant="primary", visible=False)

        with gr.Column():
            with gr.Group(visible=False) as point_cloud_row:
                point_size_slider = gr.Slider(
                    label="Point Size",
                    minimum=0.01,
                    maximum=1.0,
                    value=0.2,
                    step=0.01,
                )
                point_cloud_editor = PointCloudEditor(
                    up_axis="Z",
                    forward_axis="X",
                    lock_scale_z=True,
                    lock_scale_y=True,
                    visible=True,
                )

                pc_download = gr.File(
                    label="Point Cloud Download",
                    file_types=[".ply"],
                    file_count="single",
                )
            point_size_slider.change(
                fn=lambda x: gr.update(point_size=x),
                inputs=point_size_slider,
                outputs=point_cloud_editor,
            )

            regenerate_btn = gr.Button(
                "Re-run with point cloud", variant="primary", visible=False
            )

            output_3d = LitModel3D(
                label="3D Model",
                visible=False,
                clear_color=[0.0, 0.0, 0.0, 0.0],
                tonemapping="aces",
                contrast=1.0,
                scale=1.0,
            )
            with gr.Column(visible=False, scale=1.0) as hdr_row:
                gr.Markdown(
                    """## HDR Environment Map

                Select an HDR environment map to light the 3D model. You can also upload your own HDR environment maps.
                """
                )

                with gr.Row():
                    hdr_illumination_file = gr.File(
                        label="HDR Env Map",
                        file_types=[".hdr"],
                        file_count="single",
                    )
                    example_hdris = [
                        os.path.join("demo_files/hdri", f)
                        for f in os.listdir("demo_files/hdri")
                    ]
                    hdr_illumination_example = gr.Examples(
                        examples=example_hdris,
                        inputs=hdr_illumination_file,
                    )

                    hdr_illumination_file.change(
                        lambda x: gr.update(env_map=x.name if x is not None else None),
                        inputs=hdr_illumination_file,
                        outputs=[output_3d],
                    )

    examples = gr.Examples(
        examples=example_files, inputs=input_img, examples_per_page=11
    )

    input_img.change(
        requires_bg_remove,
        inputs=[input_img, foreground_ratio, no_crop],
        outputs=[
            run_btn,
            img_proc_state,
            background_remove_state,
            preview_removal,
            output_3d,
            hdr_row,
            point_cloud_row,
            point_cloud_editor,
            pc_download,
            regenerate_btn,
        ],
    )

    point_cloud_editor.edit(
        fn=lambda _x: gr.update(visible=True),
        inputs=point_cloud_editor,
        outputs=regenerate_btn,
    )

    regenerate_btn.click(
        regenerate_run,
        inputs=[
            background_remove_state,
            guidance_scale,
            random_seed,
            point_cloud_editor,
            remesh_option,
            vertex_count_type,
            vertex_count_slider,
            texture_size,
        ],
        outputs=[
            run_btn,
            img_proc_state,
            background_remove_state,
            preview_removal,
            output_3d,
            hdr_row,
            point_cloud_row,
            point_cloud_editor,
            pc_download,
            regenerate_btn,
        ],
    )

    run_btn.click(
        run_button,
        inputs=[
            run_btn,
            input_img,
            background_remove_state,
            foreground_ratio,
            no_crop,
            guidance_scale,
            random_seed,
            pc_upload,
            pc_cond_file,
            remesh_option,
            vertex_count_type,
            vertex_count_slider,
            texture_size,
        ],
        outputs=[
            run_btn,
            img_proc_state,
            background_remove_state,
            preview_removal,
            output_3d,
            hdr_row,
            point_cloud_row,
            point_cloud_editor,
            pc_download,
            regenerate_btn,
        ],
    )

demo.queue().launch()