Update app.py

app.py

@@ -11,64 +11,16 @@ import imageio
 from easydict import EasyDict as edict
 from PIL import Image, ImageOps
 from trellis.pipelines import TrellisImageTo3DPipeline
-from trellis.representations import MeshExtractResult
+from trellis.representations import Gaussian, MeshExtractResult
 from trellis.utils import render_utils, postprocessing_utils
 from diffusers import ControlNetModel, StableDiffusionXLControlNetPipeline, AutoencoderKL
 from diffusers import EulerAncestralDiscreteScheduler
-from huggingface_hub import HfApi
 from pathlib import Path
 
 style_list = [
-    {
-        "name": "(No style)",
-        "prompt": "{prompt}",
-        "negative_prompt": "longbody, lowres, bad anatomy, bad hands, missing fingers, extra digit, fewer digits, cropped, worst quality, low quality",
-    },
-    {
-        "name": "Cinematic",
-        "prompt": "cinematic still {prompt} . emotional, harmonious, vignette, highly detailed, high budget, bokeh, cinemascope, moody, epic, gorgeous, film grain, grainy",
-        "negative_prompt": "anime, cartoon, graphic, text, painting, crayon, graphite, abstract, glitch, deformed, mutated, ugly, disfigured",
-    },
-    {
-        "name": "3D Model",
-        "prompt": "professional 3d model {prompt} . octane render, highly detailed, volumetric, dramatic lighting",
-        "negative_prompt": "ugly, deformed, noisy, low poly, blurry, painting",
-    },
-    {
-        "name": "Anime",
-        "prompt": "anime artwork {prompt} . anime style, key visual, vibrant, studio anime,  highly detailed",
-        "negative_prompt": "photo, deformed, black and white, realism, disfigured, low contrast",
-    },
-    {
-        "name": "Digital Art",
-        "prompt": "concept art {prompt} . digital artwork, illustrative, painterly, matte painting, highly detailed",
-        "negative_prompt": "photo, photorealistic, realism, ugly",
-    },
-    {
-        "name": "Photographic",
-        "prompt": "cinematic photo {prompt} . 35mm photograph, film, bokeh, professional, 4k, highly detailed",
-        "negative_prompt": "drawing, painting, crayon, sketch, graphite, impressionist, noisy, blurry, soft, deformed, ugly",
-    },
-    {
-        "name": "Pixel art",
-        "prompt": "pixel-art {prompt} . low-res, blocky, pixel art style, 8-bit graphics",
-        "negative_prompt": "sloppy, messy, blurry, noisy, highly detailed, ultra textured, photo, realistic",
-    },
-    {
-        "name": "Fantasy art",
-        "prompt": "ethereal fantasy concept art of  {prompt} . magnificent, celestial, ethereal, painterly, epic, majestic, magical, fantasy art, cover art, dreamy",
-        "negative_prompt": "photographic, realistic, realism, 35mm film, dslr, cropped, frame, text, deformed, glitch, noise, noisy, off-center, deformed, cross-eyed, closed eyes, bad anatomy, ugly, disfigured, sloppy, duplicate, mutated, black and white",
-    },
-    {
-        "name": "Neonpunk",
-        "prompt": "neonpunk style {prompt} . cyberpunk, vaporwave, neon, vibes, vibrant, stunningly beautiful, crisp, detailed, sleek, ultramodern, magenta highlights, dark purple shadows, high contrast, cinematic, ultra detailed, intricate, professional",
-        "negative_prompt": "painting, drawing, illustration, glitch, deformed, mutated, cross-eyed, ugly, disfigured",
-    },
-    {
-        "name": "Manga",
-        "prompt": "manga style {prompt} . vibrant, high-energy, detailed, iconic, Japanese comic style",
-        "negative_prompt": "ugly, deformed, noisy, blurry, low contrast, realism, photorealistic, Western comic style",
-    },
+    {"name": "(No style)", "prompt": "{prompt}", "negative_prompt": ""},
+    {"name": "Cinematic", "prompt": "cinematic still {prompt} . emotional, harmonious, vignette, highly detailed, high budget, bokeh, cinemascope, moody, epic, gorgeous, film grain, grainy", "negative_prompt": "anime, cartoon, graphic, text, painting, crayon, graphite, abstract, glitch, deformed, mutated, ugly, disfigured"},
+    {"name": "3D Model", "prompt": "professional 3d model {prompt} . octane render, highly detailed, volumetric, dramatic lighting", "negative_prompt": "ugly, deformed, noisy, low poly, blurry, painting"},
 ]
 styles = {k["name"]: (k["prompt"], k["negative_prompt"]) for k in style_list}
 STYLE_NAMES = list(styles.keys())
@@ -77,9 +29,6 @@ MAX_SEED = np.iinfo(np.int32).max
 TMP_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'tmp')
 os.makedirs(TMP_DIR, exist_ok=True)
 
-def reset_canvas():
-    return gr.update(value={"background":Image.new("RGB", (512, 512), (255, 255, 255)), "layers":[Image.new("RGB", (512, 512), (255, 255, 255))], "composite":Image.new("RGB", (512, 512), (255, 255, 255))})
-
 def apply_style(style_name: str, positive: str, negative: str = "") -> tuple[str, str]:
     p, n = styles.get(style_name, styles[DEFAULT_STYLE_NAME])
     return p.replace("{prompt}", positive), n + negative
@@ -87,26 +36,26 @@ def apply_style(style_name: str, positive: str, negative: str = "") -> tuple[str
 def start_session(req: gr.Request):
     user_dir = os.path.join(TMP_DIR, str(req.session_hash))
     os.makedirs(user_dir, exist_ok=True)
-
+    
 def end_session(req: gr.Request):
     user_dir = os.path.join(TMP_DIR, str(req.session_hash))
     shutil.rmtree(user_dir)
 
 @spaces.GPU
 def preprocess_image(image: Image.Image, 
-                    prompt: str = "",
-                    negative_prompt: str = "",
-                    style_name: str = "",
-                    num_steps: int = 25,
-                    guidance_scale: float = 5,
-                    controlnet_conditioning_scale: float = 1.0,
-                    ) -> Image.Image:
-    width, height  = image['composite'].size
-    ratio = np.sqrt(1024. * 1024. / (width * height))
-    new_width, new_height = int(width * ratio), int(height * ratio)
-    image = image['composite'].resize((new_width, new_height))
-    image = ImageOps.invert(image)
+                    prompt: str,
+                    negative_prompt: str,
+                    style_name: str,
+                    num_steps: int,
+                    guidance_scale: float,
+                    controlnet_conditioning_scale: float) -> Image.Image:
+    width, height = image.size
+    ratio = np.sqrt(1024 * 1024 / (width * height))
+    new_size = (int(width * ratio), int(height * ratio))
+    image = image.resize(new_size)
+    
     prompt, negative_prompt = apply_style(style_name, prompt, negative_prompt)
+    
     output = pipe_control(
         prompt=prompt,
         negative_prompt=negative_prompt,
@@ -114,10 +63,9 @@ def preprocess_image(image: Image.Image,
         num_inference_steps=num_steps,
         controlnet_conditioning_scale=controlnet_conditioning_scale,
         guidance_scale=guidance_scale,
-        width=new_width,
-        height=new_height).images[0]
-    processed_image = pipeline.preprocess_image(output)
-    return (image, processed_image)
+    ).images[0]
+    
+    return pipeline.preprocess_image(output)
 
 def pack_state(gs: Gaussian, mesh: MeshExtractResult) -> dict:
     return {
@@ -134,8 +82,7 @@ def pack_state(gs: Gaussian, mesh: MeshExtractResult) -> dict:
             'faces': mesh.faces.cpu().numpy(),
         },
     }
-    
-    
+
 def unpack_state(state: dict) -> Tuple[Gaussian, edict, str]:
     gs = Gaussian(
         aabb=state['gaussian']['aabb'],
@@ -158,9 +105,6 @@ def unpack_state(state: dict) -> Tuple[Gaussian, edict, str]:
     
     return gs, mesh
 
-def get_seed(randomize_seed: bool, seed: int) -> int:
-    return np.random.randint(0, MAX_SEED) if randomize_seed else seed
-
 @spaces.GPU
 def image_to_3d(
     image: Image.Image,
@@ -173,10 +117,9 @@ def image_to_3d(
 ) -> Tuple[dict, str]:
     user_dir = os.path.join(TMP_DIR, str(req.session_hash))
     outputs = pipeline.run(
-        image[1],
+        image,
         seed=seed,
-        formats=["mesh"],
-        preprocess_image=False,
+        formats=["gaussian", "mesh"],
         sparse_structure_sampler_params={
             "steps": ss_sampling_steps,
             "cfg_strength": ss_guidance_strength,
@@ -186,10 +129,13 @@ def image_to_3d(
             "cfg_strength": slat_guidance_strength,
         },
     )
-    video = render_utils.render_video(outputs['mesh'][0], num_frames=120)['normal']
+   
+    video = render_utils.render_video(outputs['gaussian'][0], num_frames=120)['color']
+    video_geo = render_utils.render_video(outputs['mesh'][0], num_frames=120)['normal']
+    video = [np.concatenate([video[i], video_geo[i]], axis=1) for i in range(len(video))]
     video_path = os.path.join(user_dir, 'sample.mp4')
     imageio.mimsave(video_path, video, fps=15)
-    state = pack_state(outputs['mesh'][0])
+    state = pack_state(outputs['gaussian'][0], outputs['mesh'][0])
     torch.cuda.empty_cache()
     return state, video_path
 
@@ -201,127 +147,88 @@ def extract_glb(
     req: gr.Request,
 ) -> str:
     user_dir = os.path.join(TMP_DIR, str(req.session_hash))
-    mesh = unpack_state(state)
-    glb = postprocessing_utils.to_glb(mesh, simplify=mesh_simplify, texture_size=texture_size, verbose=False)
+    gs, mesh = unpack_state(state)
+    glb = postprocessing_utils.to_glb(gs, mesh, simplify=mesh_simplify, texture_size=texture_size)
     glb_path = os.path.join(user_dir, 'sample.glb')
     glb.export(glb_path)
     torch.cuda.empty_cache()
     return glb_path
 
-def reset_do_preprocess():
-    return True
-
-with gr.Blocks(delete_cache=(600, 600)) as demo:
-    gr.Markdown("""
-    ## Sketch to 3D with TRELLIS
-    1. Fast sketch to image with SDXL Flash
-    2. Scalable and versatile image to 3D generation using [TRELLIS](https://trellis3d.github.io/)
-    ### 🎨 Draw or upload a sketch and click "Generate" to create a 3D asset ✨
-    """)
+with gr.Blocks() as demo:
+    gr.Markdown("# Sketch to 3D with TRELLIS")
     with gr.Row():
         with gr.Column():
-            image_prompt = gr.ImageMask(label="Input sketch", type="pil", image_mode="RGB", height=512, value=reset_canvas())
-            with gr.Row():
-                sketch_btn = gr.Button("Process sketch")
-                generate_btn = gr.Button("Generate 3D")
-            with gr.Row():
-                prompt = gr.Textbox(label="Prompt")
-                style = gr.Dropdown(label="Style", choices=STYLE_NAMES, value=DEFAULT_STYLE_NAME)
-            with gr.Accordion(label="Generation Settings", open=False):
-                with gr.Tab(label="sketch-to-image generation"):
-                    negative_prompt = gr.Textbox(label="Negative prompt")
-                    num_steps = gr.Slider(1, 20, label="Number of steps", value=8, step=1)
-                    guidance_scale = gr.Slider(0.1, 10.0, label="Guidance scale", value=5, step=0.1)
-                    controlnet_conditioning_scale = gr.Slider(0.5, 5.0, label="ControlNet conditioning scale", value=0.85, step=0.01)
-                with gr.Tab(label="3D generation"):
-                    seed = gr.Slider(0, MAX_SEED, label="Seed", value=0, step=1)
-                    randomize_seed = gr.Checkbox(label="Randomize Seed", value=True)
-                    gr.Markdown("Stage 1: Sparse Structure Generation")
-                    with gr.Row():
-                        ss_guidance_strength = gr.Slider(0.0, 10.0, label="Guidance Strength", value=7.5, step=0.1)
-                        ss_sampling_steps = gr.Slider(1, 50, label="Sampling Steps", value=12, step=1)
-                    gr.Markdown("Stage 2: Structured Latent Generation")
-                    with gr.Row():
-                        slat_guidance_strength = gr.Slider(0.0, 10.0, label="Guidance Strength", value=3.0, step=0.1)
-                        slat_sampling_steps = gr.Slider(1, 50, label="Sampling Steps", value=12, step=1)
-            with gr.Accordion(label="GLB Extraction Settings", open=False):
-                mesh_simplify = gr.Slider(0.9, 0.98, label="Simplify", value=0.95, step=0.01)
+            image_prompt = gr.Image(label="Sketch Input", type="pil", image_mode="RGBA", height=512)
+            prompt = gr.Textbox(label="Prompt", placeholder="Describe tu modelo 3D")
+            style = gr.Dropdown(label="Style", choices=STYLE_NAMES, value=DEFAULT_STYLE_NAME)
+            
+            with gr.Accordion("Generation Settings", open=False):
+                num_steps = gr.Slider(1, 20, label="Steps", value=8, step=1)
+                guidance_scale = gr.Slider(0.1, 10.0, label="Guidance Scale", value=5.0, step=0.1)
+                controlnet_scale = gr.Slider(0.5, 5.0, label="ControlNet Scale", value=0.85, step=0.01)
+                seed = gr.Slider(0, MAX_SEED, label="Seed", value=0, step=1)
+                randomize_seed = gr.Checkbox(label="Randomize Seed", value=True)
+                
+                with gr.Group():
+                    gr.Markdown("#### Stage 1: Structure")
+                    ss_guidance = gr.Slider(0.0, 10.0, label="Guidance", value=7.5, step=0.1)
+                    ss_steps = gr.Slider(1, 50, label="Steps", value=12, step=1)
+                
+                with gr.Group():
+                    gr.Markdown("#### Stage 2: Detail")
+                    slat_guidance = gr.Slider(0.0, 10.0, label="Guidance", value=3.0, step=0.1)
+                    slat_steps = gr.Slider(1, 50, label="Steps", value=12, step=1)
+            
+            generate_btn = gr.Button("Generate 3D Model", variant="primary")
+            
+            with gr.Accordion("Export Settings", open=False):
+                mesh_simplify = gr.Slider(0.9, 0.98, label="Simplify Mesh", value=0.95, step=0.01)
                 texture_size = gr.Slider(512, 2048, label="Texture Size", value=1024, step=512)
-            with gr.Row():
-                extract_glb_btn = gr.Button("Extract GLB", interactive=False)
-            gr.Markdown("")
-
+            
+            extract_btn = gr.Button("Export GLB", interactive=False)
+        
         with gr.Column():
-            video_output = gr.Video(label="Generated 3D Asset", autoplay=True, loop=True, height=300)
-            image_prompt_processed = gr.Image(label="Processed sketch", interactive=False, type="pil", height=512)
-            model_output = LitModel3D(label="Extracted GLB", exposure=10.0, height=300)
-            download_glb = gr.DownloadButton(label="Download GLB", interactive=False)
-
-    output_buf = gr.State()
-    do_preprocess = gr.State(True)
+            video_output = gr.Video(label="3D Preview", autoplay=True, loop=True, height=300)
+            model_viewer = LitModel3D(label="3D Model Viewer", height=400)
+            download_btn = gr.DownloadButton("Download GLB", interactive=False)
 
-    with gr.Row(visible=False) as single_image_example:
-        examples = gr.Examples(
-            examples=[f'assets/example_image/{image}' for image in os.listdir("assets/example_image")],
-            inputs=[image_prompt],
-            fn=preprocess_image,
-            outputs=[image_prompt_processed],
-            run_on_click=True,
-            examples_per_page=64,
-        )
+    output_state = gr.State()
 
     demo.load(start_session)
     demo.unload(end_session)
-
-    image_prompt.clear(reset_canvas, outputs=[image_prompt])
-
-    sketch_btn.click(
-        get_seed,
+    
+    generate_btn.click(
+        lambda rand, s: np.random.randint(0, MAX_SEED) if rand else s,
         inputs=[randomize_seed, seed],
         outputs=[seed],
     ).then(
         preprocess_image,
-        inputs=[image_prompt, prompt, negative_prompt, style, num_steps, guidance_scale, controlnet_conditioning_scale],
-        outputs=[image_prompt_processed],
-    )
-
-    generate_btn.click(
-        get_seed,
-        inputs=[randomize_seed, seed],
-        outputs=[seed],
+        inputs=[image_prompt, prompt, gr.Textbox(), style, num_steps, guidance_scale, controlnet_scale],
+        outputs=[image_prompt],
     ).then(
         image_to_3d,
-        inputs=[image_prompt_processed, seed, ss_guidance_strength, ss_sampling_steps, slat_guidance_strength, slat_sampling_steps],
-        outputs=[output_buf, video_output],
+        inputs=[image_prompt, seed, ss_guidance, ss_steps, slat_guidance, slat_steps],
+        outputs=[output_state, video_output],
     ).then(
         lambda: gr.Button(interactive=True),
-        outputs=[extract_glb_btn],
-    )
-
-    video_output.clear(
-        lambda: gr.Button(interactive=False),
-        outputs=[extract_glb_btn],
+        outputs=[extract_btn],
     )
 
-    extract_glb_btn.click(
+    extract_btn.click(
         extract_glb,
-        inputs=[output_buf, mesh_simplify, texture_size],
-        outputs=[model_output, download_glb],
+        inputs=[output_state, mesh_simplify, texture_size],
+        outputs=[model_viewer, download_btn],
     ).then(
         lambda: gr.Button(interactive=True),
-        outputs=[download_glb],
-    )
-
-    model_output.clear(
-        lambda: gr.Button(interactive=False),
-        outputs=[download_glb],
+        outputs=[download_btn],
     )
 
 if __name__ == "__main__":
+    # TRELLIS pipeline
     pipeline = TrellisImageTo3DPipeline.from_pretrained("JeffreyXiang/TRELLIS-image-large")
     pipeline.cuda()
-    device = "cuda" if torch.cuda.is_available() else "cpu"
     
+    # ControlNet y SDXL
     controlnet = ControlNetModel.from_pretrained(
         "xinsir/controlnet-scribble-sdxl-1.0",
         torch_dtype=torch.float16
@@ -334,11 +241,6 @@ if __name__ == "__main__":
         torch_dtype=torch.float16,
     )
     pipe_control.scheduler = EulerAncestralDiscreteScheduler.from_config(pipe_control.scheduler.config)
-    pipe_control.to(device)
-    
-    try:
-        pipeline.preprocess_image(Image.fromarray(np.zeros((512, 512, 3), dtype=np.uint8)))
-    except:
-        pass
+    pipe_control.to("cuda")
     
     demo.launch()