import time import json import ast import io from PIL import Image, ImageDraw, ImageFont, ImageColor import xml.etree.ElementTree as ET import numpy as np import spaces import torch import gradio as gr from transformers import Qwen2_5_VLForConditionalGeneration, AutoProcessor ### Load the model and helper functions ### model_path = "Qwen/Qwen2.5-VL-3B-Instruct" device = "cuda" if torch.cuda.is_available() else "cpu" model = Qwen2_5_VLForConditionalGeneration.from_pretrained( model_path, torch_dtype=torch.bfloat16 if device == "cuda" else torch.float32, attn_implementation="sdpa" ).to(device) processor = AutoProcessor.from_pretrained(model_path) additional_colors = [colorname for (colorname, colorcode) in ImageColor.colormap.items()] def parse_json(json_output): lines = json_output.splitlines() for i, line in enumerate(lines): if line.startswith("```json"): json_output = "\n".join(lines[i+1:]) json_output = json_output.split("```")[0] break return json_output def plot_bounding_boxes(im, bounding_boxes, input_width, input_height): img = im width, height = img.size draw = ImageDraw.Draw(img) colors = ['red', 'green', 'blue', 'yellow', 'orange', 'pink', 'purple', 'cyan', 'magenta', 'lime'] + additional_colors bounding_boxes = parse_json(bounding_boxes) try: font = ImageFont.load_default(size=16.0) except: font = ImageFont.load_default() try: json_output = ast.literal_eval(bounding_boxes) except Exception as e: end_idx = bounding_boxes.rfind('"}') + len('"}') truncated_text = bounding_boxes[:end_idx] + "]" try: json_output = ast.literal_eval(truncated_text) except: json_output = [] for i, bounding_box in enumerate(json_output): color = colors[i % len(colors)] try: coords = bounding_box.get("bbox_2d", bounding_box.get("box_2d", [0,0,0,0])) abs_y1 = int(coords[1]/input_height * height) abs_x1 = int(coords[0]/input_width * width) abs_y2 = int(coords[3]/input_height * height) abs_x2 = int(coords[2]/input_width * width) except: continue if abs_x1 > abs_x2: abs_x1, abs_x2 = abs_x2, abs_x1 if abs_y1 > abs_y2: abs_y1, abs_y2 = abs_y2, abs_y1 draw.rectangle(((abs_x1, abs_y1), (abs_x2, abs_y2)), outline=color, width=4) # --- HIỂN THỊ CALO --- if "label" in bounding_box: display_text = bounding_box["label"] if "nutrition" in bounding_box: cals = bounding_box["nutrition"].get("calories_kcal", "?") display_text += f" ({cals} kcal)" try: bbox = draw.textbbox((abs_x1 + 8, abs_y1 + 6), display_text, font=font) draw.rectangle(bbox, fill="black") except AttributeError: pass draw.text((abs_x1 + 8, abs_y1 + 6), display_text, fill=color, font=font) return img def resize_image_to_max_dimension(image, max_dimension=1024): width, height = image.size if width <= max_dimension and height <= max_dimension: return image if width >= height: new_width = max_dimension new_height = int((height / width) * max_dimension) else: new_height = max_dimension new_width = int((width / height) * max_dimension) return image.resize((new_width, new_height), Image.LANCZOS) def inference(img_url, prompt, system_prompt="You are a nutrition assistant. Analyze food images, locate items, and estimate nutrition.", max_new_tokens=1024): if isinstance(img_url, str): image = Image.open(img_url) else: image = resize_image_to_max_dimension(img_url) messages = [ {"role": "system", "content": system_prompt}, {"role": "user", "content": [{"type": "text", "text": prompt}, {"image": img_url}]} ] text = processor.apply_chat_template(messages, tokenize=False, add_generation_prompt=True) inputs = processor(text=[text], images=[image], padding=True, return_tensors="pt").to(device) output_ids = model.generate(**inputs, max_new_tokens=1024) generated_ids = [output_ids[len(input_ids):] for input_ids, output_ids in zip(inputs.input_ids, output_ids)] output_text = processor.batch_decode(generated_ids, skip_special_tokens=True, clean_up_tokenization_spaces=True) input_height = inputs['image_grid_thw'][0][1]*14 input_width = inputs['image_grid_thw'][0][2]*14 return output_text[0], input_height, input_width def numpy_to_pil(numpy_array): if numpy_array.dtype != np.uint8: if numpy_array.dtype in [np.float32, np.float64] and numpy_array.max() <= 1.0: numpy_array = (numpy_array * 255).astype(np.uint8) else: numpy_array = numpy_array.astype(np.uint8) return Image.fromarray(numpy_array) ### Prompt section (Đã được rút gọn) ### prompt_thinking = """Outline the bounding box coordinates and names of each unique edible food and drink item and output all the coordinates in JSON format. Crucially, for EACH item, you must also estimate its nutritional value per serving (calories, protein, carbs, fat) and include it in the JSON under the "nutrition" key. Start with tags. Output format: [{"bbox_2d": ..., "label": ..., "nutrition": {"calories_kcal": ..., "protein_g": ..., "carbs_g": ..., "fat_g": ...}}...]. The image contains a pint of dark beer and a green salad. I will create bounding box coordinates and estimate macros. ```json [ {"bbox_2d": [27, 271, 365, 531], "label": "Pint of Guinness", "nutrition": {"calories_kcal": 210, "protein_g": 1.5, "carbs_g": 18, "fat_g": 0}}, {"bbox_2d": [343, 300, 609, 711], "label": "Green Salad", "nutrition": {"calories_kcal": 120, "protein_g": 3, "carbs_g": 10, "fat_g": 8}} ] """ prompt_no_thinking = """Outline the bounding box coordinates and names of each unique edible food and drink item and output all the coordinates in JSON format. Crucially, for EACH item, you must also estimate its nutritional value per serving (calories, protein, carbs, fat) and include it in the JSON under the "nutrition" key. Output format: [{"bbox_2d": ..., "label": ..., "nutrition": {"calories_kcal": ..., "protein_g": ..., "carbs_g": ..., "fat_g": ...}}...]. @spaces.GPU(duration=120) def infer_on_image(input_image): image_1 = numpy_to_pil(numpy_array=input_image) start_time_thinking = time.time() response, input_height, input_width = inference(img_url=image_1, prompt=prompt_thinking) total_time_thinking = round(time.time() - start_time_thinking, 4) output_image_1 = plot_bounding_boxes(image_1, response, input_width, input_height) image_2 = numpy_to_pil(numpy_array=input_image) start_time_no_thinking = time.time() response_no_thinking, input_height, input_width = inference(img_url=image_2, prompt=prompt_no_thinking) total_time_no_thinking = round(time.time() - start_time_no_thinking, 4) output_image_2 = plot_bounding_boxes(image_2, response_no_thinking, input_width, input_height) return output_image_1, response, total_time_thinking, output_image_2, response_no_thinking, total_time_no_thinking demo = gr.Interface(fn=infer_on_image, inputs=gr.Image(label="Input image"), outputs=[gr.Image(label="Image w/ thinking tags"), gr.Text(label="Raw output w/ thinking tags"), gr.Text(label="Inference time w/ thinking tags"), gr.Image(label="Image w/o thinking tags"), gr.Text(label="Raw output w/o thinking tags"), gr.Text(label="Inference time w/o thinking tags")], title="Qwen2.5-VL Food & Nutrition Detection 👁️🍔", description="Nhận diện món ăn, vẽ Bounding Box và Ước tính Dinh dưỡng.", cache_examples=False) demo.launch(debug=True)