Initial commit: Prescription Explainer with MedGemma and FHIR export

.gitignore

@@ -0,0 +1,43 @@
+# Python
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+.Python
+env/
+venv/
+.venv
+*.egg-info/
+dist/
+build/
+
+# Environment variables
+.env
+*.env
+!.env.example
+
+# IDE
+.vscode/
+.idea/
+*.swp
+*.swo
+
+# OS
+.DS_Store
+Thumbs.db
+
+# Streamlit
+.streamlit/secrets.toml
+
+# Model cache
+*.bin
+*.safetensors
+models/
+
+# Logs
+*.log
+
+# Testing
+.pytest_cache/
+.coverage
+htmlcov/

.streamlit/config.toml

@@ -0,0 +1,11 @@
+[theme]
+primaryColor = "#4CAF50"
+backgroundColor = "#FFFFFF"
+secondaryBackgroundColor = "#F0F2F6"
+textColor = "#262730"
+font = "sans serif"
+
+[server]
+headless = true
+enableCORS = false
+enableXsrfProtection = true

README.md

@@ -0,0 +1,63 @@
+---
+title: Prescription Explainer
+emoji: 💊
+colorFrom: blue
+colorTo: green
+sdk: streamlit
+sdk_version: 1.40.0
+app_file: app.py
+pinned: false
+license: mit
+---
+
+# 💊 Prescription Explainer
+
+AI-powered prescription understanding for Southeast Asian patients using Google's MedGemma.
+
+## What it does
+
+1. **Upload** a prescription image (photo or scan)
+2. **Get** a clear, patient-friendly explanation
+3. **Translate** to your preferred language (Thai, Indonesian, Vietnamese, Cambodian, Hindi, Mandarin, English)
+4. **Export** FHIR-compliant health data for sharing with healthcare providers
+
+## Features
+
+- **MedGemma-powered extraction** - Reads prescription images directly using Google's multimodal medical AI
+- **Plain-language explanations** - No medical jargon, just clear instructions
+- **Multilingual support** - 7 Southeast Asian languages
+- **FHIR R4 export** - MedicationStatement and MedicationRequest resources for health data portability
+- **Privacy-first** - No data stored, images processed and discarded
+
+## Technology Stack
+
+- **AI Model**: MedGemma 1.5 4B (`google/medgemma-1.5-4b-it`)
+- **Translation**: Gemma 3
+- **Frontend**: Streamlit
+- **FHIR**: fhir.resources (Python)
+- **Deployment**: Hugging Face Spaces
+
+## Usage
+
+```bash
+pip install -r requirements.txt
+streamlit run app.py
+```
+
+## Supported Languages
+
+- English
+- Thai (ไทย)
+- Indonesian (Bahasa Indonesia)
+- Vietnamese (Tiếng Việt)
+- Cambodian/Khmer (ភាសាខ្មែរ)
+- Hindi (हिन्दी)
+- Mandarin Simplified (简体中文)
+
+## Disclaimer
+
+This tool provides general information only. Always consult your healthcare provider for medical advice.
+
+---
+
+Built for the Google AI Hackathon 2026 - MedGemma Track

app.py

@@ -0,0 +1,182 @@
+"""Prescription Explainer - AI-powered prescription understanding for SE Asia."""
+
+import logging
+
+import streamlit as st
+from PIL import Image
+
+from src.constants import SUPPORTED_LANGUAGES
+from src.fhir_generator import FhirGenerator, parse_medications_to_dict
+from src.medgemma_service import MedGemmaService, load_medgemma_model
+from src.translation_service import TranslationService, load_translation_model
+
+# Configure logging
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+# Page configuration
+st.set_page_config(
+    page_title="Prescription Explainer",
+    page_icon="💊",
+    layout="centered",
+    initial_sidebar_state="collapsed",
+)
+
+
+@st.cache_resource
+def get_medgemma():
+    """Load and cache MedGemma model."""
+    with st.spinner("Loading AI model... This may take a moment on first run."):
+        model, processor = load_medgemma_model()
+        return MedGemmaService(model, processor)
+
+
+@st.cache_resource
+def get_translation_service():
+    """Load and cache translation model."""
+    with st.spinner("Loading translation model..."):
+        model, tokenizer = load_translation_model()
+        return TranslationService(model, tokenizer)
+
+
+@st.cache_resource
+def get_fhir_generator():
+    """Get FHIR generator instance."""
+    return FhirGenerator()
+
+
+def main():
+    """Main application entry point."""
+    # Header
+    st.title("💊 Prescription Explainer")
+    st.markdown(
+        "Upload a prescription image to get a clear explanation in your language."
+    )
+
+    # Language selector
+    selected_language = st.selectbox(
+        "Choose your language / เลือกภาษา / Pilih bahasa",
+        options=list(SUPPORTED_LANGUAGES.keys()),
+        index=0,
+    )
+
+    # File uploader
+    uploaded_file = st.file_uploader(
+        "Upload prescription image",
+        type=["jpg", "jpeg", "png", "webp"],
+        help="Take a photo of your prescription or upload an existing image",
+    )
+
+    if uploaded_file is not None:
+        # Display uploaded image
+        image = Image.open(uploaded_file)
+        st.image(image, caption="Your prescription", use_container_width=True)
+
+        # Process button
+        if st.button("📋 Explain My Prescription", type="primary", use_container_width=True):
+            try:
+                # Load services
+                medgemma = get_medgemma()
+                translation_service = get_translation_service()
+                fhir_generator = get_fhir_generator()
+
+                # Step 1: Extract medications
+                with st.spinner("Reading your prescription..."):
+                    extraction = medgemma.extract_medications(image)
+
+                # Step 2: Generate explanation using translation model
+                with st.spinner("Creating explanation..."):
+                    from src.prompts import EXPLANATION_PROMPT
+                    explanation_prompt = EXPLANATION_PROMPT.format(medication_info=extraction)
+                    explanation = translation_service.generate_text(explanation_prompt)
+
+                # Step 3: Translate if needed
+                if selected_language != "English":
+                    with st.spinner(f"Translating to {selected_language}..."):
+                        explanation = translation_service.translate_text(
+                            explanation, selected_language
+                        )
+
+                # Display results
+                st.success("Done!")
+                st.markdown("---")
+                st.subheader("📖 Your Prescription Explained")
+                st.markdown(explanation)
+
+                # FHIR Export section
+                st.markdown("---")
+                st.subheader("📤 Export Health Data (FHIR)")
+
+                try:
+                    medications = parse_medications_to_dict(extraction)
+
+                    col1, col2 = st.columns(2)
+
+                    with col1:
+                        # Generate MedicationStatement for each medication
+                        statements = []
+                        for med in medications:
+                            try:
+                                statements.append(fhir_generator.generate_medication_statement(med))
+                            except Exception as e:
+                                logger.warning(f"Skipping FHIR generation for {med.get('drug_name', 'unknown')}: {e}")
+
+                        if statements:
+                            combined_statements = "[\n" + ",\n".join(statements) + "\n]"
+
+                            st.download_button(
+                                label="💾 MedicationStatement (JSON)",
+                                data=combined_statements,
+                                file_name="medication_statement.json",
+                                mime="application/json",
+                            )
+                        else:
+                            st.info("FHIR export not available for this prescription")
+
+                    with col2:
+                        # Generate MedicationRequest for each medication
+                        requests = []
+                        for med in medications:
+                            try:
+                                requests.append(fhir_generator.generate_medication_request(med))
+                            except Exception as e:
+                                logger.warning(f"Skipping FHIR generation for {med.get('drug_name', 'unknown')}: {e}")
+
+                        if requests:
+                            combined_requests = "[\n" + ",\n".join(requests) + "\n]"
+
+                            st.download_button(
+                                label="💾 MedicationRequest (JSON)",
+                                data=combined_requests,
+                                file_name="medication_request.json",
+                                mime="application/json",
+                            )
+                        else:
+                            st.info("FHIR export not available for this prescription")
+
+                    st.caption(
+                        "FHIR R4 compliant files for sharing with healthcare providers"
+                    )
+                except Exception as e:
+                    logger.error(f"FHIR export failed: {e}")
+                    st.warning("FHIR export not available for this prescription")
+
+            except ValueError as e:
+                st.error(str(e))
+            except Exception as e:
+                logger.error(f"Processing failed: {e}")
+                st.error(
+                    "Something went wrong. Please try again with a clearer image."
+                )
+
+    # Footer
+    st.markdown("---")
+    st.caption(
+        "⚠️ This tool provides general information only. "
+        "Always consult your healthcare provider for medical advice."
+    )
+    st.caption("Built with MedGemma for the Google AI Hackathon 2026")
+
+
+if __name__ == "__main__":
+    main()

requirements.txt

@@ -0,0 +1,6 @@
+streamlit==1.40.0
+transformers>=4.50.0
+torch
+Pillow
+fhir.resources
+accelerate

src/__init__.py

@@ -0,0 +1 @@
+# Prescription Explainer Services

src/constants.py

@@ -0,0 +1,19 @@
+"""Application constants for Prescription Explainer."""
+
+# Model identifiers
+MEDGEMMA_MODEL_ID = "google/medgemma-1.5-4b-it"
+TRANSLATION_MODEL_ID = "google/gemma-3-4b-it"  # Gemma 3 for translation
+
+# Supported languages with display names and codes
+SUPPORTED_LANGUAGES = {
+    "English": "en",
+    "Thai": "th",
+    "Indonesian": "id",
+    "Vietnamese": "vi",
+    "Cambodian (Khmer)": "km",
+    "Hindi": "hi",
+    "Mandarin (Simplified)": "zh",
+}
+
+# FHIR constants
+FHIR_VERSION = "R4"

src/fhir_generator.py

@@ -0,0 +1,184 @@
+"""FHIR R4 resource generator for medication data."""
+
+import json
+import logging
+from datetime import datetime
+from typing import Any
+from uuid import uuid4
+
+from fhir.resources.medicationrequest import MedicationRequest
+from fhir.resources.medicationstatement import MedicationStatement
+from fhir.resources.codeableconcept import CodeableConcept
+from fhir.resources.dosage import Dosage
+from fhir.resources.reference import Reference
+
+logger = logging.getLogger(__name__)
+
+
+class FhirGenerator:
+    """Generator for FHIR R4 medication resources."""
+
+    def generate_medication_statement(
+        self, medication_data: dict[str, Any]
+    ) -> str:
+        """
+        Generate a FHIR MedicationStatement resource.
+
+        Args:
+            medication_data: Dictionary with medication info
+                - drug_name: Name of the medication
+                - dosage: Dosage amount and unit
+                - frequency: How often to take
+                - route: Route of administration (optional)
+                - notes: Additional instructions (optional)
+
+        Returns:
+            JSON string of FHIR MedicationStatement
+        """
+        try:
+            # Ensure drug_name is non-empty
+            drug_name = medication_data.get("drug_name", "").strip()
+            if not drug_name:
+                drug_name = "Medication (see prescription)"
+
+            # Build dosage text
+            dosage_text = self._format_dosage_text(medication_data)
+            if not dosage_text or dosage_text == "As directed":
+                dosage_text = "See prescription for details"
+
+            # Build dosage with optional route
+            route_text = medication_data.get("route", "").strip()
+            dosage_kwargs = {"text": dosage_text}
+            if route_text:
+                dosage_kwargs["route"] = CodeableConcept(text=route_text)
+
+            statement = MedicationStatement(
+                id=str(uuid4()),
+                status="active",
+                medicationCodeableConcept=CodeableConcept(text=drug_name),
+                subject=Reference(reference="Patient/example"),
+                effectiveDateTime=datetime.now().isoformat(),
+                dosage=[Dosage(**dosage_kwargs)],
+            )
+
+            return statement.json(indent=2)
+
+        except Exception as e:
+            logger.error(f"Failed to generate MedicationStatement: {e}")
+            raise ValueError("Could not generate FHIR MedicationStatement.")
+
+    def generate_medication_request(
+        self, medication_data: dict[str, Any]
+    ) -> str:
+        """
+        Generate a FHIR MedicationRequest resource.
+
+        Args:
+            medication_data: Dictionary with medication info
+                - drug_name: Name of the medication
+                - dosage: Dosage amount and unit
+                - frequency: How often to take
+                - duration: How long to take
+                - route: Route of administration (optional)
+                - notes: Additional instructions (optional)
+
+        Returns:
+            JSON string of FHIR MedicationRequest
+        """
+        try:
+            # Ensure drug_name is non-empty
+            drug_name = medication_data.get("drug_name", "").strip()
+            if not drug_name:
+                drug_name = "Medication (see prescription)"
+
+            # Build dosage text
+            dosage_text = self._format_dosage_text(medication_data)
+            if not dosage_text or dosage_text == "As directed":
+                dosage_text = "See prescription for details"
+
+            # Build dosage with optional route
+            route_text = medication_data.get("route", "").strip()
+            dosage_kwargs = {"text": dosage_text}
+            if route_text:
+                dosage_kwargs["route"] = CodeableConcept(text=route_text)
+
+            request = MedicationRequest(
+                id=str(uuid4()),
+                status="active",
+                intent="order",
+                medicationCodeableConcept=CodeableConcept(text=drug_name),
+                subject=Reference(reference="Patient/example"),
+                authoredOn=datetime.now().isoformat(),
+                dosageInstruction=[Dosage(**dosage_kwargs)],
+            )
+
+            return request.json(indent=2)
+
+        except Exception as e:
+            logger.error(f"Failed to generate MedicationRequest: {e}")
+            raise ValueError("Could not generate FHIR MedicationRequest.")
+
+    def _format_dosage_text(self, medication_data: dict[str, Any]) -> str:
+        """Format dosage information as human-readable text."""
+        parts = []
+
+        if dosage := medication_data.get("dosage"):
+            parts.append(dosage)
+
+        if frequency := medication_data.get("frequency"):
+            parts.append(frequency)
+
+        if duration := medication_data.get("duration"):
+            parts.append(f"for {duration}")
+
+        if notes := medication_data.get("notes"):
+            parts.append(f"({notes})")
+
+        return " ".join(parts) if parts else "As directed"
+
+
+def parse_medications_to_dict(extraction_text: str) -> list[dict[str, Any]]:
+    """
+    Parse extracted medication text into structured dictionaries.
+
+    Args:
+        extraction_text: Raw text from MedGemma extraction
+
+    Returns:
+        List of medication dictionaries
+    """
+    # Simple parsing - in production, would use more robust NLP
+    medications = []
+
+    lines = extraction_text.strip().split("\n")
+    current_med = {}
+
+    for line in lines:
+        line = line.strip()
+        if not line:
+            if current_med:
+                medications.append(current_med)
+                current_med = {}
+            continue
+
+        line_lower = line.lower()
+
+        if "drug" in line_lower or "medication" in line_lower or "name:" in line_lower:
+            if current_med:
+                medications.append(current_med)
+            current_med = {"drug_name": line.split(":", 1)[-1].strip()}
+        elif "dosage" in line_lower or "dose:" in line_lower:
+            current_med["dosage"] = line.split(":", 1)[-1].strip()
+        elif "frequency" in line_lower or "times" in line_lower:
+            current_med["frequency"] = line.split(":", 1)[-1].strip()
+        elif "duration" in line_lower or "days" in line_lower:
+            current_med["duration"] = line.split(":", 1)[-1].strip()
+        elif "route" in line_lower:
+            current_med["route"] = line.split(":", 1)[-1].strip()
+        elif "instruction" in line_lower or "note" in line_lower:
+            current_med["notes"] = line.split(":", 1)[-1].strip()
+
+    if current_med:
+        medications.append(current_med)
+
+    return medications if medications else [{"drug_name": "See prescription details", "notes": extraction_text}]

src/medgemma_service.py

@@ -0,0 +1,142 @@
+"""MedGemma service for prescription extraction and explanation."""
+
+import logging
+from typing import Optional
+
+import torch
+from PIL import Image
+from transformers import AutoModelForImageTextToText, AutoProcessor
+
+from .constants import MEDGEMMA_MODEL_ID
+from .prompts import EXTRACTION_PROMPT, EXPLANATION_PROMPT
+
+logger = logging.getLogger(__name__)
+
+
+class MedGemmaService:
+    """Service for extracting medications from prescription images using MedGemma."""
+
+    def __init__(self, model, processor):
+        """Initialize with pre-loaded model and processor."""
+        self.model = model
+        self.processor = processor
+
+    def extract_medications(self, image: Image.Image) -> str:
+        """
+        Extract medication information from a prescription image.
+
+        Args:
+            image: PIL Image of the prescription
+
+        Returns:
+            Extracted medication information as text
+
+        Raises:
+            ValueError: If extraction fails
+        """
+        try:
+            messages = [
+                {
+                    "role": "user",
+                    "content": [
+                        {"type": "image", "image": image},
+                        {"type": "text", "text": EXTRACTION_PROMPT},
+                    ],
+                }
+            ]
+
+            inputs = self.processor.apply_chat_template(
+                messages,
+                add_generation_prompt=True,
+                tokenize=True,
+                return_dict=True,
+                return_tensors="pt",
+            ).to(self.model.device)
+
+            input_len = inputs["input_ids"].shape[-1]
+
+            with torch.inference_mode():
+                outputs = self.model.generate(
+                    **inputs,
+                    max_new_tokens=1024,
+                    do_sample=False,
+                )
+
+            response = self.processor.decode(
+                outputs[0][input_len:], skip_special_tokens=True
+            )
+
+            return response.strip()
+
+        except Exception as e:
+            logger.error(f"Medication extraction failed: {e}")
+            raise ValueError(
+                "Could not read the prescription. Please try uploading a clearer image."
+            )
+
+    def generate_explanation(self, medication_info: str) -> str:
+        """
+        Generate a plain-language explanation of medications.
+
+        Args:
+            medication_info: Extracted medication information
+
+        Returns:
+            Patient-friendly explanation
+
+        Raises:
+            ValueError: If explanation generation fails
+        """
+        try:
+            prompt = EXPLANATION_PROMPT.format(medication_info=medication_info)
+
+            messages = [{"role": "user", "content": prompt}]
+
+            inputs = self.processor.apply_chat_template(
+                messages,
+                add_generation_prompt=True,
+                tokenize=True,
+                return_dict=True,
+                return_tensors="pt",
+            ).to(self.model.device)
+
+            input_len = inputs["input_ids"].shape[-1]
+
+            with torch.inference_mode():
+                outputs = self.model.generate(
+                    **inputs,
+                    max_new_tokens=1024,
+                    do_sample=False,
+                )
+
+            response = self.processor.decode(
+                outputs[0][input_len:], skip_special_tokens=True
+            )
+
+            return response.strip()
+
+        except Exception as e:
+            logger.error(f"Explanation generation failed: {e}", exc_info=True)
+            raise ValueError(
+                f"Could not generate explanation: {str(e)}"
+            )
+
+
+def load_medgemma_model():
+    """
+    Load MedGemma model and processor.
+
+    Returns:
+        Tuple of (model, processor)
+    """
+    logger.info(f"Loading MedGemma model: {MEDGEMMA_MODEL_ID}")
+
+    processor = AutoProcessor.from_pretrained(MEDGEMMA_MODEL_ID)
+    model = AutoModelForImageTextToText.from_pretrained(
+        MEDGEMMA_MODEL_ID,
+        torch_dtype=torch.bfloat16,
+        device_map="auto",
+    )
+
+    logger.info("MedGemma model loaded successfully")
+    return model, processor

src/prompts.py

@@ -0,0 +1,41 @@
+"""Centralized prompt templates for Prescription Explainer."""
+
+EXTRACTION_PROMPT = """You are a medical AI assistant. Analyze this prescription image and extract all medication information.
+
+For each medication found, provide:
+1. Drug name (generic and brand if visible)
+2. Dosage (amount and unit)
+3. Frequency (how often to take)
+4. Duration (how long to take)
+5. Route (oral, topical, etc.)
+6. Special instructions (if any)
+
+Format your response as a structured list. If you cannot read part of the prescription clearly, indicate that.
+
+Extract the medications from this prescription image:"""
+
+EXPLANATION_PROMPT = """You are a friendly healthcare assistant explaining medications to patients in simple terms.
+
+Given this medication information:
+{medication_info}
+
+Provide a clear, easy-to-understand explanation that includes:
+1. What each medication is for (in simple terms)
+2. How to take it correctly
+3. Important things to remember
+4. Common side effects to watch for (if applicable)
+
+Use simple language that anyone can understand. Avoid medical jargon.
+Be encouraging and supportive in your tone."""
+
+TRANSLATION_PROMPT = """Translate the following healthcare information to {target_language}.
+
+Keep the translation:
+- Accurate and faithful to the original meaning
+- Easy to understand for patients
+- Culturally appropriate
+
+Text to translate:
+{text}
+
+Provide only the translation, no explanations."""

src/translation_service.py

@@ -0,0 +1,119 @@
+"""Translation service using Gemma 3 for multilingual support."""
+
+import logging
+
+import torch
+from transformers import AutoModelForCausalLM, AutoTokenizer
+
+from .constants import SUPPORTED_LANGUAGES, TRANSLATION_MODEL_ID
+from .prompts import TRANSLATION_PROMPT
+
+logger = logging.getLogger(__name__)
+
+
+class TranslationService:
+    """Service for translating text to supported languages using Gemma 3."""
+
+    def __init__(self, model, tokenizer):
+        """Initialize with pre-loaded model and tokenizer."""
+        self.model = model
+        self.tokenizer = tokenizer
+
+    def generate_text(self, prompt: str) -> str:
+        """
+        Generate text using Gemma 3 (for explanations).
+
+        Args:
+            prompt: Text prompt for generation
+
+        Returns:
+            Generated text
+
+        Raises:
+            ValueError: If generation fails
+        """
+        try:
+            messages = [{"role": "user", "content": prompt}]
+
+            inputs = self.tokenizer.apply_chat_template(
+                messages,
+                add_generation_prompt=True,
+                tokenize=True,
+                return_dict=True,
+                return_tensors="pt",
+            ).to(self.model.device)
+
+            input_len = inputs["input_ids"].shape[-1]
+
+            with torch.inference_mode():
+                outputs = self.model.generate(
+                    **inputs,
+                    max_new_tokens=2048,
+                    do_sample=True,
+                    temperature=0.7,
+                )
+
+            response = self.tokenizer.decode(
+                outputs[0][input_len:], skip_special_tokens=True
+            )
+
+            return response.strip()
+
+        except Exception as e:
+            logger.error(f"Text generation failed: {e}")
+            raise ValueError("Could not generate text. Please try again.")
+
+    def translate_text(self, text: str, target_language: str) -> str:
+        """
+        Translate text to the target language.
+
+        Args:
+            text: Text to translate (in English)
+            target_language: Target language display name (e.g., "Thai")
+
+        Returns:
+            Translated text
+
+        Raises:
+            ValueError: If translation fails or language not supported
+        """
+        if target_language not in SUPPORTED_LANGUAGES:
+            raise ValueError(f"Unsupported language: {target_language}")
+
+        # If target is English, no translation needed
+        if target_language == "English":
+            return text
+
+        try:
+            prompt = TRANSLATION_PROMPT.format(
+                target_language=target_language,
+                text=text,
+            )
+
+            return self.generate_text(prompt)
+
+        except Exception as e:
+            logger.error(f"Translation to {target_language} failed: {e}")
+            raise ValueError(
+                f"Could not translate to {target_language}. Please try again."
+            )
+
+
+def load_translation_model():
+    """
+    Load Gemma 3 model for translation.
+
+    Returns:
+        Tuple of (model, tokenizer)
+    """
+    logger.info(f"Loading translation model: {TRANSLATION_MODEL_ID}")
+
+    tokenizer = AutoTokenizer.from_pretrained(TRANSLATION_MODEL_ID)
+    model = AutoModelForCausalLM.from_pretrained(
+        TRANSLATION_MODEL_ID,
+        torch_dtype=torch.bfloat16,
+        device_map="auto",
+    )
+
+    logger.info("Translation model loaded successfully")
+    return model, tokenizer