Spaces:
Paused
feat: Enable RoentGen-v2 synthetic X-ray generation
Browse filesAdded RoentGen-v2 (Stanford MIMI) for generating synthetic chest X-rays
from text descriptions.
Changes:
- Updated xray_generation.py to use stanfordmimi/RoentGen-v2 from HF
- Changed from fp32 to fp16 for memory efficiency
- Added tool loading in app.py for GPU environments
- Now publicly available (no need for manual weight download)
Usage examples:
- "Generate an X-ray showing pneumonia in the right lower lobe"
- "Create a chest X-ray with left-sided pleural effusion"
- "Generate a normal chest radiograph"
Tool count: 7 → 8 (with RoentGen-v2 generation)
Paper: https://arxiv.org/abs/2508.16783
Model: https://huggingface.co/stanfordmimi/RoentGen-v2
🤖 Generated with [Claude Code](https://claude.com/claude-code)
Co-Authored-By: Claude <noreply@anthropic.com>
- app.py +14 -0
- medrax/tools/xray_generation.py +10 -4
|
@@ -126,6 +126,20 @@ try:
|
|
| 126 |
except Exception as e:
|
| 127 |
print(f"✗ Failed to load web browsing tool: {e}")
|
| 128 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 129 |
checkpointer = MemorySaver()
|
| 130 |
|
| 131 |
llm = ModelFactory.create_model(
|
|
|
|
| 126 |
except Exception as e:
|
| 127 |
print(f"✗ Failed to load web browsing tool: {e}")
|
| 128 |
|
| 129 |
+
# RoentGen-v2 X-ray Generation (GPU only)
|
| 130 |
+
if device == "cuda":
|
| 131 |
+
try:
|
| 132 |
+
from medrax.tools.xray_generation import ChestXRayGeneratorTool
|
| 133 |
+
generation_tool = ChestXRayGeneratorTool(
|
| 134 |
+
model_path="stanfordmimi/RoentGen-v2",
|
| 135 |
+
temp_dir="temp",
|
| 136 |
+
device=device
|
| 137 |
+
)
|
| 138 |
+
tools.append(generation_tool)
|
| 139 |
+
print("✓ Loaded RoentGen-v2 X-ray generation tool")
|
| 140 |
+
except Exception as e:
|
| 141 |
+
print(f"✗ Failed to load RoentGen-v2 tool: {e}")
|
| 142 |
+
|
| 143 |
checkpointer = MemorySaver()
|
| 144 |
|
| 145 |
llm = ModelFactory.create_model(
|
|
@@ -42,8 +42,8 @@ class ChestXRayGeneratorTool(BaseTool):
|
|
| 42 |
|
| 43 |
def __init__(
|
| 44 |
self,
|
| 45 |
-
model_path: str = "/
|
| 46 |
-
cache_dir: str =
|
| 47 |
temp_dir: Optional[str] = None,
|
| 48 |
device: Optional[str] = "cuda",
|
| 49 |
):
|
|
@@ -51,8 +51,14 @@ class ChestXRayGeneratorTool(BaseTool):
|
|
| 51 |
super().__init__()
|
| 52 |
|
| 53 |
self.device = torch.device(device) if device else "cuda"
|
| 54 |
-
|
| 55 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 56 |
|
| 57 |
self.temp_dir = Path(temp_dir if temp_dir else tempfile.mkdtemp())
|
| 58 |
self.temp_dir.mkdir(exist_ok=True)
|
|
|
|
| 42 |
|
| 43 |
def __init__(
|
| 44 |
self,
|
| 45 |
+
model_path: str = "stanfordmimi/RoentGen-v2",
|
| 46 |
+
cache_dir: Optional[str] = None,
|
| 47 |
temp_dir: Optional[str] = None,
|
| 48 |
device: Optional[str] = "cuda",
|
| 49 |
):
|
|
|
|
| 51 |
super().__init__()
|
| 52 |
|
| 53 |
self.device = torch.device(device) if device else "cuda"
|
| 54 |
+
|
| 55 |
+
# Load RoentGen-v2 from Hugging Face
|
| 56 |
+
self.model = StableDiffusionPipeline.from_pretrained(
|
| 57 |
+
model_path,
|
| 58 |
+
cache_dir=cache_dir,
|
| 59 |
+
torch_dtype=torch.float16, # Use fp16 for efficiency
|
| 60 |
+
)
|
| 61 |
+
self.model = self.model.to(self.device)
|
| 62 |
|
| 63 |
self.temp_dir = Path(temp_dir if temp_dir else tempfile.mkdtemp())
|
| 64 |
self.temp_dir.mkdir(exist_ok=True)
|