samwell Claude commited on
Commit
8dd90e9
·
1 Parent(s): 37bdbfa

feat: Enable RoentGen-v2 synthetic X-ray generation

Browse files

Added RoentGen-v2 (Stanford MIMI) for generating synthetic chest X-rays
from text descriptions.

Changes:
- Updated xray_generation.py to use stanfordmimi/RoentGen-v2 from HF
- Changed from fp32 to fp16 for memory efficiency
- Added tool loading in app.py for GPU environments
- Now publicly available (no need for manual weight download)

Usage examples:
- "Generate an X-ray showing pneumonia in the right lower lobe"
- "Create a chest X-ray with left-sided pleural effusion"
- "Generate a normal chest radiograph"

Tool count: 7 → 8 (with RoentGen-v2 generation)

Paper: https://arxiv.org/abs/2508.16783
Model: https://huggingface.co/stanfordmimi/RoentGen-v2

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>

Files changed (2) hide show
  1. app.py +14 -0
  2. medrax/tools/xray_generation.py +10 -4
app.py CHANGED
@@ -126,6 +126,20 @@ try:
126
  except Exception as e:
127
  print(f"✗ Failed to load web browsing tool: {e}")
128
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
129
  checkpointer = MemorySaver()
130
 
131
  llm = ModelFactory.create_model(
 
126
  except Exception as e:
127
  print(f"✗ Failed to load web browsing tool: {e}")
128
 
129
+ # RoentGen-v2 X-ray Generation (GPU only)
130
+ if device == "cuda":
131
+ try:
132
+ from medrax.tools.xray_generation import ChestXRayGeneratorTool
133
+ generation_tool = ChestXRayGeneratorTool(
134
+ model_path="stanfordmimi/RoentGen-v2",
135
+ temp_dir="temp",
136
+ device=device
137
+ )
138
+ tools.append(generation_tool)
139
+ print("✓ Loaded RoentGen-v2 X-ray generation tool")
140
+ except Exception as e:
141
+ print(f"✗ Failed to load RoentGen-v2 tool: {e}")
142
+
143
  checkpointer = MemorySaver()
144
 
145
  llm = ModelFactory.create_model(
medrax/tools/xray_generation.py CHANGED
@@ -42,8 +42,8 @@ class ChestXRayGeneratorTool(BaseTool):
42
 
43
  def __init__(
44
  self,
45
- model_path: str = "/model-weights/roentgen",
46
- cache_dir: str = "/model-weights",
47
  temp_dir: Optional[str] = None,
48
  device: Optional[str] = "cuda",
49
  ):
@@ -51,8 +51,14 @@ class ChestXRayGeneratorTool(BaseTool):
51
  super().__init__()
52
 
53
  self.device = torch.device(device) if device else "cuda"
54
- self.model = StableDiffusionPipeline.from_pretrained(model_path, cache_dir=cache_dir)
55
- self.model = self.model.to(torch.float32).to(self.device)
 
 
 
 
 
 
56
 
57
  self.temp_dir = Path(temp_dir if temp_dir else tempfile.mkdtemp())
58
  self.temp_dir.mkdir(exist_ok=True)
 
42
 
43
  def __init__(
44
  self,
45
+ model_path: str = "stanfordmimi/RoentGen-v2",
46
+ cache_dir: Optional[str] = None,
47
  temp_dir: Optional[str] = None,
48
  device: Optional[str] = "cuda",
49
  ):
 
51
  super().__init__()
52
 
53
  self.device = torch.device(device) if device else "cuda"
54
+
55
+ # Load RoentGen-v2 from Hugging Face
56
+ self.model = StableDiffusionPipeline.from_pretrained(
57
+ model_path,
58
+ cache_dir=cache_dir,
59
+ torch_dtype=torch.float16, # Use fp16 for efficiency
60
+ )
61
+ self.model = self.model.to(self.device)
62
 
63
  self.temp_dir = Path(temp_dir if temp_dir else tempfile.mkdtemp())
64
  self.temp_dir.mkdir(exist_ok=True)