Handle Colab meta-tensor load path with automatic device_map fallback

run_standalone_inference.py

@@ -154,6 +154,18 @@ def _top_k_filter(logits, top_k: int):
     return masked
 
 
+def _has_meta_tensors(model) -> bool:
+    import torch
+
+    for p in model.parameters():
+        if isinstance(p, torch.Tensor) and p.is_meta:
+            return True
+    for b in model.buffers():
+        if isinstance(b, torch.Tensor) and b.is_meta:
+            return True
+    return False
+
+
 def generate_compat(model, input_ids, max_new_tokens: int, temperature: float, top_k: int, seed: int):
     import torch
     import torch.nn.functional as F
@@ -212,7 +224,20 @@ def main() -> None:
         low_cpu_mem_usage=bool(device.type == "cuda" and args.use_device_map),
     )
     if device.type == "cpu" or (device.type == "cuda" and not args.use_device_map):
-        model = model.to(device)
+        if _has_meta_tensors(model):
+            print("[warn] Detected meta tensors after load; retrying with device_map='auto' fallback.")
+            del model
+            if torch.cuda.is_available():
+                torch.cuda.empty_cache()
+            model = AutoModelForCausalLM.from_pretrained(
+                local,
+                trust_remote_code=True,
+                torch_dtype=torch_dtype,
+                device_map="auto" if device.type == "cuda" else None,
+                low_cpu_mem_usage=bool(device.type == "cuda"),
+            )
+        else:
+            model = model.to(device)
     model.eval()
 
     cfg = model.config