Spaces:
Running on Zero
Running on Zero
Upload app.py with huggingface_hub
Browse files
app.py
CHANGED
|
@@ -372,6 +372,28 @@ def decode_frames(video_path, target_fps=TARGET_FPS, max_frames=MAX_FRAMES):
|
|
| 372 |
# ---------------------------------------------------------------------------
|
| 373 |
# GPU inference.
|
| 374 |
# ---------------------------------------------------------------------------
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 375 |
def _encode_siglip_images(frames, batch_size=64):
|
| 376 |
feats = []
|
| 377 |
for i in range(0, len(frames), batch_size):
|
|
@@ -379,7 +401,7 @@ def _encode_siglip_images(frames, batch_size=64):
|
|
| 379 |
inputs = siglip_processor(images=batch, return_tensors="pt")
|
| 380 |
pixel_values = inputs["pixel_values"].to("cuda", dtype=torch.bfloat16)
|
| 381 |
with torch.no_grad():
|
| 382 |
-
f = siglip_model.get_image_features(pixel_values=pixel_values)
|
| 383 |
f = f.float()
|
| 384 |
f = f / f.norm(dim=-1, keepdim=True)
|
| 385 |
feats.append(f.cpu())
|
|
@@ -396,7 +418,7 @@ def _encode_siglip_text(query):
|
|
| 396 |
)
|
| 397 |
input_ids = inputs["input_ids"].to("cuda")
|
| 398 |
with torch.no_grad():
|
| 399 |
-
f = siglip_model.get_text_features(input_ids=input_ids)
|
| 400 |
f = f.float()
|
| 401 |
f = f / f.norm(dim=-1, keepdim=True)
|
| 402 |
return f.cpu().squeeze(0) # (D,)
|
|
|
|
| 372 |
# ---------------------------------------------------------------------------
|
| 373 |
# GPU inference.
|
| 374 |
# ---------------------------------------------------------------------------
|
| 375 |
+
def _as_tensor(out):
|
| 376 |
+
"""Coerce a get_*_features() return value to a plain feature tensor.
|
| 377 |
+
|
| 378 |
+
Depending on the transformers version, get_image_features / get_text_features
|
| 379 |
+
may return a tensor directly or a ModelOutput wrapping it.
|
| 380 |
+
"""
|
| 381 |
+
if isinstance(out, torch.Tensor):
|
| 382 |
+
return out
|
| 383 |
+
for attr in ("image_embeds", "text_embeds", "pooler_output", "last_hidden_state"):
|
| 384 |
+
val = getattr(out, attr, None)
|
| 385 |
+
if isinstance(val, torch.Tensor):
|
| 386 |
+
return val
|
| 387 |
+
# Fall back to first tensor element (ModelOutput is tuple-like).
|
| 388 |
+
try:
|
| 389 |
+
first = out[0]
|
| 390 |
+
if isinstance(first, torch.Tensor):
|
| 391 |
+
return first
|
| 392 |
+
except Exception:
|
| 393 |
+
pass
|
| 394 |
+
raise TypeError(f"Cannot extract feature tensor from {type(out)}")
|
| 395 |
+
|
| 396 |
+
|
| 397 |
def _encode_siglip_images(frames, batch_size=64):
|
| 398 |
feats = []
|
| 399 |
for i in range(0, len(frames), batch_size):
|
|
|
|
| 401 |
inputs = siglip_processor(images=batch, return_tensors="pt")
|
| 402 |
pixel_values = inputs["pixel_values"].to("cuda", dtype=torch.bfloat16)
|
| 403 |
with torch.no_grad():
|
| 404 |
+
f = _as_tensor(siglip_model.get_image_features(pixel_values=pixel_values))
|
| 405 |
f = f.float()
|
| 406 |
f = f / f.norm(dim=-1, keepdim=True)
|
| 407 |
feats.append(f.cpu())
|
|
|
|
| 418 |
)
|
| 419 |
input_ids = inputs["input_ids"].to("cuda")
|
| 420 |
with torch.no_grad():
|
| 421 |
+
f = _as_tensor(siglip_model.get_text_features(input_ids=input_ids))
|
| 422 |
f = f.float()
|
| 423 |
f = f / f.norm(dim=-1, keepdim=True)
|
| 424 |
return f.cpu().squeeze(0) # (D,)
|