Spaces:
Running
Running
Fix delete errors and add startup timing logs
Browse files- Wrap delete_player_from_hf and delete_session_from_hf in try/except
so a failed HF folder deletion no longer causes the endpoint to
return 500 after local deletion already succeeded
- Add [SPORALIZE]-prefixed timing logs to startup, weight resolution,
storage sync, and pipeline execution so cold-start slowness is
visible in HF Spaces logs
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
api.py
CHANGED
|
@@ -115,6 +115,8 @@ def sync_storage_from_hf(force: bool = False):
|
|
| 115 |
if not force and (now - _storage_last_sync_ts) < STORAGE_SYNC_INTERVAL_SECONDS:
|
| 116 |
return
|
| 117 |
|
|
|
|
|
|
|
| 118 |
sync_cache_root = os.path.join(RUNTIME_ROOT, "storage-sync-cache")
|
| 119 |
os.makedirs(sync_cache_root, exist_ok=True)
|
| 120 |
local_repo_dir = os.path.join(sync_cache_root, safe_name(STORAGE_DATASET_REPO_ID))
|
|
@@ -133,6 +135,7 @@ def sync_storage_from_hf(force: bool = False):
|
|
| 133 |
shutil.rmtree(STORAGE_ROOT, ignore_errors=True)
|
| 134 |
shutil.copytree(source_storage, STORAGE_ROOT, dirs_exist_ok=True)
|
| 135 |
_storage_last_sync_ts = time.time()
|
|
|
|
| 136 |
|
| 137 |
|
| 138 |
def push_session_to_hf(player_id: str, session_id: str, session_dir: str):
|
|
@@ -284,28 +287,36 @@ def load_pipeline_callable(pipeline_root: str):
|
|
| 284 |
def ensure_weight_file(spec: dict, pipeline_root: str):
|
| 285 |
override_path = os.environ.get(spec["override_env"])
|
| 286 |
if override_path and os.path.isfile(override_path):
|
|
|
|
| 287 |
return override_path
|
| 288 |
|
| 289 |
pipeline_weight = os.path.join(pipeline_root, "Weights", spec["filename"])
|
| 290 |
if os.path.isfile(pipeline_weight):
|
|
|
|
| 291 |
return pipeline_weight
|
| 292 |
|
| 293 |
local_fallback = spec.get("local_fallback")
|
| 294 |
if local_fallback and os.path.isfile(local_fallback):
|
|
|
|
| 295 |
return local_fallback
|
| 296 |
|
| 297 |
os.makedirs(WEIGHTS_RUNTIME_ROOT, exist_ok=True)
|
| 298 |
cached_path = os.path.join(WEIGHTS_RUNTIME_ROOT, spec["filename"])
|
| 299 |
if os.path.isfile(cached_path):
|
|
|
|
| 300 |
return cached_path
|
| 301 |
|
| 302 |
-
|
|
|
|
|
|
|
| 303 |
repo_id=spec["repo_id"],
|
| 304 |
repo_type=spec.get("repo_type", "model"),
|
| 305 |
filename=spec["repo_file"],
|
| 306 |
token=get_hf_token(),
|
| 307 |
local_dir=WEIGHTS_RUNTIME_ROOT,
|
| 308 |
)
|
|
|
|
|
|
|
| 309 |
|
| 310 |
|
| 311 |
def ensure_runtime_ready(force: bool = False):
|
|
@@ -995,8 +1006,14 @@ def get_cors_origins():
|
|
| 995 |
|
| 996 |
@app.on_event("startup")
|
| 997 |
def startup_event():
|
|
|
|
|
|
|
| 998 |
ensure_runtime_ready()
|
|
|
|
|
|
|
| 999 |
sync_storage_from_hf(force=True)
|
|
|
|
|
|
|
| 1000 |
|
| 1001 |
|
| 1002 |
@app.get("/healthz")
|
|
@@ -1113,7 +1130,10 @@ def delete_session(session_id: str):
|
|
| 1113 |
player_dir = os.path.dirname(session_dir)
|
| 1114 |
player_id = os.path.basename(player_dir)
|
| 1115 |
shutil.rmtree(session_dir, ignore_errors=True)
|
| 1116 |
-
|
|
|
|
|
|
|
|
|
|
| 1117 |
|
| 1118 |
if os.path.isdir(player_dir) and not os.listdir(player_dir):
|
| 1119 |
os.rmdir(player_dir)
|
|
@@ -1128,7 +1148,10 @@ def delete_player(player_id: str):
|
|
| 1128 |
raise HTTPException(status_code=404, detail="Player storage not found")
|
| 1129 |
|
| 1130 |
shutil.rmtree(player_dir, ignore_errors=True)
|
| 1131 |
-
|
|
|
|
|
|
|
|
|
|
| 1132 |
return {"status": "deleted", "playerId": player_id}
|
| 1133 |
|
| 1134 |
cors_origins = get_cors_origins()
|
|
@@ -1627,7 +1650,8 @@ def run_analysis_job(
|
|
| 1627 |
|
| 1628 |
ensure_not_cancelled()
|
| 1629 |
|
| 1630 |
-
print(f"
|
|
|
|
| 1631 |
set_progress(client_id, 20.0, "Extracting 3D Kinematics", status="running", sessionId=session_id)
|
| 1632 |
|
| 1633 |
def progress_tracker(current_act, total_act, step, total_frames):
|
|
@@ -1650,6 +1674,7 @@ def run_analysis_job(
|
|
| 1650 |
return True
|
| 1651 |
|
| 1652 |
reports = runtime["run_pipeline"](camera_map, utils_paths, sizes, progress_tracker)
|
|
|
|
| 1653 |
ensure_not_cancelled()
|
| 1654 |
|
| 1655 |
set_progress(client_id, 92.0, "Preparing Playback Assets", status="running", sessionId=session_id)
|
|
|
|
| 115 |
if not force and (now - _storage_last_sync_ts) < STORAGE_SYNC_INTERVAL_SECONDS:
|
| 116 |
return
|
| 117 |
|
| 118 |
+
print(f"[SPORALIZE] Storage sync: downloading from HF repo '{STORAGE_DATASET_REPO_ID}'...", flush=True)
|
| 119 |
+
t0 = time.time()
|
| 120 |
sync_cache_root = os.path.join(RUNTIME_ROOT, "storage-sync-cache")
|
| 121 |
os.makedirs(sync_cache_root, exist_ok=True)
|
| 122 |
local_repo_dir = os.path.join(sync_cache_root, safe_name(STORAGE_DATASET_REPO_ID))
|
|
|
|
| 135 |
shutil.rmtree(STORAGE_ROOT, ignore_errors=True)
|
| 136 |
shutil.copytree(source_storage, STORAGE_ROOT, dirs_exist_ok=True)
|
| 137 |
_storage_last_sync_ts = time.time()
|
| 138 |
+
print(f"[SPORALIZE] Storage sync: done in {time.time() - t0:.1f}s", flush=True)
|
| 139 |
|
| 140 |
|
| 141 |
def push_session_to_hf(player_id: str, session_id: str, session_dir: str):
|
|
|
|
| 287 |
def ensure_weight_file(spec: dict, pipeline_root: str):
|
| 288 |
override_path = os.environ.get(spec["override_env"])
|
| 289 |
if override_path and os.path.isfile(override_path):
|
| 290 |
+
print(f"[SPORALIZE] Weight '{spec['filename']}': using env override at {override_path}", flush=True)
|
| 291 |
return override_path
|
| 292 |
|
| 293 |
pipeline_weight = os.path.join(pipeline_root, "Weights", spec["filename"])
|
| 294 |
if os.path.isfile(pipeline_weight):
|
| 295 |
+
print(f"[SPORALIZE] Weight '{spec['filename']}': found in pipeline bundle", flush=True)
|
| 296 |
return pipeline_weight
|
| 297 |
|
| 298 |
local_fallback = spec.get("local_fallback")
|
| 299 |
if local_fallback and os.path.isfile(local_fallback):
|
| 300 |
+
print(f"[SPORALIZE] Weight '{spec['filename']}': using local fallback at {local_fallback}", flush=True)
|
| 301 |
return local_fallback
|
| 302 |
|
| 303 |
os.makedirs(WEIGHTS_RUNTIME_ROOT, exist_ok=True)
|
| 304 |
cached_path = os.path.join(WEIGHTS_RUNTIME_ROOT, spec["filename"])
|
| 305 |
if os.path.isfile(cached_path):
|
| 306 |
+
print(f"[SPORALIZE] Weight '{spec['filename']}': cache hit at {cached_path}", flush=True)
|
| 307 |
return cached_path
|
| 308 |
|
| 309 |
+
print(f"[SPORALIZE] Weight '{spec['filename']}': not cached, downloading from HF repo '{spec['repo_id']}'...", flush=True)
|
| 310 |
+
t0 = time.time()
|
| 311 |
+
result = hf_hub_download(
|
| 312 |
repo_id=spec["repo_id"],
|
| 313 |
repo_type=spec.get("repo_type", "model"),
|
| 314 |
filename=spec["repo_file"],
|
| 315 |
token=get_hf_token(),
|
| 316 |
local_dir=WEIGHTS_RUNTIME_ROOT,
|
| 317 |
)
|
| 318 |
+
print(f"[SPORALIZE] Weight '{spec['filename']}': downloaded in {time.time() - t0:.1f}s -> {result}", flush=True)
|
| 319 |
+
return result
|
| 320 |
|
| 321 |
|
| 322 |
def ensure_runtime_ready(force: bool = False):
|
|
|
|
| 1006 |
|
| 1007 |
@app.on_event("startup")
|
| 1008 |
def startup_event():
|
| 1009 |
+
print("[SPORALIZE] Server startup: initializing runtime...", flush=True)
|
| 1010 |
+
t0 = time.time()
|
| 1011 |
ensure_runtime_ready()
|
| 1012 |
+
print(f"[SPORALIZE] Runtime ready in {time.time() - t0:.1f}s (pipeline_source={runtime_state.get('pipeline_source')})", flush=True)
|
| 1013 |
+
t1 = time.time()
|
| 1014 |
sync_storage_from_hf(force=True)
|
| 1015 |
+
print(f"[SPORALIZE] Storage sync done in {time.time() - t1:.1f}s", flush=True)
|
| 1016 |
+
print(f"[SPORALIZE] Startup complete in {time.time() - t0:.1f}s total", flush=True)
|
| 1017 |
|
| 1018 |
|
| 1019 |
@app.get("/healthz")
|
|
|
|
| 1130 |
player_dir = os.path.dirname(session_dir)
|
| 1131 |
player_id = os.path.basename(player_dir)
|
| 1132 |
shutil.rmtree(session_dir, ignore_errors=True)
|
| 1133 |
+
try:
|
| 1134 |
+
delete_session_from_hf(player_id, session_id)
|
| 1135 |
+
except Exception:
|
| 1136 |
+
pass
|
| 1137 |
|
| 1138 |
if os.path.isdir(player_dir) and not os.listdir(player_dir):
|
| 1139 |
os.rmdir(player_dir)
|
|
|
|
| 1148 |
raise HTTPException(status_code=404, detail="Player storage not found")
|
| 1149 |
|
| 1150 |
shutil.rmtree(player_dir, ignore_errors=True)
|
| 1151 |
+
try:
|
| 1152 |
+
delete_player_from_hf(player_id)
|
| 1153 |
+
except Exception:
|
| 1154 |
+
pass
|
| 1155 |
return {"status": "deleted", "playerId": player_id}
|
| 1156 |
|
| 1157 |
cors_origins = get_cors_origins()
|
|
|
|
| 1650 |
|
| 1651 |
ensure_not_cancelled()
|
| 1652 |
|
| 1653 |
+
print(f"[SPORALIZE] Pipeline start: session={session_id} target_size={target_w}x{target_h} cameras={list(camera_map.keys())}", flush=True)
|
| 1654 |
+
_pipeline_t0 = time.time()
|
| 1655 |
set_progress(client_id, 20.0, "Extracting 3D Kinematics", status="running", sessionId=session_id)
|
| 1656 |
|
| 1657 |
def progress_tracker(current_act, total_act, step, total_frames):
|
|
|
|
| 1674 |
return True
|
| 1675 |
|
| 1676 |
reports = runtime["run_pipeline"](camera_map, utils_paths, sizes, progress_tracker)
|
| 1677 |
+
print(f"[SPORALIZE] Pipeline done: session={session_id} elapsed={time.time() - _pipeline_t0:.1f}s actions={len(reports)}", flush=True)
|
| 1678 |
ensure_not_cancelled()
|
| 1679 |
|
| 1680 |
set_progress(client_id, 92.0, "Preparing Playback Assets", status="running", sessionId=session_id)
|