Open LLM Distribution Leaderboard
The first bucket snapshot is being generated. Refresh this page in a few minutes.
#!/usr/bin/env python3 """Bucket-backed HTTP app for the Open LLM Distribution Leaderboard Space.""" from __future__ import annotations import json import os import threading import time from contextlib import asynccontextmanager from pathlib import Path from typing import Any from fastapi import FastAPI, Header, HTTPException from fastapi.responses import HTMLResponse, JSONResponse, Response from leaderboard_core import DEFAULT_MAX_AGE_DAYS, DEFAULT_MIN_DOWNLOADS, DEFAULT_TOP_N, refresh DATA_DIR = Path(os.environ.get("OPEN_MODEL_BUCKET_DIR", "/data")) LOCAL_FALLBACK_DIR = Path(os.environ.get("OPEN_MODEL_LOCAL_DATA_DIR", "data")) DEFAULT_BUCKET_URI = "hf://buckets/osolmaz/leaderboard" LATEST_FILES = ( "groups.csv", "groups.json", "index.html", "leaderboard.json", "models.csv", "models.jsonl", "summary.json", ) REFRESH_LOCK = threading.Lock() SNAPSHOT_LOCK = threading.Lock() STOP_EVENT = threading.Event() LAST_SNAPSHOT_FETCH_AT = 0.0 def snapshot_ready(root: Path) -> bool: latest = root / "latest" return (latest / "index.html").exists() and (latest / "leaderboard.json").exists() def data_dir() -> Path: if snapshot_ready(DATA_DIR): return DATA_DIR if snapshot_ready(LOCAL_FALLBACK_DIR): return LOCAL_FALLBACK_DIR if DATA_DIR.exists(): return DATA_DIR return LOCAL_FALLBACK_DIR def int_env(name: str, default: int) -> int: raw = os.environ.get(name) if raw is None or raw == "": return default return int(raw) def optional_int_env(name: str) -> int | None: raw = os.environ.get(name) if raw is None or raw == "": return None return int(raw) def truthy_env(name: str, default: bool) -> bool: raw = os.environ.get(name) if raw is None: return default return raw.lower() in {"1", "true", "yes", "on"} def latest_dir() -> Path: return data_dir() / "latest" def latest_html() -> Path: return latest_dir() / "index.html" def latest_json() -> Path: return latest_dir() / "leaderboard.json" def bucket_uri() -> str: return os.environ.get("OPEN_MODEL_BUCKET_URI", DEFAULT_BUCKET_URI).rstrip("/") def snapshot_fetch_interval_seconds() -> int: raw = os.environ.get("OPEN_MODEL_BUCKET_FETCH_RETRY_SECONDS", "300") return max(int(raw), 0) def fetch_latest_from_bucket(force: bool = False) -> None: global LAST_SNAPSHOT_FETCH_AT if snapshot_ready(DATA_DIR) or snapshot_ready(LOCAL_FALLBACK_DIR): return uri = bucket_uri() if not uri: return now = time.time() if not force and now - LAST_SNAPSHOT_FETCH_AT < snapshot_fetch_interval_seconds(): return with SNAPSHOT_LOCK: if snapshot_ready(DATA_DIR) or snapshot_ready(LOCAL_FALLBACK_DIR): return now = time.time() if not force and now - LAST_SNAPSHOT_FETCH_AT < snapshot_fetch_interval_seconds(): return LAST_SNAPSHOT_FETCH_AT = now try: from huggingface_hub import HfFileSystem token = os.environ.get("HF_TOKEN") or os.environ.get("HUGGING_FACE_HUB_TOKEN") fs = HfFileSystem(token=token) if token else HfFileSystem() target = LOCAL_FALLBACK_DIR / "latest" target.mkdir(parents=True, exist_ok=True) for filename in LATEST_FILES: destination = target / filename temporary = target / f".{filename}.tmp" fs.get(f"{uri}/latest/{filename}", str(temporary)) temporary.replace(destination) print(f"fetched latest snapshot from {uri}", flush=True) except Exception as err: # noqa: BLE001 print(f"failed to fetch latest snapshot from {uri}: {err}", flush=True) def read_json(path: Path) -> dict[str, Any]: if not path.exists(): return {} try: return json.loads(path.read_text(encoding="utf-8")) except Exception as err: # noqa: BLE001 print(f"failed to read json {path}: {err}", flush=True) return {} def file_size(path: Path) -> int | None: try: return path.stat().st_size except OSError: return None def run_refresh() -> dict[str, Any]: token = os.environ.get("HF_TOKEN") or os.environ.get("HUGGING_FACE_HUB_TOKEN") with REFRESH_LOCK: return refresh( output_dir=data_dir(), min_downloads=int_env("OPEN_MODEL_MIN_DOWNLOADS", DEFAULT_MIN_DOWNLOADS), top_n=int_env("OPEN_MODEL_TOP_N", DEFAULT_TOP_N), max_age_days=int_env("OPEN_MODEL_MAX_AGE_DAYS", DEFAULT_MAX_AGE_DAYS), limit_per_creator=optional_int_env("OPEN_MODEL_LIMIT_PER_CREATOR"), token=token, ) def refresh_if_missing_or_forced(force: bool) -> None: fetch_latest_from_bucket(force=False) if not force and latest_html().exists() and latest_json().exists(): return try: run_refresh() except Exception as err: # noqa: BLE001 print(f"refresh failed: {err}", flush=True) def refresh_loop() -> None: interval = int_env("OPEN_MODEL_REFRESH_INTERVAL_SECONDS", 86_400) refresh_if_missing_or_forced(force=truthy_env("OPEN_MODEL_REFRESH_ON_STARTUP", True)) while not STOP_EVENT.wait(interval): refresh_if_missing_or_forced(force=True) @asynccontextmanager async def lifespan(_: FastAPI): thread: threading.Thread | None = None fetch_latest_from_bucket(force=True) if truthy_env("OPEN_MODEL_ENABLE_SCHEDULER", True): thread = threading.Thread(target=refresh_loop, name="leaderboard-refresh", daemon=True) thread.start() yield STOP_EVENT.set() if thread and thread.is_alive(): thread.join(timeout=3) APP_TITLE = "Open LLM Distribution Leaderboard" app = FastAPI(title=APP_TITLE, lifespan=lifespan) def loading_page() -> str: return """
The first bucket snapshot is being generated. Refresh this page in a few minutes.