from __future__ import annotations import json import os from datetime import datetime, timezone from io import BytesIO from pathlib import Path from typing import Any from huggingface_hub import HfApi, hf_hub_download, list_repo_files BASE_DIR = Path(__file__).resolve().parent SEED_PATH = BASE_DIR / "data" / "leaderboard.json" LOCAL_RUNTIME_DIR = BASE_DIR / "runtime_submissions" def _dataset_config() -> tuple[str, str] | None: repo_id = os.getenv("LEADERBOARD_DATASET_REPO", "").strip() token = os.getenv("HF_TOKEN", "").strip() return (repo_id, token) if repo_id and token else None def _read_json(path: str | Path) -> Any: return json.loads(Path(path).read_text(encoding="utf-8")) def load_leaderboard() -> list[dict[str, Any]]: records = list(_read_json(SEED_PATH)) config = _dataset_config() if config: repo_id, token = config try: files = list_repo_files(repo_id, repo_type="dataset", token=token) for filename in files: if not filename.startswith("results/") or not filename.endswith(".json"): continue local_path = hf_hub_download( repo_id, filename, repo_type="dataset", token=token, ) record = _read_json(local_path) if record.get("status") == "accepted": records.append(record) except Exception: pass elif LOCAL_RUNTIME_DIR.exists(): for path in LOCAL_RUNTIME_DIR.glob("results/*.json"): record = _read_json(path) if record.get("status") == "accepted": records.append(record) def score(item: dict[str, Any]) -> float: value = item.get("f1_lm", item.get("overall")) try: return float(value) except (TypeError, ValueError): return float("-inf") return sorted( records, key=lambda item: ( -score(item), str(item.get("model", "")).lower(), ), ) def queue_submission(metadata: dict[str, Any], prediction_path: str | Path) -> None: metadata = { **metadata, "status": "pending", "submitted_at": datetime.now(timezone.utc).isoformat(), } submission_id = metadata["submission_id"] prediction_target = f"predictions/{submission_id}.json" metadata_target = f"pending/{submission_id}.json" config = _dataset_config() if config: repo_id, token = config api = HfApi(token=token) api.upload_file( path_or_fileobj=str(prediction_path), path_in_repo=prediction_target, repo_id=repo_id, repo_type="dataset", commit_message=f"Add predictions for {submission_id}", ) api.upload_file( path_or_fileobj=BytesIO( json.dumps(metadata, ensure_ascii=False, indent=2).encode("utf-8") ), path_in_repo=metadata_target, repo_id=repo_id, repo_type="dataset", commit_message=f"Queue submission {submission_id}", ) return (LOCAL_RUNTIME_DIR / "predictions").mkdir(parents=True, exist_ok=True) (LOCAL_RUNTIME_DIR / "pending").mkdir(parents=True, exist_ok=True) destination = LOCAL_RUNTIME_DIR / prediction_target destination.write_bytes(Path(prediction_path).read_bytes()) (LOCAL_RUNTIME_DIR / metadata_target).write_text( json.dumps(metadata, ensure_ascii=False, indent=2), encoding="utf-8", ) def load_pending_submission(submission_id: str) -> dict[str, Any]: filename = f"pending/{submission_id}.json" config = _dataset_config() if config: repo_id, token = config local_path = hf_hub_download(repo_id, filename, repo_type="dataset", token=token) return _read_json(local_path) return _read_json(LOCAL_RUNTIME_DIR / filename) def publish_result(record: dict[str, Any]) -> None: submission_id = record["submission_id"] filename = f"results/{submission_id}.json" payload = json.dumps(record, ensure_ascii=False, indent=2).encode("utf-8") config = _dataset_config() if config: repo_id, token = config HfApi(token=token).upload_file( path_or_fileobj=BytesIO(payload), path_in_repo=filename, repo_id=repo_id, repo_type="dataset", commit_message=f"Publish result {submission_id}", ) return destination = LOCAL_RUNTIME_DIR / filename destination.parent.mkdir(parents=True, exist_ok=True) destination.write_bytes(payload)