Deploy AISE-Bench leaderboard

.gitignore

@@ -0,0 +1,3 @@
+runtime_submissions/
+__pycache__/
+*.log

README.md

@@ -1,10 +1,12 @@
 ---
-title: AISE Bench Leaderboard
-emoji: 📊
-colorFrom: indigo
+title: AISE-Bench Leaderboard
+emoji: compass
+colorFrom: blue
 colorTo: purple
 sdk: static
 pinned: false
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# AISE-Bench Leaderboard
+
+Static leaderboard page for AISE-Bench.

app.py

@@ -0,0 +1,350 @@
+from __future__ import annotations
+
+import html
+
+import gradio as gr
+
+from storage import load_leaderboard
+
+
+APP_CSS = """
+footer, .api-docs, .show-api, .built-with, [data-testid="api-info"] { display: none !important; }
+.gradio-container {
+  max-width: 100% !important;
+  padding: 18px 24px 24px !important;
+  font-family: Inter, ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif !important;
+  font-size: 14px !important;
+}
+.hero { text-align: center; margin-bottom: 8px; }
+.hero h1 { font-size: 32px !important; line-height: 1.2 !important; margin: 8px 0 6px !important; }
+.hero p { color: #4b5563; font-size: 14px !important; line-height: 1.6 !important; margin: 0 !important; }
+.hero-links { display: flex; justify-content: center; gap: 10px; flex-wrap: wrap; margin-top: 8px; }
+.hero-links a { color: #2563eb; text-decoration: none; }
+.hero-links a:hover { text-decoration: underline; }
+.summary-grid { display: grid; grid-template-columns: repeat(3, minmax(180px, 1fr)); gap: 12px; margin: 14px 0 18px; }
+.summary-card { border: 1px solid #e5e7eb; border-radius: 10px; padding: 14px 16px; background: #ffffff; }
+.summary-card strong { display: block; color: #111827; font-size: 18px; margin-bottom: 3px; }
+.summary-card span { color: #6b7280; font-size: 13px; }
+.submission-guide { max-width: 1100px; font-size: 14px !important; line-height: 1.65 !important; margin: 8px auto 22px; }
+.submission-guide h2 { font-size: 21px !important; margin: 22px 0 8px !important; }
+.submission-guide h3 { font-size: 17px !important; margin: 18px 0 8px !important; }
+.submission-guide p, .submission-guide li { font-size: 14px !important; line-height: 1.65 !important; }
+.submission-guide code { font-size: 13px !important; }
+.submission-guide pre code { font-size: 13px !important; line-height: 1.5 !important; }
+.leaderboard-title h2 { font-size: 24px !important; text-align: center; margin: 18px 0 4px !important; }
+.leaderboard-title p { color: #6b7280; font-size: 14px !important; text-align: center; margin: 0 0 12px !important; }
+.leaderboard-toolbar { display: flex; align-items: center; justify-content: flex-end; margin: 8px 0 10px; }
+.table-meta { color: #6b7280; display: flex; gap: 12px; font-size: 13px; }
+.aise-table-wrap { height: min(68vh, 760px); min-height: 460px; overflow: auto; border: 1px solid #e5e7eb; border-radius: 10px; background: #ffffff; }
+.aise-table { border-collapse: separate; border-spacing: 0; width: 100%; min-width: 1320px; color: #263238; font-size: 13px; }
+.aise-table th, .aise-table td { border-bottom: 1px solid #edf0f3; border-right: 1px solid #edf0f3; padding: 9px 10px; text-align: center; white-space: nowrap; }
+.aise-table th { position: sticky; z-index: 2; background: #f8fafc; color: #111827; font-weight: 650; user-select: none; }
+.aise-table thead tr:first-child th { top: 0; background: #e5e7eb; font-size: 13px; }
+.aise-table thead tr:nth-child(2) th { top: 38px; cursor: pointer; }
+.aise-table th[rowspan="2"] { top: 0; cursor: pointer; }
+.aise-table tbody tr:nth-child(even) { background: #fafafa; }
+.aise-table tbody tr:hover { background: #eef5ff; }
+.aise-table .rank { width: 48px; color: #4b5563; font-variant-numeric: tabular-nums; }
+.aise-table .model { min-width: 190px; text-align: left; white-space: normal; }
+.aise-table .model a, .aise-table .model-name { color: #2563eb; font-size: 14px; font-weight: 650; text-decoration: none; }
+.aise-table .model a:hover { text-decoration: underline; }
+.aise-table .org { display: block; color: #8a8f98; font-size: 12px; margin-top: 3px; }
+.aise-table .num { font-variant-numeric: tabular-nums; }
+.verified-badge { display: inline-flex; align-items: center; border-radius: 999px; font-size: 11px; font-weight: 700; padding: 3px 8px; }
+.verified-yes { background: #dcfce7; color: #166534; }
+.verified-no { background: #f3f4f6; color: #6b7280; }
+.aise-table .empty { padding: 28px; color: #7b8088; }
+.submit-actions button { min-height: 42px !important; font-size: 14px !important; font-weight: 600 !important; }
+.section-title h2 { font-size: 21px !important; line-height: 1.3 !important; margin: 22px 0 8px !important; }
+label, .label-wrap, input, textarea { font-size: 14px !important; }
+@media (max-width: 760px) {
+  .gradio-container { padding: 12px !important; }
+  .summary-grid { grid-template-columns: 1fr; }
+  .table-meta span:last-child { display: none; }
+}
+"""
+
+
+TABLE_JS = """
+() => {
+  const table = document.getElementById("aise-leaderboard-table");
+  if (!table) return;
+
+  const tbody = table.querySelector("tbody");
+  const rows = Array.from(tbody.querySelectorAll("tr[data-entry]"));
+  const headers = table.querySelectorAll("th[data-sort]");
+  const countElement = document.getElementById("aise-table-count");
+  const numericColumns = new Set([
+    "Rank", "Precision", "Recall", "Format", "Edit Dist.", "Para. Acc.",
+    "Success", "Correct.", "Complete.", "Faithful.", "F1-LM"
+  ]);
+  let sortColumn = "F1-LM";
+  let sortDirection = "desc";
+
+  function parseValue(row, column) {
+    const cell = row.querySelector(`[data-col="${column}"]`);
+    if (!cell) return "";
+    const raw = cell.dataset.value || cell.textContent || "";
+    if (numericColumns.has(column)) {
+      const value = Number.parseFloat(raw);
+      return Number.isNaN(value) ? -Infinity : value;
+    }
+    return raw.toLowerCase();
+  }
+
+  function applyTableState() {
+    const sortedRows = rows.slice().sort((left, right) => {
+      const leftValue = parseValue(left, sortColumn);
+      const rightValue = parseValue(right, sortColumn);
+      if (leftValue < rightValue) return sortDirection === "asc" ? -1 : 1;
+      if (leftValue > rightValue) return sortDirection === "asc" ? 1 : -1;
+      return 0;
+    });
+
+    sortedRows.forEach((row, index) => {
+      tbody.appendChild(row);
+      const rank = index + 1;
+      const rankCell = row.querySelector('[data-col="Rank"]');
+      rankCell.textContent = String(rank);
+      rankCell.dataset.value = String(rank);
+    });
+
+    countElement.textContent = `${sortedRows.length} entries`;
+    headers.forEach(header => {
+      header.classList.remove("sort-asc", "sort-desc");
+      if (header.dataset.sort === sortColumn) {
+        header.classList.add(sortDirection === "asc" ? "sort-asc" : "sort-desc");
+      }
+    });
+  }
+
+  headers.forEach(header => {
+    header.addEventListener("click", () => {
+      const column = header.dataset.sort;
+      if (sortColumn === column) {
+        sortDirection = sortDirection === "asc" ? "desc" : "asc";
+      } else {
+        sortColumn = column;
+        sortDirection = numericColumns.has(column) ? "desc" : "asc";
+      }
+      applyTableState();
+    });
+  });
+
+  applyTableState();
+}
+"""
+
+
+METRIC_COLUMNS = [
+    ("Precision", "precision", None),
+    ("Recall", "recall", None),
+    ("Format", "format", "clarity"),
+    ("Edit Dist.", "edit_distance", None),
+    ("Para. Acc.", "para_acc", "planning"),
+    ("Success", "success", "execution"),
+    ("Correct.", "correct", "correctness"),
+    ("Complete.", "complete", "completeness"),
+    ("Faithful.", "faithful", "faithfulness"),
+    ("F1-LM", "f1_lm", "overall"),
+]
+
+
+def _metric(record: dict, primary: str, fallback: str | None = None) -> object:
+    value = record.get(primary)
+    if value is None and fallback:
+        value = record.get(fallback)
+    return value
+
+
+def _format_metric(value: object) -> str:
+    if value is None or value == "":
+        return "/"
+    if isinstance(value, str):
+        return html.escape(value)
+    try:
+        return f"{float(value):.4f}".rstrip("0").rstrip(".")
+    except (TypeError, ValueError):
+        return html.escape(str(value))
+
+
+def _metric_value(value: object) -> str:
+    try:
+        return str(float(value))
+    except (TypeError, ValueError):
+        return ""
+
+
+def _leaderboard_html() -> str:
+    rows = []
+    for rank, record in enumerate(load_leaderboard(), start=1):
+        model = html.escape(str(record.get("model", "")))
+        organization = html.escape(str(record.get("organization", "")))
+        model_url = str(record.get("model_url", ""))
+        if _valid_url(model_url):
+            model_cell = f'<a href="{html.escape(model_url, quote=True)}" target="_blank">{model}</a>'
+        else:
+            model_cell = f'<span class="model-name">{model}</span>'
+        model_cell += f'<span class="org">{organization}</span>'
+        metric_cells = []
+        for label, primary, fallback in METRIC_COLUMNS:
+            metric = _metric(record, primary, fallback)
+            metric_cells.append(
+                f'<td class="num" data-col="{label}" data-value="{_metric_value(metric)}">'
+                f'{_format_metric(metric)}</td>'
+            )
+        verified = bool(record.get("verified"))
+        verified_label = "Verified" if verified else "Pending"
+        verified_class = "verified-yes" if verified else "verified-no"
+        rows.append(
+            f"""
+            <tr data-entry="true">
+              <td class="rank" data-col="Rank" data-value="{rank}">{rank}</td>
+              <td class="model" data-col="Model" data-value="{model}">{model_cell}</td>
+              {''.join(metric_cells)}
+              <td data-col="Verified" data-value="{verified_label}"><span class="verified-badge {verified_class}">{verified_label}</span></td>
+            </tr>
+            """
+        )
+
+    empty = '<tr><td class="empty" colspan="13">No verified submissions yet.</td></tr>'
+    return f"""
+    <div class="leaderboard-toolbar">
+      <div class="table-meta">
+        <span id="aise-table-count">{len(rows)} entries</span>
+        <span>Click a column header to sort</span>
+      </div>
+    </div>
+    <div class="aise-table-wrap">
+      <table id="aise-leaderboard-table" class="aise-table">
+        <thead>
+          <tr>
+            <th rowspan="2" data-sort="Rank">#</th>
+            <th rowspan="2" data-sort="Model">Model</th>
+            <th colspan="3">References and Formatting</th>
+            <th colspan="3">API-based Judge</th>
+            <th colspan="4">Answer Content</th>
+            <th rowspan="2" data-sort="Verified">Status</th>
+          </tr>
+          <tr>
+            <th data-sort="Precision">Precision</th><th data-sort="Recall">Recall</th><th data-sort="Format">Format</th>
+            <th data-sort="Edit Dist.">Edit Dist.</th><th data-sort="Para. Acc.">Para. Acc.</th><th data-sort="Success">Success</th>
+            <th data-sort="Correct.">Correct.</th><th data-sort="Complete.">Complete.</th><th data-sort="Faithful.">Faithful.</th><th data-sort="F1-LM">F1-LM</th>
+          </tr>
+        </thead>
+        <tbody>{''.join(rows) or empty}</tbody>
+      </table>
+    </div>
+    """
+
+
+with gr.Blocks(
+    title="AISE-Bench Leaderboard",
+    analytics_enabled=False,
+) as demo:
+    gr.Markdown(
+        """
+        # AISE-Bench Leaderboard
+
+        Benchmarking full-cycle academic information-seeking agents on planning, execution,
+        reference grounding, and answer quality.
+
+        <div class="hero-links">
+          <a href="https://aise-bench.github.io/" target="_blank" rel="noopener noreferrer">Project Page</a>
+          <span aria-hidden="true">&middot;</span>
+          <a href="https://arxiv.org/abs/2607.20498" target="_blank" rel="noopener noreferrer">Paper</a>
+          <span aria-hidden="true">&middot;</span>
+          <a href="https://huggingface.co/datasets/zhengyang6666/AISE-Bench" target="_blank" rel="noopener noreferrer">Dataset</a>
+          <span aria-hidden="true">&middot;</span>
+          <a href="https://github.com/zai-org/AISE-Bench" target="_blank" rel="noopener noreferrer">GitHub</a>
+        </div>
+        """,
+        elem_classes=["hero"],
+    )
+
+    gr.HTML(
+        """
+        <div class="summary-grid">
+          <div class="summary-card"><strong>500</strong><span>official test questions</span></div>
+          <div class="summary-card"><strong>10 metrics</strong><span>process and answer evaluation</span></div>
+          <div class="summary-card"><strong>Verified only</strong><span>scores produced by the official evaluator</span></div>
+        </div>
+        """
+    )
+
+    gr.Markdown(
+        """
+        ## How to Submit
+
+        1. Fork [this repository](https://huggingface.co/datasets/zhengyang6666/AISE-Bench).
+        2. Create a new branch for your submission.
+        3. Add your submission folder under
+           `submissions/<organization>__<model>__<input_config>/`.
+        4. Open a Pull Request with the new submission folder.
+
+        ### Submission Directory Requirements
+
+        Each submission directory must contain the metadata and predictions for one
+        model/input configuration pair:
+
+        ```text
+        <organization>__<model>__<input_config>/
+          metadata.yaml
+          predictions.jsonl
+          generation_config.json      # optional, recommended
+          artifacts/                  # optional logs or prompt notes
+        ```
+
+        Use URL-safe directory names. Replace spaces, slashes, and special characters
+        with hyphens; keep `input_config` as `TEXT`.
+
+        ### `metadata.yaml`
+
+        ```yaml
+        model_name: "My Model"
+        model_url: https://...
+        ```
+
+        ### `predictions.jsonl`
+
+        Each line must be one JSON object:
+
+        - `index`: one-based sequential position; it must be exactly `1, 2, ..., N` in file order
+        - `qid`: stable question identifier from the dataset
+        - `question`: original real academic search query submitted by AMiner users
+        - `planning_text`: model-produced multi-step API planning sequence
+        - `api_input`: standardized input parameters for each API call
+        - `api_output`: AMiner academic KG API results, entity IDs, and execution status
+        - `result_edit`: model-produced final answer grounded by reference citations `[1]`
+
+        ```json
+        {
+          "index": 1,
+          "qid": 3,
+          "question": "question text",
+          "planning_text": [],
+          "api_input": {},
+          "api_output": {},
+          "result_edit": "{\"answer\":\"...\",\"reference\":{\"[1]\":\"...\"}}"
+        }
+        ```
+
+        The file must contain exactly one line for every item in `test.json`.
+        `qid` must match the corresponding item in `test.json`.
+        """,
+        elem_classes=["submission-guide"],
+    )
+
+    gr.Markdown(
+        """
+        ## Leaderboard
+
+        Official results on the 500-question test set are sorted by **F1-LM** by default. The table
+        is intentionally empty until the first submission completes official evaluation and review.
+        """,
+        elem_classes=["leaderboard-title"],
+    )
+    leaderboard = gr.HTML(value=_leaderboard_html())
+
+if __name__ == "__main__":
+    demo.launch(css=APP_CSS, js=TABLE_JS)

data/leaderboard.json

@@ -0,0 +1 @@
+[]

publish_result.py

@@ -0,0 +1,71 @@
+from __future__ import annotations
+
+import argparse
+import json
+from datetime import datetime, timezone
+from pathlib import Path
+
+from storage import load_pending_submission, publish_result
+
+
+REQUIRED_METRICS = {
+    "precision",
+    "recall",
+    "clarity",
+    "correctness",
+    "integrality",
+    "completeness",
+    "faithfulness",
+    "success",
+    "edit_distance",
+}
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description="Publish an evaluated AISE-Bench submission.")
+    parser.add_argument("submission_id")
+    parser.add_argument("metrics_json", type=Path)
+    parser.add_argument("--grounding", type=float, required=True)
+    parser.add_argument("--overall", type=float, required=True)
+    return parser.parse_args()
+
+
+def main() -> None:
+    args = parse_args()
+    metrics = json.loads(args.metrics_json.read_text(encoding="utf-8-sig"))
+    missing = REQUIRED_METRICS - metrics.keys()
+    if missing:
+        raise SystemExit(f"Metrics file is missing: {', '.join(sorted(missing))}")
+
+    metadata = load_pending_submission(args.submission_id)
+    result = {
+        **metadata,
+        "status": "accepted",
+        "evaluated_at": datetime.now(timezone.utc).isoformat(),
+        "format": float(metrics["clarity"]),
+        "edit_distance": float(metrics["edit_distance"]),
+        "para_acc": metrics.get("para_acc", metrics.get("parameter_accuracy")),
+        "success": float(metrics["success"]),
+        "correct": float(metrics["correctness"]),
+        "complete": float(metrics["completeness"]),
+        "faithful": float(metrics["faithfulness"]),
+        "f1_lm": metrics.get("f1_lm"),
+        "planning": float(metrics["edit_distance"]),
+        "execution": float(metrics["success"]),
+        "precision": float(metrics["precision"]),
+        "recall": float(metrics["recall"]),
+        "clarity": float(metrics["clarity"]),
+        "correctness": float(metrics["correctness"]),
+        "integrality": float(metrics["integrality"]),
+        "completeness": float(metrics["completeness"]),
+        "faithfulness": float(metrics["faithfulness"]),
+        "grounding": args.grounding,
+        "overall": args.overall,
+        "verified": True,
+    }
+    publish_result(result)
+    print(f"Published {args.submission_id}")
+
+
+if __name__ == "__main__":
+    main()

requirements.txt

@@ -0,0 +1,2 @@
+gradio>=6.0,<7
+huggingface_hub>=0.28

schema.py

@@ -0,0 +1,101 @@
+from __future__ import annotations
+
+import hashlib
+import json
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+
+MAX_SUBMISSION_BYTES = 100 * 1024 * 1024
+
+
+@dataclass(frozen=True)
+class Track:
+    name: str
+    expected_samples: int
+
+
+TRACKS = {
+    "test-500": Track("Official Test 500", 500),
+}
+
+REQUIRED_PREDICTION_FIELDS = {
+    "index",
+    "qid",
+    "question",
+    "planning_text",
+    "api_input",
+    "api_output",
+    "result_edit",
+}
+
+
+def load_and_validate_submission(path: str | Path, track_id: str) -> tuple[list[dict[str, Any]], str]:
+    submission_path = Path(path)
+    if not submission_path.is_file():
+        raise ValueError("Submission file does not exist.")
+    if submission_path.stat().st_size > MAX_SUBMISSION_BYTES:
+        raise ValueError("Submission file exceeds the 100 MB limit.")
+    if track_id not in TRACKS:
+        raise ValueError(f"Unknown track: {track_id}")
+
+    try:
+        payload = json.loads(submission_path.read_text(encoding="utf-8"))
+    except (UnicodeDecodeError, json.JSONDecodeError) as exc:
+        raise ValueError(f"Submission must be valid UTF-8 JSON: {exc}") from exc
+
+    predictions = payload.get("predictions") if isinstance(payload, dict) else payload
+    if not isinstance(predictions, list):
+        raise ValueError('Submission must be a JSON list or {"predictions": [...]}.')
+
+    expected_samples = TRACKS[track_id].expected_samples
+    if len(predictions) != expected_samples:
+        raise ValueError(
+            f"Track {track_id} requires {expected_samples} predictions; received {len(predictions)}."
+        )
+
+    seen_ids: set[str] = set()
+    for position, prediction in enumerate(predictions, start=1):
+        if not isinstance(prediction, dict):
+            raise ValueError(f"Prediction #{position} must be a JSON object.")
+        missing_fields = REQUIRED_PREDICTION_FIELDS - prediction.keys()
+        if missing_fields:
+            missing = ", ".join(sorted(missing_fields))
+            raise ValueError(f"Prediction #{position} is missing required fields: {missing}.")
+        if "id" in prediction:
+            raise ValueError(f"Prediction #{position} must use 'qid'; field 'id' is not accepted.")
+
+        sequence_index = prediction["index"]
+        if isinstance(sequence_index, bool) or not isinstance(sequence_index, int):
+            raise ValueError(f"Prediction #{position} field 'index' must be an integer.")
+        if sequence_index != position:
+            raise ValueError(
+                f"Prediction #{position} must have index {position}; received {sequence_index}."
+            )
+
+        qid = prediction["qid"]
+        if isinstance(qid, bool) or not isinstance(qid, (int, str)):
+            raise ValueError(f"Prediction #{position} field 'qid' must be an integer or string.")
+        prediction_id = str(qid).strip()
+        if not prediction_id:
+            raise ValueError(f"Prediction #{position} field 'qid' must not be empty.")
+        if prediction_id in seen_ids:
+            raise ValueError(f"Duplicate prediction qid: {prediction_id}")
+        seen_ids.add(prediction_id)
+
+        if not isinstance(prediction["question"], str):
+            raise ValueError(f"Prediction {prediction_id} field 'question' must be a string.")
+        if not isinstance(prediction["planning_text"], (str, list)):
+            raise ValueError(
+                f"Prediction {prediction_id} field 'planning_text' must be a string or list."
+            )
+        if not isinstance(prediction["api_input"], (list, dict)):
+            raise ValueError(f"Prediction {prediction_id} field 'api_input' must be a list or object.")
+        if not isinstance(prediction["api_output"], (list, dict)):
+            raise ValueError(f"Prediction {prediction_id} field 'api_output' must be a list or object.")
+        if not isinstance(prediction["result_edit"], str):
+            raise ValueError(f"Prediction {prediction_id} field 'result_edit' must be a string.")
+
+    digest = hashlib.sha256(submission_path.read_bytes()).hexdigest()
+    return predictions, digest

storage.py

@@ -0,0 +1,142 @@
+from __future__ import annotations
+
+import json
+import os
+from datetime import datetime, timezone
+from io import BytesIO
+from pathlib import Path
+from typing import Any
+
+from huggingface_hub import HfApi, hf_hub_download, list_repo_files
+
+
+BASE_DIR = Path(__file__).resolve().parent
+SEED_PATH = BASE_DIR / "data" / "leaderboard.json"
+LOCAL_RUNTIME_DIR = BASE_DIR / "runtime_submissions"
+
+
+def _dataset_config() -> tuple[str, str] | None:
+    repo_id = os.getenv("LEADERBOARD_DATASET_REPO", "").strip()
+    token = os.getenv("HF_TOKEN", "").strip()
+    return (repo_id, token) if repo_id and token else None
+
+
+def _read_json(path: str | Path) -> Any:
+    return json.loads(Path(path).read_text(encoding="utf-8"))
+
+
+def load_leaderboard() -> list[dict[str, Any]]:
+    records = list(_read_json(SEED_PATH))
+    config = _dataset_config()
+
+    if config:
+        repo_id, token = config
+        try:
+            files = list_repo_files(repo_id, repo_type="dataset", token=token)
+            for filename in files:
+                if not filename.startswith("results/") or not filename.endswith(".json"):
+                    continue
+                local_path = hf_hub_download(
+                    repo_id,
+                    filename,
+                    repo_type="dataset",
+                    token=token,
+                )
+                record = _read_json(local_path)
+                if record.get("status") == "accepted":
+                    records.append(record)
+        except Exception:
+            pass
+    elif LOCAL_RUNTIME_DIR.exists():
+        for path in LOCAL_RUNTIME_DIR.glob("results/*.json"):
+            record = _read_json(path)
+            if record.get("status") == "accepted":
+                records.append(record)
+
+    def score(item: dict[str, Any]) -> float:
+        value = item.get("f1_lm", item.get("overall"))
+        try:
+            return float(value)
+        except (TypeError, ValueError):
+            return float("-inf")
+
+    return sorted(
+        records,
+        key=lambda item: (
+            -score(item),
+            str(item.get("model", "")).lower(),
+        ),
+    )
+
+
+def queue_submission(metadata: dict[str, Any], prediction_path: str | Path) -> None:
+    metadata = {
+        **metadata,
+        "status": "pending",
+        "submitted_at": datetime.now(timezone.utc).isoformat(),
+    }
+    submission_id = metadata["submission_id"]
+    prediction_target = f"predictions/{submission_id}.json"
+    metadata_target = f"pending/{submission_id}.json"
+    config = _dataset_config()
+
+    if config:
+        repo_id, token = config
+        api = HfApi(token=token)
+        api.upload_file(
+            path_or_fileobj=str(prediction_path),
+            path_in_repo=prediction_target,
+            repo_id=repo_id,
+            repo_type="dataset",
+            commit_message=f"Add predictions for {submission_id}",
+        )
+        api.upload_file(
+            path_or_fileobj=BytesIO(
+                json.dumps(metadata, ensure_ascii=False, indent=2).encode("utf-8")
+            ),
+            path_in_repo=metadata_target,
+            repo_id=repo_id,
+            repo_type="dataset",
+            commit_message=f"Queue submission {submission_id}",
+        )
+        return
+
+    (LOCAL_RUNTIME_DIR / "predictions").mkdir(parents=True, exist_ok=True)
+    (LOCAL_RUNTIME_DIR / "pending").mkdir(parents=True, exist_ok=True)
+    destination = LOCAL_RUNTIME_DIR / prediction_target
+    destination.write_bytes(Path(prediction_path).read_bytes())
+    (LOCAL_RUNTIME_DIR / metadata_target).write_text(
+        json.dumps(metadata, ensure_ascii=False, indent=2),
+        encoding="utf-8",
+    )
+
+
+def load_pending_submission(submission_id: str) -> dict[str, Any]:
+    filename = f"pending/{submission_id}.json"
+    config = _dataset_config()
+    if config:
+        repo_id, token = config
+        local_path = hf_hub_download(repo_id, filename, repo_type="dataset", token=token)
+        return _read_json(local_path)
+    return _read_json(LOCAL_RUNTIME_DIR / filename)
+
+
+def publish_result(record: dict[str, Any]) -> None:
+    submission_id = record["submission_id"]
+    filename = f"results/{submission_id}.json"
+    payload = json.dumps(record, ensure_ascii=False, indent=2).encode("utf-8")
+    config = _dataset_config()
+    if config:
+        repo_id, token = config
+        HfApi(token=token).upload_file(
+            path_or_fileobj=BytesIO(payload),
+            path_in_repo=filename,
+            repo_id=repo_id,
+            repo_type="dataset",
+            commit_message=f"Publish result {submission_id}",
+        )
+        return
+
+    destination = LOCAL_RUNTIME_DIR / filename
+    destination.parent.mkdir(parents=True, exist_ok=True)
+    destination.write_bytes(payload)