| from __future__ import annotations |
|
|
| import html |
|
|
| import gradio as gr |
|
|
| from storage import load_leaderboard |
|
|
|
|
| APP_CSS = """ |
| footer, .api-docs, .show-api, .built-with, [data-testid="api-info"] { display: none !important; } |
| .gradio-container { |
| max-width: 100% !important; |
| padding: 18px 24px 24px !important; |
| font-family: Inter, ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif !important; |
| font-size: 14px !important; |
| } |
| .hero { text-align: center; margin-bottom: 8px; } |
| .hero h1 { font-size: 32px !important; line-height: 1.2 !important; margin: 8px 0 6px !important; } |
| .hero p { color: #4b5563; font-size: 14px !important; line-height: 1.6 !important; margin: 0 !important; } |
| .hero-links { display: flex; justify-content: center; gap: 10px; flex-wrap: wrap; margin-top: 8px; } |
| .hero-links a { color: #2563eb; text-decoration: none; } |
| .hero-links a:hover { text-decoration: underline; } |
| .summary-grid { display: grid; grid-template-columns: repeat(3, minmax(180px, 1fr)); gap: 12px; margin: 14px 0 18px; } |
| .summary-card { border: 1px solid #e5e7eb; border-radius: 10px; padding: 14px 16px; background: #ffffff; } |
| .summary-card strong { display: block; color: #111827; font-size: 18px; margin-bottom: 3px; } |
| .summary-card span { color: #6b7280; font-size: 13px; } |
| .submission-guide { max-width: 1100px; font-size: 14px !important; line-height: 1.65 !important; margin: 8px auto 22px; } |
| .submission-guide h2 { font-size: 21px !important; margin: 22px 0 8px !important; } |
| .submission-guide h3 { font-size: 17px !important; margin: 18px 0 8px !important; } |
| .submission-guide p, .submission-guide li { font-size: 14px !important; line-height: 1.65 !important; } |
| .submission-guide code { font-size: 13px !important; } |
| .submission-guide pre code { font-size: 13px !important; line-height: 1.5 !important; } |
| .leaderboard-title h2 { font-size: 24px !important; text-align: center; margin: 18px 0 4px !important; } |
| .leaderboard-title p { color: #6b7280; font-size: 14px !important; text-align: center; margin: 0 0 12px !important; } |
| .leaderboard-toolbar { display: flex; align-items: center; justify-content: flex-end; margin: 8px 0 10px; } |
| .table-meta { color: #6b7280; display: flex; gap: 12px; font-size: 13px; } |
| .aise-table-wrap { height: min(68vh, 760px); min-height: 460px; overflow: auto; border: 1px solid #e5e7eb; border-radius: 10px; background: #ffffff; } |
| .aise-table { border-collapse: separate; border-spacing: 0; width: 100%; min-width: 1320px; color: #263238; font-size: 13px; } |
| .aise-table th, .aise-table td { border-bottom: 1px solid #edf0f3; border-right: 1px solid #edf0f3; padding: 9px 10px; text-align: center; white-space: nowrap; } |
| .aise-table th { position: sticky; z-index: 2; background: #f8fafc; color: #111827; font-weight: 650; user-select: none; } |
| .aise-table thead tr:first-child th { top: 0; background: #e5e7eb; font-size: 13px; } |
| .aise-table thead tr:nth-child(2) th { top: 38px; cursor: pointer; } |
| .aise-table th[rowspan="2"] { top: 0; cursor: pointer; } |
| .aise-table tbody tr:nth-child(even) { background: #fafafa; } |
| .aise-table tbody tr:hover { background: #eef5ff; } |
| .aise-table .rank { width: 48px; color: #4b5563; font-variant-numeric: tabular-nums; } |
| .aise-table .model { min-width: 190px; text-align: left; white-space: normal; } |
| .aise-table .model a, .aise-table .model-name { color: #2563eb; font-size: 14px; font-weight: 650; text-decoration: none; } |
| .aise-table .model a:hover { text-decoration: underline; } |
| .aise-table .org { display: block; color: #8a8f98; font-size: 12px; margin-top: 3px; } |
| .aise-table .num { font-variant-numeric: tabular-nums; } |
| .verified-badge { display: inline-flex; align-items: center; border-radius: 999px; font-size: 11px; font-weight: 700; padding: 3px 8px; } |
| .verified-yes { background: #dcfce7; color: #166534; } |
| .verified-no { background: #f3f4f6; color: #6b7280; } |
| .aise-table .empty { padding: 28px; color: #7b8088; } |
| .submit-actions button { min-height: 42px !important; font-size: 14px !important; font-weight: 600 !important; } |
| .section-title h2 { font-size: 21px !important; line-height: 1.3 !important; margin: 22px 0 8px !important; } |
| label, .label-wrap, input, textarea { font-size: 14px !important; } |
| @media (max-width: 760px) { |
| .gradio-container { padding: 12px !important; } |
| .summary-grid { grid-template-columns: 1fr; } |
| .table-meta span:last-child { display: none; } |
| } |
| """ |
|
|
|
|
| TABLE_JS = """ |
| () => { |
| const table = document.getElementById("aise-leaderboard-table"); |
| if (!table) return; |
| |
| const tbody = table.querySelector("tbody"); |
| const rows = Array.from(tbody.querySelectorAll("tr[data-entry]")); |
| const headers = table.querySelectorAll("th[data-sort]"); |
| const countElement = document.getElementById("aise-table-count"); |
| const numericColumns = new Set([ |
| "Rank", "Precision", "Recall", "Format", "Edit Dist.", "Para. Acc.", |
| "Success", "Correct.", "Complete.", "Faithful.", "F1-LM" |
| ]); |
| let sortColumn = "F1-LM"; |
| let sortDirection = "desc"; |
| |
| function parseValue(row, column) { |
| const cell = row.querySelector(`[data-col="${column}"]`); |
| if (!cell) return ""; |
| const raw = cell.dataset.value || cell.textContent || ""; |
| if (numericColumns.has(column)) { |
| const value = Number.parseFloat(raw); |
| return Number.isNaN(value) ? -Infinity : value; |
| } |
| return raw.toLowerCase(); |
| } |
| |
| function applyTableState() { |
| const sortedRows = rows.slice().sort((left, right) => { |
| const leftValue = parseValue(left, sortColumn); |
| const rightValue = parseValue(right, sortColumn); |
| if (leftValue < rightValue) return sortDirection === "asc" ? -1 : 1; |
| if (leftValue > rightValue) return sortDirection === "asc" ? 1 : -1; |
| return 0; |
| }); |
| |
| sortedRows.forEach((row, index) => { |
| tbody.appendChild(row); |
| const rank = index + 1; |
| const rankCell = row.querySelector('[data-col="Rank"]'); |
| rankCell.textContent = String(rank); |
| rankCell.dataset.value = String(rank); |
| }); |
| |
| countElement.textContent = `${sortedRows.length} entries`; |
| headers.forEach(header => { |
| header.classList.remove("sort-asc", "sort-desc"); |
| if (header.dataset.sort === sortColumn) { |
| header.classList.add(sortDirection === "asc" ? "sort-asc" : "sort-desc"); |
| } |
| }); |
| } |
| |
| headers.forEach(header => { |
| header.addEventListener("click", () => { |
| const column = header.dataset.sort; |
| if (sortColumn === column) { |
| sortDirection = sortDirection === "asc" ? "desc" : "asc"; |
| } else { |
| sortColumn = column; |
| sortDirection = numericColumns.has(column) ? "desc" : "asc"; |
| } |
| applyTableState(); |
| }); |
| }); |
| |
| applyTableState(); |
| } |
| """ |
|
|
|
|
| METRIC_COLUMNS = [ |
| ("Precision", "precision", None), |
| ("Recall", "recall", None), |
| ("Format", "format", "clarity"), |
| ("Edit Dist.", "edit_distance", None), |
| ("Para. Acc.", "para_acc", "planning"), |
| ("Success", "success", "execution"), |
| ("Correct.", "correct", "correctness"), |
| ("Complete.", "complete", "completeness"), |
| ("Faithful.", "faithful", "faithfulness"), |
| ("F1-LM", "f1_lm", "overall"), |
| ] |
|
|
|
|
| def _metric(record: dict, primary: str, fallback: str | None = None) -> object: |
| value = record.get(primary) |
| if value is None and fallback: |
| value = record.get(fallback) |
| return value |
|
|
|
|
| def _format_metric(value: object) -> str: |
| if value is None or value == "": |
| return "/" |
| if isinstance(value, str): |
| return html.escape(value) |
| try: |
| return f"{float(value):.4f}".rstrip("0").rstrip(".") |
| except (TypeError, ValueError): |
| return html.escape(str(value)) |
|
|
|
|
| def _metric_value(value: object) -> str: |
| try: |
| return str(float(value)) |
| except (TypeError, ValueError): |
| return "" |
|
|
|
|
| def _leaderboard_html() -> str: |
| rows = [] |
| for rank, record in enumerate(load_leaderboard(), start=1): |
| model = html.escape(str(record.get("model", ""))) |
| organization = html.escape(str(record.get("organization", ""))) |
| model_url = str(record.get("model_url", "")) |
| if _valid_url(model_url): |
| model_cell = f'<a href="{html.escape(model_url, quote=True)}" target="_blank">{model}</a>' |
| else: |
| model_cell = f'<span class="model-name">{model}</span>' |
| model_cell += f'<span class="org">{organization}</span>' |
| metric_cells = [] |
| for label, primary, fallback in METRIC_COLUMNS: |
| metric = _metric(record, primary, fallback) |
| metric_cells.append( |
| f'<td class="num" data-col="{label}" data-value="{_metric_value(metric)}">' |
| f'{_format_metric(metric)}</td>' |
| ) |
| verified = bool(record.get("verified")) |
| verified_label = "Verified" if verified else "Pending" |
| verified_class = "verified-yes" if verified else "verified-no" |
| rows.append( |
| f""" |
| <tr data-entry="true"> |
| <td class="rank" data-col="Rank" data-value="{rank}">{rank}</td> |
| <td class="model" data-col="Model" data-value="{model}">{model_cell}</td> |
| {''.join(metric_cells)} |
| <td data-col="Verified" data-value="{verified_label}"><span class="verified-badge {verified_class}">{verified_label}</span></td> |
| </tr> |
| """ |
| ) |
|
|
| empty = '<tr><td class="empty" colspan="13">No verified submissions yet.</td></tr>' |
| return f""" |
| <div class="leaderboard-toolbar"> |
| <div class="table-meta"> |
| <span id="aise-table-count">{len(rows)} entries</span> |
| <span>Click a column header to sort</span> |
| </div> |
| </div> |
| <div class="aise-table-wrap"> |
| <table id="aise-leaderboard-table" class="aise-table"> |
| <thead> |
| <tr> |
| <th rowspan="2" data-sort="Rank">#</th> |
| <th rowspan="2" data-sort="Model">Model</th> |
| <th colspan="3">References and Formatting</th> |
| <th colspan="3">API-based Judge</th> |
| <th colspan="4">Answer Content</th> |
| <th rowspan="2" data-sort="Verified">Status</th> |
| </tr> |
| <tr> |
| <th data-sort="Precision">Precision</th><th data-sort="Recall">Recall</th><th data-sort="Format">Format</th> |
| <th data-sort="Edit Dist.">Edit Dist.</th><th data-sort="Para. Acc.">Para. Acc.</th><th data-sort="Success">Success</th> |
| <th data-sort="Correct.">Correct.</th><th data-sort="Complete.">Complete.</th><th data-sort="Faithful.">Faithful.</th><th data-sort="F1-LM">F1-LM</th> |
| </tr> |
| </thead> |
| <tbody>{''.join(rows) or empty}</tbody> |
| </table> |
| </div> |
| """ |
|
|
|
|
| with gr.Blocks( |
| title="AISE-Bench Leaderboard", |
| analytics_enabled=False, |
| ) as demo: |
| gr.Markdown( |
| """ |
| # AISE-Bench Leaderboard |
| |
| Benchmarking full-cycle academic information-seeking agents on planning, execution, |
| reference grounding, and answer quality. |
| |
| <div class="hero-links"> |
| <a href="https://aise-bench.github.io/" target="_blank" rel="noopener noreferrer">Project Page</a> |
| <span aria-hidden="true">·</span> |
| <a href="https://arxiv.org/abs/2607.20498" target="_blank" rel="noopener noreferrer">Paper</a> |
| <span aria-hidden="true">·</span> |
| <a href="https://huggingface.co/datasets/zhengyang6666/AISE-Bench" target="_blank" rel="noopener noreferrer">Dataset</a> |
| <span aria-hidden="true">·</span> |
| <a href="https://github.com/zai-org/AISE-Bench" target="_blank" rel="noopener noreferrer">GitHub</a> |
| </div> |
| """, |
| elem_classes=["hero"], |
| ) |
|
|
| gr.HTML( |
| """ |
| <div class="summary-grid"> |
| <div class="summary-card"><strong>500</strong><span>official test questions</span></div> |
| <div class="summary-card"><strong>10 metrics</strong><span>process and answer evaluation</span></div> |
| <div class="summary-card"><strong>Verified only</strong><span>scores produced by the official evaluator</span></div> |
| </div> |
| """ |
| ) |
|
|
| gr.Markdown( |
| """ |
| ## How to Submit |
| |
| 1. Fork [this repository](https://huggingface.co/datasets/zhengyang6666/AISE-Bench). |
| 2. Create a new branch for your submission. |
| 3. Add your submission folder under |
| `submissions/<organization>__<model>__<input_config>/`. |
| 4. Open a Pull Request with the new submission folder. |
| |
| ### Submission Directory Requirements |
| |
| Each submission directory must contain the metadata and predictions for one |
| model/input configuration pair: |
| |
| ```text |
| <organization>__<model>__<input_config>/ |
| metadata.yaml |
| predictions.jsonl |
| generation_config.json # optional, recommended |
| artifacts/ # optional logs or prompt notes |
| ``` |
| |
| Use URL-safe directory names. Replace spaces, slashes, and special characters |
| with hyphens; keep `input_config` as `TEXT`. |
| |
| ### `metadata.yaml` |
| |
| ```yaml |
| model_name: "My Model" |
| model_url: https://... |
| ``` |
| |
| ### `predictions.jsonl` |
| |
| Each line must be one JSON object: |
| |
| - `index`: one-based sequential position; it must be exactly `1, 2, ..., N` in file order |
| - `qid`: stable question identifier from the dataset |
| - `question`: original real academic search query submitted by AMiner users |
| - `planning_text`: model-produced multi-step API planning sequence |
| - `api_input`: standardized input parameters for each API call |
| - `api_output`: AMiner academic KG API results, entity IDs, and execution status |
| - `result_edit`: model-produced final answer grounded by reference citations `[1]` |
| |
| ```json |
| { |
| "index": 1, |
| "qid": 3, |
| "question": "question text", |
| "planning_text": [], |
| "api_input": {}, |
| "api_output": {}, |
| "result_edit": "{\"answer\":\"...\",\"reference\":{\"[1]\":\"...\"}}" |
| } |
| ``` |
| |
| The file must contain exactly one line for every item in `test.json`. |
| `qid` must match the corresponding item in `test.json`. |
| """, |
| elem_classes=["submission-guide"], |
| ) |
|
|
| gr.Markdown( |
| """ |
| ## Leaderboard |
| |
| Official results on the 500-question test set are sorted by **F1-LM** by default. The table |
| is intentionally empty until the first submission completes official evaluation and review. |
| """, |
| elem_classes=["leaderboard-title"], |
| ) |
| leaderboard = gr.HTML(value=_leaderboard_html()) |
|
|
| if __name__ == "__main__": |
| demo.launch(css=APP_CSS, js=TABLE_JS) |
|
|
