| <!doctype html> |
| <html lang="en"> |
| <head> |
| <meta charset="utf-8"> |
| <meta name="viewport" content="width=device-width, initial-scale=1"> |
| <title>AISE-Bench Leaderboard</title> |
| <style> |
| :root { |
| color-scheme: light; |
| font-family: Inter, ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif; |
| color: #111827; |
| background: #ffffff; |
| } |
| * { box-sizing: border-box; } |
| body { margin: 0; background: #ffffff; } |
| main { max-width: 1480px; margin: 0 auto; padding: 18px 24px 36px; } |
| .hero { text-align: center; margin-bottom: 8px; } |
| .hero h1 { font-size: 32px; line-height: 1.2; margin: 8px 0 6px; } |
| .hero p { color: #4b5563; font-size: 14px; line-height: 1.6; margin: 0; } |
| .hero-links { display: flex; justify-content: center; gap: 10px; flex-wrap: wrap; margin-top: 8px; } |
| a { color: #2563eb; text-decoration: none; } |
| a:hover { text-decoration: underline; } |
| .summary-grid { |
| display: grid; |
| grid-template-columns: repeat(3, minmax(0, 1fr)); |
| gap: 12px; |
| margin: 18px 0; |
| } |
| .summary-card { |
| border: 1px solid #e5e7eb; |
| border-radius: 8px; |
| padding: 14px 16px; |
| background: #f9fafb; |
| } |
| .summary-card strong { display: block; font-size: 22px; line-height: 1.1; } |
| .summary-card span { display: block; color: #6b7280; font-size: 13px; margin-top: 4px; } |
| .submission-guide { |
| max-width: 980px; |
| margin: 20px 0 28px; |
| font-size: 15px; |
| line-height: 1.65; |
| } |
| .submission-guide h2 { font-size: 24px; margin: 0 0 12px; } |
| .submission-guide h3 { font-size: 18px; margin: 22px 0 8px; } |
| .submission-guide code { |
| background: #f3f4f6; |
| border: 1px solid #e5e7eb; |
| border-radius: 4px; |
| padding: 1px 4px; |
| } |
| pre { |
| background: #f8fafc; |
| border: 1px solid #e5e7eb; |
| border-radius: 8px; |
| overflow-x: auto; |
| padding: 12px 14px; |
| line-height: 1.45; |
| } |
| pre code { background: transparent; border: 0; padding: 0; } |
| .leaderboard-title { margin-top: 10px; } |
| .leaderboard-title h2 { font-size: 24px; margin-bottom: 6px; } |
| .leaderboard-title p { color: #4b5563; margin-top: 0; } |
| .table-shell { overflow-x: auto; border: 1px solid #e5e7eb; border-radius: 8px; } |
| table { width: 100%; border-collapse: collapse; font-size: 13px; white-space: nowrap; } |
| th, td { padding: 10px 12px; border-bottom: 1px solid #e5e7eb; text-align: right; } |
| th { |
| background: #f9fafb; |
| color: #374151; |
| cursor: pointer; |
| user-select: none; |
| font-weight: 650; |
| } |
| th:first-child, td:first-child, |
| th:nth-child(2), td:nth-child(2), |
| th:nth-child(3), td:nth-child(3) { text-align: left; } |
| tbody tr:last-child td { border-bottom: 0; } |
| .model-cell strong { display: block; font-size: 14px; } |
| .model-cell span { display: block; color: #6b7280; font-size: 12px; margin-top: 2px; } |
| .verified { color: #047857; font-weight: 650; } |
| .empty { color: #6b7280; text-align: center; padding: 24px; } |
| @media (max-width: 760px) { |
| main { padding: 12px; } |
| .summary-grid { grid-template-columns: 1fr; } |
| .hero h1 { font-size: 26px; } |
| } |
| </style> |
| </head> |
| <body> |
| <main> |
| <section class="hero"> |
| <h1>AISE-Bench Leaderboard</h1> |
| <p>Evaluating academic information-seeking agents across planning, API execution, reference grounding, and answer quality.</p> |
| <div class="hero-links"> |
| <a href="https://aise-bench.github.io/" target="_blank" rel="noopener noreferrer">Project Page</a> |
| <span aria-hidden="true">·</span> |
| <a href="https://arxiv.org/abs/2607.20498" target="_blank" rel="noopener noreferrer">Paper</a> |
| <span aria-hidden="true">·</span> |
| <a href="https://huggingface.co/datasets/zhengyang6666/AISE-Bench" target="_blank" rel="noopener noreferrer">Dataset</a> |
| <span aria-hidden="true">·</span> |
| <a href="https://github.com/zai-org/AISE-Bench" target="_blank" rel="noopener noreferrer">GitHub</a> |
| </div> |
| </section> |
|
|
| <section class="summary-grid" aria-label="Benchmark summary"> |
| <div class="summary-card"><strong>500</strong><span>official test questions</span></div> |
| <div class="summary-card"><strong>10 metrics</strong><span>process and answer evaluation</span></div> |
| <div class="summary-card"><strong>Verified only</strong><span>scores produced by the official evaluator</span></div> |
| </section> |
|
|
| <section class="submission-guide"> |
| <h2>How to Submit</h2> |
| <ol> |
| <li>Fork <a href="https://huggingface.co/datasets/zhengyang6666/AISE-Bench-submissions" target="_blank" rel="noopener noreferrer">this repository</a>.</li> |
| <li>Create a new branch for your submission.</li> |
| <li>Add your submission folder under <code>submissions/<organization>__<model>__<input_config>/</code>.</li> |
| <li>Open a Pull Request with the new submission folder.</li> |
| </ol> |
|
|
| <h3>Submission Directory Requirements</h3> |
| <p>Each submission directory must contain the metadata and predictions for one model/input configuration pair:</p> |
| <pre><code><organization>__<model>__<input_config>/ |
| metadata.yaml |
| predictions.jsonl |
| generation_config.json # optional, recommended |
| artifacts/ # optional logs or prompt notes</code></pre> |
| <p>Use URL-safe directory names. Replace spaces, slashes, and special characters with hyphens; keep <code>input_config</code> as <code>TEXT</code>.</p> |
|
|
| <h3><code>metadata.yaml</code></h3> |
| <pre><code>model_name: "My Model" |
| model_url: https://...</code></pre> |
|
|
| <h3><code>predictions.jsonl</code></h3> |
| <p>Each line must be one JSON object:</p> |
| <pre><code>{ |
| "index": 1, |
| "qid": 3, |
| "question": "question text", |
| "planning_text": [], |
| "api_input": {}, |
| "api_output": {}, |
| "result_edit": "{\"answer\":\"...\",\"reference\":{\"[1]\":\"...\"}}" |
| }</code></pre> |
| <p>The file must contain exactly one line for every item in <code>test.json</code>. <code>qid</code> must match the corresponding item in <code>test.json</code>.</p> |
| </section> |
|
|
| <section class="leaderboard-title"> |
| <h2>Leaderboard</h2> |
| <p>Official results on the 500-question test set are sorted by <strong>F1-LM</strong> by default. The table is intentionally empty until the first submission completes official evaluation and review.</p> |
| </section> |
|
|
| <section class="table-shell"> |
| <table id="leaderboard"> |
| <thead> |
| <tr> |
| <th data-type="number">Rank</th> |
| <th data-type="text">Model</th> |
| <th data-type="text">Track</th> |
| <th data-type="number">Planning</th> |
| <th data-type="number">Exec.</th> |
| <th data-type="number">Precision</th> |
| <th data-type="number">Recall</th> |
| <th data-type="number">Format</th> |
| <th data-type="number">Edit Dist.</th> |
| <th data-type="number">Para. Acc.</th> |
| <th data-type="number">Success</th> |
| <th data-type="number">Correct.</th> |
| <th data-type="number">Complete.</th> |
| <th data-type="number">Faithful.</th> |
| <th data-type="number">F1-LM</th> |
| </tr> |
| </thead> |
| <tbody> |
| <tr><td class="empty" colspan="15">No accepted submissions yet.</td></tr> |
| </tbody> |
| </table> |
| </section> |
| </main> |
| <script> |
| const table = document.getElementById("leaderboard"); |
| const body = table.querySelector("tbody"); |
| const emptyRow = body.querySelector(".empty"); |
| table.querySelectorAll("th").forEach((header, index) => { |
| header.addEventListener("click", () => { |
| if (emptyRow) return; |
| const type = header.dataset.type; |
| const rows = [...body.querySelectorAll("tr")]; |
| const direction = header.dataset.direction === "asc" ? "desc" : "asc"; |
| table.querySelectorAll("th").forEach((item) => delete item.dataset.direction); |
| header.dataset.direction = direction; |
| rows.sort((left, right) => { |
| const a = left.children[index].dataset.value || left.children[index].textContent.trim(); |
| const b = right.children[index].dataset.value || right.children[index].textContent.trim(); |
| const result = type === "number" ? Number(a) - Number(b) : a.localeCompare(b); |
| return direction === "asc" ? result : -result; |
| }); |
| rows.forEach((row) => body.appendChild(row)); |
| }); |
| }); |
| </script> |
| </body> |
| </html> |
|
|
