diff --git "a/output/hierarchy_explorer.html" "b/output/hierarchy_explorer.html"
deleted file mode 100644--- "a/output/hierarchy_explorer.html"
+++ /dev/null
@@ -1,479 +0,0 @@
-<!doctype html>
-<html lang="en">
-<head>
-<meta charset="utf-8">
-<title>Hierarchy explorer · cleaned</title>
-<meta name="viewport" content="width=device-width, initial-scale=1">
-<style>
-  :root {
-    --fg: #111;
-    --fg-muted: #444;
-    --fg-subtle: #888;
-    --bg: #fff;
-    --bg-warm: #f7f5f2;
-    --border-soft: #e5e2dc;
-    --border-strong: #1a1a1a;
-    --accent: #c2410c;
-  }
-  @media (prefers-color-scheme: dark) {
-    :root {
-      --fg: #f5f4f1;
-      --fg-muted: #c8c4be;
-      --fg-subtle: #888884;
-      --bg: #15140f;
-      --bg-warm: #1d1c17;
-      --border-soft: #2a2924;
-      --border-strong: #f5f4f1;
-      --accent: #ff6b35;
-    }
-  }
-  * { box-sizing: border-box; }
-  body { margin: 0; font: 13px/1.5 -apple-system, BlinkMacSystemFont, "Segoe UI", system-ui, sans-serif; color: var(--fg); background: var(--bg); }
-  header { padding: 24px 32px; border-bottom: 1px solid var(--border-soft); }
-  header h1 { margin: 0 0 6px; font-size: 18px; font-weight: 600; letter-spacing: -0.01em; }
-  header .meta { color: var(--fg-subtle); font: 11px/1.6 ui-monospace, SFMono-Regular, "JetBrains Mono", Menlo, monospace; text-transform: uppercase; letter-spacing: 0.12em; }
-  header .stats { margin-top: 12px; display: flex; flex-wrap: wrap; gap: 16px; font: 11px/1.4 ui-monospace, SFMono-Regular, Menlo, monospace; }
-  header .stats span { color: var(--fg-muted); }
-  header .stats b { color: var(--fg); font-weight: 600; margin-right: 4px; }
-  .toolbar { padding: 12px 32px; border-bottom: 1px solid var(--border-soft); display: flex; flex-wrap: wrap; gap: 12px; align-items: center; background: var(--bg-warm); position: sticky; top: 0; z-index: 5; }
-  .toolbar input[type="text"] { font: 13px ui-sans-serif, system-ui, sans-serif; padding: 6px 10px; border: 1px solid var(--border-soft); background: var(--bg); color: var(--fg); width: 280px; }
-  .toolbar select { font: 12px ui-sans-serif, system-ui, sans-serif; padding: 6px 8px; border: 1px solid var(--border-soft); background: var(--bg); color: var(--fg); }
-  .toolbar button { font: 11px ui-monospace, Menlo, monospace; text-transform: uppercase; letter-spacing: 0.1em; padding: 6px 10px; border: 1px solid var(--border-soft); background: var(--bg); color: var(--fg); cursor: pointer; }
-  .toolbar button:hover { border-color: var(--border-strong); }
-  .toolbar .grow { flex: 1; }
-  .toolbar .count { font: 11px ui-monospace, Menlo, monospace; color: var(--fg-subtle); text-transform: uppercase; letter-spacing: 0.1em; }
-  main { padding: 0 32px 64px; }
-  .tab-bar { display: flex; gap: 0; border-bottom: 1px solid var(--border-soft); margin-bottom: 16px; }
-  .tab-bar button { font: 11px ui-monospace, Menlo, monospace; text-transform: uppercase; letter-spacing: 0.1em; padding: 12px 16px; border: none; background: transparent; color: var(--fg-subtle); cursor: pointer; border-bottom: 2px solid transparent; margin-bottom: -1px; }
-  .tab-bar button.on { color: var(--fg); border-bottom-color: var(--accent); }
-  details { border-bottom: 1px solid var(--border-soft); padding: 6px 0; }
-  details > summary { list-style: none; cursor: pointer; padding: 6px 0; user-select: none; }
-  details > summary::-webkit-details-marker { display: none; }
-  details > summary:hover { background: var(--bg-warm); }
-  .fam { padding: 8px 0; }
-  .fam-head { display: flex; align-items: baseline; gap: 12px; }
-  .fam-head .name { font-weight: 600; font-size: 14px; }
-  .fam-head .key { font: 10px ui-monospace, Menlo, monospace; color: var(--fg-subtle); text-transform: uppercase; letter-spacing: 0.1em; }
-  .fam-head .badges { margin-left: auto; display: flex; gap: 8px; align-items: baseline; }
-  .badge { font: 10px ui-monospace, Menlo, monospace; padding: 2px 6px; border: 1px solid var(--border-soft); color: var(--fg-muted); text-transform: uppercase; letter-spacing: 0.08em; }
-  .badge.cat { color: var(--accent); border-color: var(--accent); }
-  .badge.size { color: var(--fg-subtle); }
-  .badge.level { color: var(--bg); background: var(--fg); border-color: var(--fg); font-weight: 600; }
-  .badge.level.slice-tag { background: var(--fg-muted); border-color: var(--fg-muted); margin-right: 6px; }
-  .pill-level { font-size: 8px; opacity: 0.6; margin-right: 4px; padding: 1px 3px; background: var(--fg); color: var(--bg); }
-  .indent { padding-left: 24px; }
-  .composite { margin: 6px 0; padding: 6px 0 6px 0; border-left: 2px solid var(--border-soft); }
-  .composite-head { display: flex; align-items: baseline; gap: 10px; padding: 4px 0 4px 8px; cursor: pointer; }
-  .composite-head:hover { background: var(--bg-warm); }
-  .composite-head .name { font-weight: 600; font-size: 12.5px; }
-  .composite-head .key { font: 10px ui-monospace, Menlo, monospace; color: var(--fg-subtle); }
-  .bench { margin: 4px 0; padding: 4px 8px; border-left: 1px dashed var(--border-soft); }
-  .bench-head { display: flex; align-items: baseline; gap: 10px; cursor: pointer; }
-  .bench-head .name { font-weight: 500; font-size: 12.5px; }
-  .bench-head .key { font: 10px ui-monospace, Menlo, monospace; color: var(--fg-subtle); }
-  .bench-head .star { color: var(--accent); }
-  .bench-detail { margin-top: 6px; padding: 8px 12px; background: var(--bg-warm); font-size: 12px; }
-  .bench-detail dl { margin: 0; display: grid; grid-template-columns: 130px 1fr; gap: 4px 12px; }
-  .bench-detail dt { color: var(--fg-subtle); font: 10px ui-monospace, Menlo, monospace; text-transform: uppercase; letter-spacing: 0.1em; }
-  .bench-detail dd { margin: 0; }
-  .slice-list { list-style: none; padding: 0; margin: 6px 0 0; }
-  .slice-list li { padding: 3px 0; border-top: 1px dashed var(--border-soft); font-size: 12px; }
-  .slice-list li:first-child { border-top: none; }
-  .slice-list .slice-name { color: var(--fg); font-weight: 500; }
-  .slice-list .slice-key { color: var(--fg-subtle); font: 10px ui-monospace, Menlo, monospace; margin-left: 8px; }
-  .metric-pill { display: inline-block; font: 10px ui-monospace, Menlo, monospace; padding: 1px 6px; border: 1px solid var(--border-soft); margin: 1px 3px 1px 0; color: var(--fg-muted); }
-  .metric-pill.primary { color: var(--accent); border-color: var(--accent); }
-  .tag-pill { display: inline-block; font: 10px ui-monospace, Menlo, monospace; padding: 1px 5px; background: var(--bg-warm); margin: 1px 3px 1px 0; color: var(--fg-muted); text-transform: uppercase; letter-spacing: 0.05em; }
-  .overlap-row { display: grid; grid-template-columns: minmax(0, 2fr) 60px minmax(0, 3fr); gap: 12px; padding: 8px 0; border-bottom: 1px solid var(--border-soft); align-items: baseline; }
-  .overlap-row .canonical-name { font-weight: 600; font-size: 13px; }
-  .overlap-row .canonical-key { font: 10px ui-monospace, Menlo, monospace; color: var(--fg-subtle); }
-  .overlap-row .n { text-align: center; font: 12px ui-monospace, Menlo, monospace; color: var(--fg-muted); }
-  .overlap-row .apps { display: flex; flex-wrap: wrap; gap: 4px 8px; font-size: 11px; }
-  .overlap-row .apps span { padding: 2px 6px; border: 1px solid var(--border-soft); }
-  .overlap-row .apps b { color: var(--accent); font-weight: 600; margin-right: 4px; }
-  .empty { padding: 32px; text-align: center; color: var(--fg-subtle); font: 11px ui-monospace, Menlo, monospace; text-transform: uppercase; letter-spacing: 0.15em; }
-  .hidden { display: none; }
-  .pane { display: none; }
-  .pane.on { display: block; }
-</style>
-</head>
-<body>
-<header>
-  <h1>Hierarchy explorer</h1>
-  <div class="meta" id="meta"></div>
-  <div class="stats" id="stats"></div>
-</header>
-<div class="toolbar">
-  <input type="text" id="search" placeholder="Search families, composites, benchmarks, splits, tags…" autocomplete="off">
-  <select id="categoryFilter"><option value="">All categories</option></select>
-  <button id="expandAll" type="button">Expand all</button>
-  <button id="collapseAll" type="button">Collapse all</button>
-  <span class="grow"></span>
-  <span class="count" id="resultCount"></span>
-</div>
-<main>
-  <div class="tab-bar">
-    <button data-tab="hierarchy" class="on" type="button">Hierarchy</button>
-    <button data-tab="overlaps" type="button">Cross-suite overlaps (57)</button>
-  </div>
-  <div class="pane on" id="pane-hierarchy"></div>
-  <div class="pane" id="pane-overlaps"></div>
-</main>
-<script>
-const DATA = {"cleanerStatus":"applied lib/clean-hierarchy.ts (with comparison-index)","generatedAt":"2026-05-05T11:41:25Z","schemaVersion":"v3.hierarchy.1","stats":{"families":50,"benchmarks":596,"overlaps":57,"family_count":50,"composite_count":10,"benchmark_count":596,"slice_count":690,"metric_count":718,"metric_rows_scanned":101843},"families":[{"key":"ace","display_name":"Mercor ACE","category":"General","tags":{"domains":["DIY/home improvement","food/recipes","gaming design","shopping/products"],"languages":["Not specified"],"tasks":["Not specified"]},"evals_count":1,"eval_summary_ids":["ace%2Face"],"reproducibility_summary":{"results_total":12,"has_reproducibility_gap_count":12,"populated_ratio_avg":0.9285714285714287},"provenance_summary":{"total_results":12,"total_groups":12,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":12,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":12,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"standalone_benchmarks":[{"key":"ace","display_name":"ACE","family_id":"ace","is_slice":false,"is_overall":true,"primary_metric_key":"score","has_card":true,"tags":{"domains":["DIY/home improvement","food/recipes","shopping/products","gaming design"],"languages":["Not specified"],"tasks":["Not specified"]},"metrics":[{"key":"score","display_name":"Score","sources":["Mercor"],"models_count":12,"is_primary":true}],"slices":[{"key":"ace","display_name":"ace","is_bare_stem":true,"metrics":[{"key":"score","display_name":"Score","sources":["Mercor"]}],"derivedTags":["multimodal","agentic"]},{"key":"diy","display_name":"DIY","is_bare_stem":false,"metrics":[{"key":"score","display_name":"Score","sources":["Mercor"]}],"derivedTags":["multimodal","agentic"]},{"key":"food","display_name":"Food","is_bare_stem":false,"metrics":[{"key":"score","display_name":"Score","sources":["Mercor"]}],"derivedTags":["multimodal","agentic"]},{"key":"gaming","display_name":"Gaming","is_bare_stem":false,"metrics":[{"key":"score","display_name":"Score","sources":["Mercor"]}],"derivedTags":["multimodal","agentic"]},{"key":"shopping","display_name":"Shopping","is_bare_stem":false,"metrics":[{"key":"score","display_name":"Score","sources":["Mercor"]}],"derivedTags":["multimodal","agentic"]}],"summary_eval_ids":["ace%2Face"],"reproducibility_summary":{"results_total":12,"has_reproducibility_gap_count":12,"populated_ratio_avg":0.9285714285714287},"provenance_summary":{"total_results":12,"total_groups":12,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":12,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":12,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":true,"derivedTags":["multimodal","agentic"]}],"derivedTags":["multimodal","agentic"]},{"key":"agentharm","display_name":"AgentHarm","category":"General","tags":{"domains":[],"languages":[],"tasks":[]},"evals_count":20,"eval_summary_ids":["agentharm%2FCopyright","agentharm%2FCybercrime","agentharm%2FDrugs","agentharm%2FHate","agentharm%2FSexual","agentharm%2Fagentharm","agentharm%2Fdisinformation"],"reproducibility_summary":{"results_total":72,"has_reproducibility_gap_count":72,"populated_ratio_avg":0.10714285714285711},"provenance_summary":{"total_results":72,"total_groups":72,"multi_source_groups":10,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":72,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":72,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"benchmarks":[],"standalone_benchmarks":[{"key":"agentharm","display_name":"AgentHarm","family_id":"agentharm","is_slice":false,"is_overall":true,"primary_metric_key":"inspect_evals/avg_full_score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"inspect_evals/avg_full_score","display_name":"inspect_evals/avg_full_score","sources":["Arcadia Impact"],"models_count":8,"is_primary":true},{"key":"inspect_evals/avg_refusals","display_name":"inspect_evals/avg_refusals","sources":["Arcadia Impact"],"models_count":8,"is_primary":false},{"key":"inspect_evals/avg_score","display_name":"inspect_evals/avg_score","sources":["Arcadia Impact"],"models_count":8,"is_primary":false},{"key":"inspect_evals/avg_score_non_refusals","display_name":"inspect_evals/avg_score_non_refusals","sources":["Arcadia Impact"],"models_count":8,"is_primary":false}],"slices":[{"key":"agentharm","display_name":"AgentHarm","slice_key":null,"is_bare_stem":true,"metrics":[{"key":"inspect_evals/avg_full_score","display_name":"inspect_evals/avg_full_score","sources":["Arcadia Impact"],"models_count":8,"is_primary":true},{"key":"inspect_evals/avg_refusals","display_name":"inspect_evals/avg_refusals","sources":["Arcadia Impact"],"models_count":8,"is_primary":false},{"key":"inspect_evals/avg_score","display_name":"inspect_evals/avg_score","sources":["Arcadia Impact"],"models_count":8,"is_primary":false},{"key":"inspect_evals/avg_score_non_refusals","display_name":"inspect_evals/avg_score_non_refusals","sources":["Arcadia Impact"],"models_count":8,"is_primary":false}],"derivedTags":["safety","agentic"]},{"key":"Copyright","display_name":"Copyright","slice_key":"Copyright","is_bare_stem":false,"metrics":[{"key":"Copyright_avg_refusals","display_name":"Copyright_avg_refusals","sources":["Arcadia Impact"],"models_count":8,"is_primary":true},{"key":"Copyright_avg_scores","display_name":"Copyright_avg_scores","sources":["Arcadia Impact"],"models_count":8,"is_primary":false}],"derivedTags":["safety","agentic"]},{"key":"Cybercrime","display_name":"Cybercrime","slice_key":"Cybercrime","is_bare_stem":false,"metrics":[{"key":"Cybercrime_avg_refusals","display_name":"Cybercrime_avg_refusals","sources":["Arcadia Impact"],"models_count":8,"is_primary":true},{"key":"Cybercrime_avg_scores","display_name":"Cybercrime_avg_scores","sources":["Arcadia Impact"],"models_count":8,"is_primary":false}],"derivedTags":["safety","agentic"]},{"key":"Drugs","display_name":"Drugs","slice_key":"Drugs","is_bare_stem":false,"metrics":[{"key":"Drugs_avg_refusals","display_name":"Drugs_avg_refusals","sources":["Arcadia Impact"],"models_count":8,"is_primary":true},{"key":"Drugs_avg_scores","display_name":"Drugs_avg_scores","sources":["Arcadia Impact"],"models_count":8,"is_primary":false}],"derivedTags":["safety","agentic"]},{"key":"Hate","display_name":"Hate","slice_key":"Hate","is_bare_stem":false,"metrics":[{"key":"Hate_avg_refusals","display_name":"Hate_avg_refusals","sources":["Arcadia Impact"],"models_count":8,"is_primary":true},{"key":"Hate_avg_scores","display_name":"Hate_avg_scores","sources":["Arcadia Impact"],"models_count":8,"is_primary":false}],"derivedTags":["safety","agentic"]},{"key":"Sexual","display_name":"Sexual","slice_key":"Sexual","is_bare_stem":false,"metrics":[{"key":"Sexual_avg_refusals","display_name":"Sexual_avg_refusals","sources":["Arcadia Impact"],"models_count":8,"is_primary":true},{"key":"Sexual_avg_scores","display_name":"Sexual_avg_scores","sources":["Arcadia Impact"],"models_count":8,"is_primary":false}],"derivedTags":["safety","agentic"]},{"key":"disinformation","display_name":"Disinformation","slice_key":"disinformation","is_bare_stem":false,"metrics":[{"key":"Disinformation_avg_refusals","display_name":"Disinformation_avg_refusals","sources":["Arcadia Impact"]},{"key":"Disinformation_avg_scores","display_name":"Disinformation_avg_scores","sources":["Arcadia Impact"]}],"derivedTags":["safety","agentic"]}],"summary_eval_ids":["agentharm%2Fagentharm","agentharm%2FCopyright","agentharm%2FCybercrime","agentharm%2FDrugs","agentharm%2FHate","agentharm%2FSexual","agentharm%2Fdisinformation"],"reproducibility_summary":{"results_total":8,"has_reproducibility_gap_count":8,"populated_ratio_avg":0.10714285714285712},"provenance_summary":{"total_results":8,"total_groups":8,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":8,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":8,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":true,"derivedTags":["safety","agentic"]}],"composites":[],"derivedTags":["safety","agentic"]},{"key":"apex-agents","display_name":"Mercor APEX-Agents","category":"Agentic","tags":{"domains":["consulting","corporate law","finance","investment banking","legal","management consulting"],"languages":["English"],"tasks":["Demonstrating advanced knowledge","Executing realistic professional work prompts requiring multi-application tool use","Planning over long horizons within realistic project scenarios","Producing deliverables in investment banking, management consulting, and corporate law domains","Question answering","Reasoning","Text generation","Using multiple applications"]},"evals_count":5,"eval_summary_ids":["apex-agents%2Fapex-agents","apex-agents%2Fapex-v1"],"reproducibility_summary":{"results_total":27,"has_reproducibility_gap_count":27,"populated_ratio_avg":0.9285714285714286},"provenance_summary":{"total_results":27,"total_groups":27,"multi_source_groups":1,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":27,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":27,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"benchmarks":[{"key":"apex-agents","display_name":"APEX Agents","family_id":"apex-agents","is_slice":false,"is_overall":true,"primary_metric_key":"mean-score","has_card":true,"tags":{"domains":["investment banking","management consulting","corporate law"],"languages":["English"],"tasks":["Executing realistic professional work prompts requiring multi-application tool use","Producing deliverables in investment banking, management consulting, and corporate law domains"]},"metrics":[{"key":"mean-score","display_name":"Mean Score","sources":["Mercor"],"models_count":19,"is_primary":true},{"key":"pass-at-1","display_name":"Pass@1","sources":["Mercor"],"models_count":13,"is_primary":false},{"key":"pass-at-8","display_name":"Pass@8","sources":["Mercor"],"models_count":8,"is_primary":false},{"key":"score","display_name":"Score","sources":["Mercor"],"models_count":9,"is_primary":false}],"slices":[{"key":"corporate law","display_name":"Corporate Law","is_bare_stem":false,"metrics":[{"key":"pass-at-1","display_name":"Pass@1","sources":["Mercor"]}],"derivedTags":["agentic","applied_reasoning"]},{"key":"corporate lawyer","display_name":"Corporate Lawyer","is_bare_stem":false,"metrics":[{"key":"mean-score","display_name":"Mean Score","sources":["Mercor"]}],"derivedTags":["agentic","applied_reasoning"]},{"key":"management consulting","display_name":"Management Consulting","is_bare_stem":false,"metrics":[{"key":"pass-at-1","display_name":"Pass@1","sources":["Mercor"]}],"derivedTags":["agentic","applied_reasoning"]},{"key":"overall","display_name":"Overall","is_bare_stem":false,"metrics":[{"key":"pass-at-1","display_name":"Pass@1","sources":["Mercor"]},{"key":"pass-at-8","display_name":"Pass@8","sources":["Mercor"]},{"key":"score","display_name":"Score","sources":["Mercor"]}],"derivedTags":["agentic","applied_reasoning"]}],"summary_eval_ids":["apex-agents%2Fapex-agents"],"reproducibility_summary":{"results_total":19,"has_reproducibility_gap_count":19,"populated_ratio_avg":0.9285714285714285},"provenance_summary":{"total_results":19,"total_groups":19,"multi_source_groups":1,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":19,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":19,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":true,"derivedTags":["agentic","applied_reasoning"]}],"derivedTags":["agentic","applied_reasoning"]},{"key":"apex-v1","display_name":"Mercor APEX-v1","category":"Reasoning","tags":{"domains":["consulting","corporate law","finance","investment banking","legal","management consulting"],"languages":["English"],"tasks":["Demonstrating advanced knowledge","Planning over long horizons within realistic project scenarios","Question answering","Reasoning","Text generation","Using multiple applications"]},"evals_count":2,"eval_summary_ids":["apex-v1%2Fapex-v1"],"reproducibility_summary":{"results_total":7,"has_reproducibility_gap_count":7,"populated_ratio_avg":0.9285714285714287},"provenance_summary":{"total_results":7,"total_groups":7,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":7,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":7,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"standalone_benchmarks":[{"key":"apex-v1","display_name":"APEX v1","family_id":"apex-v1","is_slice":false,"is_overall":true,"primary_metric_key":"mean-score","has_card":true,"tags":{"domains":["investment banking","management consulting","corporate law","finance","legal","consulting"],"languages":["English"],"tasks":["Text generation","Question answering","Reasoning","Demonstrating advanced knowledge","Using multiple applications","Planning over long horizons within realistic project scenarios"]},"metrics":[{"key":"mean-score","display_name":"Mean Score","sources":["Mercor"],"models_count":7,"is_primary":true},{"key":"score","display_name":"Score","sources":["Mercor"],"models_count":7,"is_primary":false}],"slices":[{"key":"apex-v1","display_name":"apex-v1","is_bare_stem":true,"metrics":[{"key":"mean-score","display_name":"Mean Score","sources":["Mercor"]}],"derivedTags":["agentic","applied_reasoning"]},{"key":"big law","display_name":"Big Law","is_bare_stem":false,"metrics":[{"key":"score","display_name":"Score","sources":["Mercor"]}],"derivedTags":["agentic","applied_reasoning"]},{"key":"consulting","display_name":"Consulting","is_bare_stem":false,"metrics":[{"key":"score","display_name":"Score","sources":["Mercor"]}],"derivedTags":["agentic","applied_reasoning"]},{"key":"investment banking","display_name":"Investment Banking","is_bare_stem":false,"metrics":[{"key":"score","display_name":"Score","sources":["Mercor"]}],"derivedTags":["agentic","applied_reasoning"]},{"key":"medicine (md)","display_name":"Medicine (MD)","is_bare_stem":false,"metrics":[{"key":"score","display_name":"Score","sources":["Mercor"]}],"derivedTags":["agentic","applied_reasoning"]}],"summary_eval_ids":["apex-v1%2Fapex-v1"],"reproducibility_summary":{"results_total":7,"has_reproducibility_gap_count":7,"populated_ratio_avg":0.9285714285714287},"provenance_summary":{"total_results":7,"total_groups":7,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":7,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":7,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":true,"derivedTags":["agentic","applied_reasoning"]}],"derivedTags":["agentic","applied_reasoning"]},{"key":"arc-agi","display_name":"ARC-AGI","category":"Reasoning","tags":{"domains":["abstraction and reasoning","artificial general intelligence","visual reasoning"],"languages":["Not specified"],"tasks":["Constructing output grids corresponding to test inputs","Using demonstration pairs to understand a task's nature"]},"evals_count":12,"eval_summary_ids":["arc-agi%2Farc-agi-v1-public-eval","arc-agi%2Farc-agi-v1-semi-private","arc-agi%2Farc-agi-v2-private-eval","arc-agi%2Farc-agi-v2-public-eval","arc-agi%2Farc-agi-v2-semi-private","arc-agi%2Farc-agi-v3-semi-private"],"reproducibility_summary":{"results_total":348,"has_reproducibility_gap_count":348,"populated_ratio_avg":0.9285714285714295},"provenance_summary":{"total_results":348,"total_groups":348,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":348,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":348,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"benchmarks":[{"key":"arc-agi-v1-public-eval","display_name":"ARC-AGI v1 Public Eval","family_id":"arc-agi","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["artificial general intelligence","abstraction and reasoning","visual reasoning"],"languages":["Not specified"],"tasks":["Using demonstration pairs to understand a task's nature","Constructing output grids corresponding to test inputs"]},"metrics":[{"key":"cost-per-task","display_name":"Cost Per Task","sources":["ARC Prize"],"models_count":75,"is_primary":false},{"key":"score","display_name":"Score","sources":["ARC Prize"],"models_count":75,"is_primary":true}],"slices":[],"summary_eval_ids":["arc-agi%2Farc-agi-v1-public-eval"],"reproducibility_summary":{"results_total":75,"has_reproducibility_gap_count":75,"populated_ratio_avg":0.9285714285714293},"provenance_summary":{"total_results":75,"total_groups":75,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":75,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":75,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":true,"derivedTags":["logical_reasoning","commonsense_reasoning","general"]},{"key":"arc-agi-v1-semi-private","display_name":"ARC-AGI v1 Semi-Private","family_id":"arc-agi","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["artificial general intelligence","abstraction and reasoning","visual reasoning"],"languages":["Not specified"],"tasks":["Using demonstration pairs to understand a task's nature","Constructing output grids corresponding to test inputs"]},"metrics":[{"key":"cost-per-task","display_name":"Cost Per Task","sources":["ARC Prize"],"models_count":89,"is_primary":false},{"key":"score","display_name":"Score","sources":["ARC Prize"],"models_count":89,"is_primary":true}],"slices":[],"summary_eval_ids":["arc-agi%2Farc-agi-v1-semi-private"],"reproducibility_summary":{"results_total":89,"has_reproducibility_gap_count":89,"populated_ratio_avg":0.9285714285714295},"provenance_summary":{"total_results":89,"total_groups":89,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":89,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":89,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["logical_reasoning","commonsense_reasoning","general"]},{"key":"arc-agi-v2-private-eval","display_name":"ARC-AGI v2 Private Eval","family_id":"arc-agi","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["artificial general intelligence","abstraction and reasoning","visual reasoning"],"languages":["Not specified"],"tasks":["Using demonstration pairs to understand a task's nature","Constructing output grids corresponding to test inputs"]},"metrics":[{"key":"cost-per-task","display_name":"Cost Per Task","sources":["ARC Prize"],"models_count":3,"is_primary":false},{"key":"score","display_name":"Score","sources":["ARC Prize"],"models_count":3,"is_primary":true}],"slices":[],"summary_eval_ids":["arc-agi%2Farc-agi-v2-private-eval"],"reproducibility_summary":{"results_total":3,"has_reproducibility_gap_count":3,"populated_ratio_avg":0.9285714285714285},"provenance_summary":{"total_results":3,"total_groups":3,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":3,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":3,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["logical_reasoning","commonsense_reasoning","general"]},{"key":"arc-agi-v2-public-eval","display_name":"ARC-AGI v2 Public Eval","family_id":"arc-agi","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["artificial general intelligence","abstraction and reasoning","visual reasoning"],"languages":["Not specified"],"tasks":["Using demonstration pairs to understand a task's nature","Constructing output grids corresponding to test inputs"]},"metrics":[{"key":"cost-per-task","display_name":"Cost Per Task","sources":["ARC Prize"],"models_count":85,"is_primary":false},{"key":"score","display_name":"Score","sources":["ARC Prize"],"models_count":85,"is_primary":true}],"slices":[],"summary_eval_ids":["arc-agi%2Farc-agi-v2-public-eval"],"reproducibility_summary":{"results_total":85,"has_reproducibility_gap_count":85,"populated_ratio_avg":0.9285714285714294},"provenance_summary":{"total_results":85,"total_groups":85,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":85,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":85,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["logical_reasoning","commonsense_reasoning","general"]},{"key":"arc-agi-v2-semi-private","display_name":"ARC-AGI v2 Semi-Private","family_id":"arc-agi","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["artificial general intelligence","abstraction and reasoning","visual reasoning"],"languages":["Not specified"],"tasks":["Using demonstration pairs to understand a task's nature","Constructing output grids corresponding to test inputs"]},"metrics":[{"key":"cost-per-task","display_name":"Cost Per Task","sources":["ARC Prize"],"models_count":92,"is_primary":false},{"key":"score","display_name":"Score","sources":["ARC Prize"],"models_count":92,"is_primary":true}],"slices":[],"summary_eval_ids":["arc-agi%2Farc-agi-v2-semi-private"],"reproducibility_summary":{"results_total":92,"has_reproducibility_gap_count":92,"populated_ratio_avg":0.9285714285714295},"provenance_summary":{"total_results":92,"total_groups":92,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":92,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":92,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["logical_reasoning","commonsense_reasoning","general"]},{"key":"arc-agi-v3-semi-private","display_name":"ARC-AGI v3 Semi-Private","family_id":"arc-agi","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["artificial general intelligence","abstraction and reasoning","visual reasoning"],"languages":["Not specified"],"tasks":["Using demonstration pairs to understand a task's nature","Constructing output grids corresponding to test inputs"]},"metrics":[{"key":"cost-per-task","display_name":"Cost Per Task","sources":["ARC Prize"],"models_count":4,"is_primary":false},{"key":"score","display_name":"Score","sources":["ARC Prize"],"models_count":4,"is_primary":true}],"slices":[],"summary_eval_ids":["arc-agi%2Farc-agi-v3-semi-private"],"reproducibility_summary":{"results_total":4,"has_reproducibility_gap_count":4,"populated_ratio_avg":0.9285714285714286},"provenance_summary":{"total_results":4,"total_groups":4,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":4,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":4,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["logical_reasoning","commonsense_reasoning","general"]}],"derivedTags":["logical_reasoning","commonsense_reasoning","general"]},{"key":"artificial-analysis","display_name":"Artificial Analysis","category":"Reasoning","tags":{"domains":["Not specified","STEM","airline services","biology","business","chemistry","code generation","coding competitions","computer programming","computer science","customer service","economics","engineering","health","history","humanities","law","materials science","math","mathematical problem-solving","mathematics","natural sciences","philosophy","physics","psychology","retail","scientific computing","social sciences","software development"],"languages":["English","Not specified","code"],"tasks":["Assisting simulated users with diverse requests","Code execution","Code generation","Code generation for scientific problems","Coding problem solving","Consulting policies","Gathering information","Mathematical problem solving","Multiple-choice question answering","Not specified","Numerical methods implementation","Scientific calculation","Self-repair","Single integer answer generation (0-999 range)","System simulation","Test output prediction","Text generation for solving mathematical problems","Using APIs to resolve user intents"]},"evals_count":16,"eval_summary_ids":["artificial-analysis-llms%2Faa-lcr","artificial-analysis-llms%2Faime","artificial-analysis-llms%2Fartificial-analysis-coding-index","artificial-analysis-llms%2Fartificial-analysis-intelligence-index","artificial-analysis-llms%2Fartificial-analysis-llms","artificial-analysis-llms%2Fartificial-analysis-math-index","artificial-analysis-llms%2Fgpqa","artificial-analysis-llms%2Fhle","artificial-analysis-llms%2Fifbench","artificial-analysis-llms%2Flivecodebench","artificial-analysis-llms%2Fmath-500","artificial-analysis-llms%2Fmmlu-pro","artificial-analysis-llms%2Fscicode","artificial-analysis-llms%2Ftau2-bench","artificial-analysis-llms%2Fterminalbench-hard"],"reproducibility_summary":{"results_total":3525,"has_reproducibility_gap_count":3525,"populated_ratio_avg":0.532654508611954},"provenance_summary":{"total_results":3525,"total_groups":3525,"multi_source_groups":374,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":3525,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":3525,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"benchmarks":[{"key":"aa-lcr","display_name":"AA-LCR","family_id":"aa-lcr","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["Artificial Analysis"],"models_count":236,"is_primary":true}],"slices":[{"key":"artificial analysis","display_name":"artificial analysis","is_bare_stem":false,"metrics":[{"key":"score","display_name":"Score","sources":["Artificial Analysis"]}],"derivedTags":["general"]}],"summary_eval_ids":["artificial-analysis-llms%2Faa-lcr"],"reproducibility_summary":{"results_total":236,"has_reproducibility_gap_count":236,"populated_ratio_avg":0.10714285714285744},"provenance_summary":{"total_results":236,"total_groups":236,"multi_source_groups":12,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":236,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":236,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["general","applied_reasoning"]},{"key":"artificial-analysis-coding-index","display_name":"Artificial Analysis Coding Index","family_id":"artificial-analysis-coding-index","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["Artificial Analysis"],"models_count":241,"is_primary":true}],"slices":[],"summary_eval_ids":["artificial-analysis-llms%2Fartificial-analysis-coding-index"],"reproducibility_summary":{"results_total":241,"has_reproducibility_gap_count":241,"populated_ratio_avg":0.10714285714285744},"provenance_summary":{"total_results":241,"total_groups":241,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":241,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":241,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["general"]},{"key":"artificial-analysis-intelligence-index","display_name":"Artificial Analysis Intelligence Index","family_id":"artificial-analysis-intelligence-index","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["Artificial Analysis"],"models_count":308,"is_primary":true}],"slices":[],"summary_eval_ids":["artificial-analysis-llms%2Fartificial-analysis-intelligence-index"],"reproducibility_summary":{"results_total":308,"has_reproducibility_gap_count":308,"populated_ratio_avg":0.10714285714285737},"provenance_summary":{"total_results":308,"total_groups":308,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":308,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":308,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":true,"derivedTags":["general"]},{"key":"artificial-analysis-llms","display_name":"Artificial Analysis LLMs","family_id":"artificial-analysis-llms","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["Not specified"],"languages":["Not specified"],"tasks":["Not specified"]},"metrics":[{"key":"score","display_name":"Score","sources":["Artificial Analysis"],"models_count":314,"is_primary":true}],"slices":[{"key":"artificial analysis","display_name":"artificial analysis","is_bare_stem":false,"metrics":[{"key":"score","display_name":"Score","sources":["Artificial Analysis"]}],"derivedTags":["general"]}],"summary_eval_ids":["artificial-analysis-llms%2Fartificial-analysis-llms"],"reproducibility_summary":{"results_total":314,"has_reproducibility_gap_count":314,"populated_ratio_avg":0.9285714285714258},"provenance_summary":{"total_results":314,"total_groups":314,"multi_source_groups":1,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":314,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":314,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["general"]},{"key":"artificial-analysis-math-index","display_name":"Artificial Analysis Math Index","family_id":"artificial-analysis-math-index","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["Artificial Analysis"],"models_count":165,"is_primary":true}],"slices":[],"summary_eval_ids":["artificial-analysis-llms%2Fartificial-analysis-math-index"],"reproducibility_summary":{"results_total":165,"has_reproducibility_gap_count":165,"populated_ratio_avg":0.10714285714285735},"provenance_summary":{"total_results":165,"total_groups":165,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":165,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":165,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["general"]},{"key":"gpqa","display_name":"GPQA","family_id":"gpqa","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["biology","physics","chemistry"],"languages":["English"],"tasks":["Multiple-choice question answering"]},"metrics":[{"key":"score","display_name":"Score","sources":["Artificial Analysis"],"models_count":288,"is_primary":true}],"slices":[{"key":"artificial analysis","display_name":"artificial analysis","is_bare_stem":false,"metrics":[{"key":"score","display_name":"Score","sources":["Artificial Analysis"]}],"derivedTags":["general"]}],"summary_eval_ids":["artificial-analysis-llms%2Fgpqa"],"reproducibility_summary":{"results_total":288,"has_reproducibility_gap_count":288,"populated_ratio_avg":0.928571428571424},"provenance_summary":{"total_results":288,"total_groups":288,"multi_source_groups":87,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":288,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":288,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["natural_sciences","general"]},{"key":"hle","display_name":"HLE","family_id":"hle","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["Artificial Analysis"],"models_count":284,"is_primary":true}],"slices":[{"key":"artificial analysis","display_name":"artificial analysis","is_bare_stem":false,"metrics":[{"key":"score","display_name":"Score","sources":["Artificial Analysis"]}],"derivedTags":["general"]}],"summary_eval_ids":["artificial-analysis-llms%2Fhle"],"reproducibility_summary":{"results_total":284,"has_reproducibility_gap_count":284,"populated_ratio_avg":0.10714285714285747},"provenance_summary":{"total_results":284,"total_groups":284,"multi_source_groups":14,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":284,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":284,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["general"]},{"key":"ifbench","display_name":"IFBench","family_id":"ifbench","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["Artificial Analysis"],"models_count":237,"is_primary":true}],"slices":[{"key":"artificial analysis","display_name":"artificial analysis","is_bare_stem":false,"metrics":[{"key":"score","display_name":"Score","sources":["Artificial Analysis"]}],"derivedTags":["general"]}],"summary_eval_ids":["artificial-analysis-llms%2Fifbench"],"reproducibility_summary":{"results_total":237,"has_reproducibility_gap_count":237,"populated_ratio_avg":0.10714285714285744},"provenance_summary":{"total_results":237,"total_groups":237,"multi_source_groups":13,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":237,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":237,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["linguistic_core","general"]},{"key":"math-500","display_name":"MATH-500","family_id":"math","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["mathematics"],"languages":["English"],"tasks":["Text generation for solving mathematical problems"]},"metrics":[{"key":"score","display_name":"Score","sources":["Artificial Analysis"],"models_count":138,"is_primary":true}],"slices":[{"key":"artificial analysis","display_name":"artificial analysis","is_bare_stem":false,"metrics":[{"key":"score","display_name":"Score","sources":["Artificial Analysis"]}],"derivedTags":["general"]}],"summary_eval_ids":["artificial-analysis-llms%2Fmath-500"],"reproducibility_summary":{"results_total":138,"has_reproducibility_gap_count":138,"populated_ratio_avg":0.9285714285714298},"provenance_summary":{"total_results":138,"total_groups":138,"multi_source_groups":37,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":138,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":138,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["mathematics","general"]},{"key":"scicode","display_name":"SciCode","family_id":"scicode","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["natural sciences","code generation","scientific computing","mathematics","physics","chemistry","biology","materials science"],"languages":["code"],"tasks":["Code generation for scientific problems","Numerical methods implementation","System simulation","Scientific calculation"]},"metrics":[{"key":"score","display_name":"Score","sources":["Artificial Analysis"],"models_count":283,"is_primary":true}],"slices":[{"key":"artificial analysis","display_name":"artificial analysis","is_bare_stem":false,"metrics":[{"key":"score","display_name":"Score","sources":["Artificial Analysis"]}],"derivedTags":["general"]}],"summary_eval_ids":["artificial-analysis-llms%2Fscicode"],"reproducibility_summary":{"results_total":283,"has_reproducibility_gap_count":283,"populated_ratio_avg":0.9285714285714237},"provenance_summary":{"total_results":283,"total_groups":283,"multi_source_groups":19,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":283,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":283,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["natural_sciences","software_engineering","general"]},{"key":"tau2-bench","display_name":"Tau2-Bench","family_id":"tau2-bench","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["customer service","retail","airline services"],"languages":["Not specified"],"tasks":["Assisting simulated users with diverse requests","Gathering information","Consulting policies","Using APIs to resolve user intents"]},"metrics":[{"key":"score","display_name":"Score","sources":["Artificial Analysis"],"models_count":231,"is_primary":true}],"slices":[],"summary_eval_ids":["artificial-analysis-llms%2Ftau2-bench"],"reproducibility_summary":{"results_total":231,"has_reproducibility_gap_count":231,"populated_ratio_avg":0.9285714285714244},"provenance_summary":{"total_results":231,"total_groups":231,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":231,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":231,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["agentic"]},{"key":"terminalbench-hard","display_name":"TerminalBench Hard","family_id":"terminal-bench","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["Artificial Analysis"],"models_count":228,"is_primary":true}],"slices":[{"key":"artificial analysis","display_name":"artificial analysis","is_bare_stem":false,"metrics":[{"key":"score","display_name":"Score","sources":["Artificial Analysis"]}],"derivedTags":["general"]}],"summary_eval_ids":["artificial-analysis-llms%2Fterminalbench-hard"],"reproducibility_summary":{"results_total":228,"has_reproducibility_gap_count":228,"populated_ratio_avg":0.10714285714285743},"provenance_summary":{"total_results":228,"total_groups":228,"multi_source_groups":32,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":228,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":228,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["agentic","general"]}],"derivedTags":["general","applied_reasoning","natural_sciences","linguistic_core","mathematics","software_engineering","agentic"]},{"key":"bfcl","display_name":"BFCL","category":"tool-use","tags":{"domains":["API interactions","function calling","multi-step reasoning","tool use"],"languages":["English"],"tasks":["Chatting capability","Function relevance detection","Java function calls","JavaScript function calls","Multi-turn/multi-step function calling","Multiple function selection","Parallel function calls","Parallel multiple function calls","REST API calls","SQL query generation","Simple function calls"]},"evals_count":15,"eval_summary_ids":["bfcl%2Fbfcl-live","bfcl%2Fbfcl-multi-turn","bfcl%2Fbfcl-non-live","bfcl%2Fbfcl-web-search","bfcl%2Fformat_sensitivity","bfcl%2Fmemory","bfcl%2Frelevance"],"reproducibility_summary":{"results_total":562,"has_reproducibility_gap_count":562,"populated_ratio_avg":0.6157854600915104},"provenance_summary":{"total_results":562,"total_groups":562,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":562,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":562,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"benchmarks":[],"standalone_benchmarks":[{"key":"bfcl","display_name":"BFCL","tags":{"domains":[],"languages":[],"tasks":[]},"summary_eval_ids":["bfcl%2Fbfcl-live","bfcl%2Fbfcl-multi-turn","bfcl%2Fbfcl-non-live","bfcl%2Fbfcl-web-search","bfcl%2Fformat_sensitivity","bfcl%2Fmemory","bfcl%2Frelevance"],"slices":[{"key":"bfcl","display_name":"BFCL","slice_key":null,"is_bare_stem":true,"metrics":[],"derivedTags":["agentic","software_engineering"]},{"key":"bfcl-live","display_name":"BFCL Live","slice_key":"bfcl-live","is_bare_stem":false,"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["UC Berkeley Gorilla"],"models_count":87,"is_primary":true},{"key":"ast-accuracy","display_name":"AST Accuracy","sources":["UC Berkeley Gorilla"],"models_count":87,"is_primary":false}],"derivedTags":["agentic","software_engineering"]},{"key":"bfcl-multi-turn","display_name":"BFCL Multi-turn","slice_key":"bfcl-multi-turn","is_bare_stem":false,"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["UC Berkeley Gorilla"],"models_count":87,"is_primary":true}],"derivedTags":["agentic","software_engineering"]},{"key":"bfcl-non-live","display_name":"BFCL Non-live","slice_key":"bfcl-non-live","is_bare_stem":false,"metrics":[{"key":"ast-accuracy","display_name":"AST Accuracy","sources":["UC Berkeley Gorilla"],"models_count":87,"is_primary":true}],"derivedTags":["agentic","software_engineering"]},{"key":"bfcl-web-search","display_name":"BFCL Web Search","slice_key":"bfcl-web-search","is_bare_stem":false,"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["UC Berkeley Gorilla"],"models_count":87,"is_primary":true}],"derivedTags":["agentic","software_engineering"]},{"key":"format_sensitivity","display_name":"Format Sensitivity","slice_key":"format_sensitivity","is_bare_stem":false,"metrics":[{"key":"max-delta","display_name":"max-delta","sources":["UC Berkeley Gorilla"],"models_count":40,"is_primary":false},{"key":"score","display_name":"Score","sources":["UC Berkeley Gorilla"],"models_count":40,"is_primary":true}],"derivedTags":["agentic","software_engineering"]},{"key":"memory","display_name":"Memory","slice_key":"memory","is_bare_stem":false,"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["UC Berkeley Gorilla"],"models_count":87,"is_primary":true}],"derivedTags":["agentic","software_engineering"]},{"key":"relevance","display_name":"Relevance","slice_key":"relevance","is_bare_stem":false,"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["UC Berkeley Gorilla"],"models_count":87,"is_primary":true}],"derivedTags":["agentic","software_engineering"]}],"derivedTags":["agentic","software_engineering"]}],"composites":[],"derivedTags":["agentic","software_engineering"]},{"key":"caparena","display_name":"CapArena-Auto","category":"General","tags":{"domains":[],"languages":[],"tasks":[]},"evals_count":5,"eval_summary_ids":["caparena-auto%2Fcaparena-auto-avg","caparena-auto%2Fcaparena-caption-length","caparena-auto%2Fcaparena-vs-cogvlm-19b","caparena-auto%2Fcaparena-vs-gpt-4o","caparena-auto%2Fcaparena-vs-minicpm-8b"],"reproducibility_summary":{"results_total":110,"has_reproducibility_gap_count":110,"populated_ratio_avg":0.10714285714285716},"provenance_summary":{"total_results":110,"total_groups":110,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":110,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":110,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"benchmarks":[],"standalone_benchmarks":[{"key":"caparena-auto","display_name":"CapArena-Auto","tags":{"domains":[],"languages":[],"tasks":[]},"summary_eval_ids":["caparena-auto%2Fcaparena-auto-avg","caparena-auto%2Fcaparena-caption-length","caparena-auto%2Fcaparena-vs-cogvlm-19b","caparena-auto%2Fcaparena-vs-gpt-4o","caparena-auto%2Fcaparena-vs-minicpm-8b"],"slices":[{"key":"caparena-auto","display_name":"CapArena-Auto","slice_key":null,"is_bare_stem":true,"metrics":[],"derivedTags":["agentic"]},{"key":"caparena-auto-avg","display_name":"Caparena-AUTO-AVG","slice_key":"caparena-auto-avg","is_bare_stem":false,"metrics":[{"key":"score","display_name":"Score","sources":["CapArena (Cheng et al., 2025)"],"models_count":22,"is_primary":true}],"derivedTags":["agentic"]},{"key":"caparena-caption-length","display_name":"Caparena-Caption-Length","slice_key":"caparena-caption-length","is_bare_stem":false,"metrics":[{"key":"score","display_name":"Score","sources":["CapArena (Cheng et al., 2025)"],"models_count":22,"is_primary":true}],"derivedTags":["agentic"]},{"key":"caparena-vs-cogvlm-19b","display_name":"Caparena-VS-Cogvlm-19b","slice_key":"caparena-vs-cogvlm-19b","is_bare_stem":false,"metrics":[{"key":"score","display_name":"Score","sources":["CapArena (Cheng et al., 2025)"],"models_count":22,"is_primary":true}],"derivedTags":["agentic"]},{"key":"caparena-vs-gpt-4o","display_name":"Caparena-VS-GPT-4o","slice_key":"caparena-vs-gpt-4o","is_bare_stem":false,"metrics":[{"key":"score","display_name":"Score","sources":["CapArena (Cheng et al., 2025)"],"models_count":22,"is_primary":true}],"derivedTags":["agentic"]},{"key":"caparena-vs-minicpm-8b","display_name":"Caparena-VS-Minicpm-8b","slice_key":"caparena-vs-minicpm-8b","is_bare_stem":false,"metrics":[{"key":"score","display_name":"Score","sources":["CapArena (Cheng et al., 2025)"],"models_count":22,"is_primary":true}],"derivedTags":["agentic"]}],"derivedTags":["agentic"]}],"composites":[],"derivedTags":["agentic"]},{"key":"cocoabench","display_name":"CocoaBench","category":"General","tags":{"domains":["business","entertainment","everyday tasks","research","shopping"],"languages":["Not specified"],"tasks":["Complex problem-solving requiring integration of multiple capabilities"]},"evals_count":2,"eval_summary_ids":["cocoabench%2Fcocoabench"],"reproducibility_summary":{"results_total":10,"has_reproducibility_gap_count":10,"populated_ratio_avg":0.9285714285714286},"provenance_summary":{"total_results":10,"total_groups":10,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":10,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":10,"groups_with_variant_check":2,"groups_with_cross_party_check":0,"variant_divergent_count":2,"cross_party_divergent_count":0},"standalone_benchmarks":[{"key":"cocoabench","display_name":"CocoaBench","family_id":"cocoabench","is_slice":false,"is_overall":true,"primary_metric_key":"score","has_card":true,"tags":{"domains":["research","entertainment","shopping","business","everyday tasks"],"languages":["Not specified"],"tasks":["Complex problem-solving requiring integration of multiple capabilities"]},"metrics":[{"key":"cost-per-task","display_name":"Cost Per Task","sources":["CocoaBench"],"models_count":7,"is_primary":false},{"key":"score","display_name":"Score","sources":["CocoaBench"],"models_count":10,"is_primary":true}],"slices":[],"summary_eval_ids":["cocoabench%2Fcocoabench"],"reproducibility_summary":{"results_total":10,"has_reproducibility_gap_count":10,"populated_ratio_avg":0.9285714285714286},"provenance_summary":{"total_results":10,"total_groups":10,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":10,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":10,"groups_with_variant_check":2,"groups_with_cross_party_check":0,"variant_divergent_count":2,"cross_party_divergent_count":0},"is_primary":true,"derivedTags":["agentic"]}],"derivedTags":["agentic"]},{"key":"commonsense-qa","display_name":"commonsense_qa","category":"General","tags":{"domains":[],"languages":[],"tasks":[]},"evals_count":1,"eval_summary_ids":["commonsense-qa%2Fcommonsense_qa"],"reproducibility_summary":{"results_total":11,"has_reproducibility_gap_count":11,"populated_ratio_avg":0.10714285714285712},"provenance_summary":{"total_results":11,"total_groups":11,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":11,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":11,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"standalone_benchmarks":[{"key":"commonsense_qa","display_name":"Commonsense Qa","family_id":"commonsense_qa","is_slice":false,"is_overall":false,"primary_metric_key":"accuracy","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["Arcadia Impact"],"models_count":11,"is_primary":true}],"slices":[],"summary_eval_ids":["commonsense-qa%2Fcommonsense_qa"],"reproducibility_summary":{"results_total":11,"has_reproducibility_gap_count":11,"populated_ratio_avg":0.10714285714285712},"provenance_summary":{"total_results":11,"total_groups":11,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":11,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":11,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":true,"derivedTags":["commonsense_reasoning"]}],"derivedTags":["commonsense_reasoning"]},{"key":"cvebench","display_name":"cvebench","category":"General","tags":{"domains":[],"languages":[],"tasks":[]},"evals_count":1,"eval_summary_ids":["cvebench%2Fcvebench"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":1,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":1,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"standalone_benchmarks":[{"key":"cvebench","display_name":"Cvebench","family_id":"cvebench","is_slice":false,"is_overall":true,"primary_metric_key":"mean","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"mean","display_name":"mean","sources":["Arcadia Impact"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["cvebench%2Fcvebench"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":1,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":1,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":true,"derivedTags":["software_engineering","safety"]}],"derivedTags":["software_engineering","safety"]},{"key":"cybench","display_name":"cybench","category":"General","tags":{"domains":["cybersecurity"],"languages":["Not specified"],"tasks":["Capture the Flag (CTF) tasks","Cybersecurity penetration testing"]},"evals_count":1,"eval_summary_ids":["cybench%2Fcybench"],"reproducibility_summary":{"results_total":9,"has_reproducibility_gap_count":9,"populated_ratio_avg":0.9285714285714286},"provenance_summary":{"total_results":9,"total_groups":9,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":9,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":9,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"standalone_benchmarks":[{"key":"cybench","display_name":"Cybench","family_id":"cybench","is_slice":false,"is_overall":true,"primary_metric_key":"accuracy","has_card":true,"tags":{"domains":["cybersecurity"],"languages":["Not specified"],"tasks":["Capture the Flag (CTF) tasks","Cybersecurity penetration testing"]},"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["Arcadia Impact"],"models_count":9,"is_primary":true}],"slices":[{"key":"cybench","display_name":"cybench","is_bare_stem":true,"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["Arcadia Impact"]}],"derivedTags":["software_engineering","safety"]}],"summary_eval_ids":["cybench%2Fcybench"],"reproducibility_summary":{"results_total":9,"has_reproducibility_gap_count":9,"populated_ratio_avg":0.9285714285714286},"provenance_summary":{"total_results":9,"total_groups":9,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":9,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":9,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":true,"derivedTags":["software_engineering","safety"]}],"derivedTags":["software_engineering","safety"]},{"key":"cyse2","display_name":"CySE2","category":"security","tags":{"domains":[],"languages":[],"tasks":[]},"evals_count":5,"eval_summary_ids":["cyse2-interpreter-abuse%2Fcyse2_interpreter_abuse","cyse2-prompt-injection%2Fcyse2_prompt_injection","cyse2-vulnerability-exploit%2Fcyse2_vulnerability_exploit"],"reproducibility_summary":{"results_total":31,"has_reproducibility_gap_count":31,"populated_ratio_avg":0.10714285714285712},"provenance_summary":{"total_results":31,"total_groups":31,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":31,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":31,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"composites":[],"standalone_benchmarks":[{"key":"cyse2","display_name":"CySE2","tags":{"domains":[],"languages":[],"tasks":[]},"summary_eval_ids":["cyse2-interpreter-abuse%2Fcyse2_interpreter_abuse","cyse2-prompt-injection%2Fcyse2_prompt_injection","cyse2-vulnerability-exploit%2Fcyse2_vulnerability_exploit"],"slices":[{"key":"cyse2","display_name":"CySE2","slice_key":null,"is_bare_stem":true,"metrics":[],"derivedTags":["software_engineering","safety"]},{"key":"cyse2_interpreter_abuse","display_name":"Cyse2 Interpreter Abuse","slice_key":"cyse2_interpreter_abuse","is_bare_stem":false,"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["Arcadia Impact"],"models_count":10,"is_primary":true}],"derivedTags":["software_engineering","safety"]},{"key":"cyse2_prompt_injection","display_name":"Cyse2 Prompt Injection","slice_key":"cyse2_prompt_injection","is_bare_stem":false,"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["Arcadia Impact"],"models_count":10,"is_primary":true}],"derivedTags":["software_engineering","safety"]},{"key":"cyse2_vulnerability_exploit","display_name":"Cyse2 Vulnerability Exploit","slice_key":"cyse2_vulnerability_exploit","is_bare_stem":false,"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["Arcadia Impact"],"models_count":11,"is_primary":true},{"key":"mean","display_name":"mean","sources":["Arcadia Impact"],"models_count":11,"is_primary":false},{"key":"std","display_name":"std","sources":["Arcadia Impact"],"models_count":11,"is_primary":false}],"derivedTags":["software_engineering","safety"]}],"derivedTags":["software_engineering","safety"]}],"benchmarks":[],"derivedTags":["software_engineering","safety"]},{"key":"exgentic-open-agent","display_name":"Exgentic Open Agent","category":"Agentic","tags":{"domains":["Not specified","app automation","autonomous agents","code generation","deep-research agents","information retrieval","interactive coding","issue resolution","network optimization","performance analysis","question answering","signal processing","software engineering","telecommunications","tool use"],"languages":["Not specified","Python"],"tasks":["Answering complex fact-seeking questions that require deep research","Evaluating end-to-end agents for answer generation","Evaluating retriever-only systems using retrieval metrics","Interactive code generation","Issue resolution","Multi-application interaction via APIs","Not specified","Reasoning and sequential decision-making for digital tasks","Solving mathematical problems with numerical solutions in the telecom domain"]},"evals_count":6,"eval_summary_ids":["exgentic-open-agent%2Fappworld-test-normal","exgentic-open-agent%2Fbrowsecompplus","exgentic-open-agent%2Fswe-bench","exgentic-open-agent%2Ftau2-airline","exgentic-open-agent%2Ftau2-retail","exgentic-open-agent%2Ftau2-telecom"],"reproducibility_summary":{"results_total":18,"has_reproducibility_gap_count":18,"populated_ratio_avg":0.9285714285714284},"provenance_summary":{"total_results":18,"total_groups":18,"multi_source_groups":6,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":18,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":18,"groups_with_variant_check":18,"groups_with_cross_party_check":0,"variant_divergent_count":18,"cross_party_divergent_count":0},"benchmarks":[{"key":"appworld-test-normal","display_name":"AppWorld Test Normal","family_id":"appworld","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["autonomous agents","interactive coding","tool use","app automation"],"languages":["Not specified"],"tasks":["Multi-application interaction via APIs","Interactive code generation","Reasoning and sequential decision-making for digital tasks"]},"metrics":[{"key":"score","display_name":"Score","sources":["Exgentic"],"models_count":3,"is_primary":true}],"slices":[],"summary_eval_ids":["exgentic-open-agent%2Fappworld-test-normal"],"reproducibility_summary":{"results_total":3,"has_reproducibility_gap_count":3,"populated_ratio_avg":0.9285714285714285},"provenance_summary":{"total_results":3,"total_groups":3,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":3,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":3,"groups_with_variant_check":3,"groups_with_cross_party_check":0,"variant_divergent_count":3,"cross_party_divergent_count":0},"is_primary":true,"derivedTags":["agentic"]},{"key":"browsecompplus","display_name":"BrowseCompPlus","family_id":"browsecompplus","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["information retrieval","question answering","deep-research agents"],"languages":[],"tasks":["Answering complex fact-seeking questions that require deep research","Evaluating end-to-end agents for answer generation","Evaluating retriever-only systems using retrieval metrics"]},"metrics":[{"key":"score","display_name":"Score","sources":["Exgentic"],"models_count":3,"is_primary":true}],"slices":[],"summary_eval_ids":["exgentic-open-agent%2Fbrowsecompplus"],"reproducibility_summary":{"results_total":3,"has_reproducibility_gap_count":3,"populated_ratio_avg":0.9285714285714285},"provenance_summary":{"total_results":3,"total_groups":3,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":3,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":3,"groups_with_variant_check":3,"groups_with_cross_party_check":0,"variant_divergent_count":3,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["agentic"]},{"key":"tau2-airline","display_name":"Tau2-Airline","family_id":"tau2-bench","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["Not specified"],"languages":["Not specified"],"tasks":["Not specified"]},"metrics":[{"key":"score","display_name":"Score","sources":["Exgentic"],"models_count":3,"is_primary":true}],"slices":[{"key":"tau-bench-2/airline","display_name":"tau-bench-2/airline","is_bare_stem":false,"metrics":[{"key":"score","display_name":"Score","sources":["Exgentic"]}],"derivedTags":["agentic"]}],"summary_eval_ids":["exgentic-open-agent%2Ftau2-airline"],"reproducibility_summary":{"results_total":3,"has_reproducibility_gap_count":3,"populated_ratio_avg":0.9285714285714285},"provenance_summary":{"total_results":3,"total_groups":3,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":3,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":3,"groups_with_variant_check":3,"groups_with_cross_party_check":0,"variant_divergent_count":3,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["agentic"]},{"key":"tau2-retail","display_name":"Tau2-Retail","family_id":"tau2-bench","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["Not specified"],"languages":["Not specified"],"tasks":["Not specified"]},"metrics":[{"key":"score","display_name":"Score","sources":["Exgentic"],"models_count":3,"is_primary":true}],"slices":[{"key":"tau-bench-2/retail","display_name":"tau-bench-2/retail","is_bare_stem":false,"metrics":[{"key":"score","display_name":"Score","sources":["Exgentic"]}],"derivedTags":["agentic"]}],"summary_eval_ids":["exgentic-open-agent%2Ftau2-retail"],"reproducibility_summary":{"results_total":3,"has_reproducibility_gap_count":3,"populated_ratio_avg":0.9285714285714285},"provenance_summary":{"total_results":3,"total_groups":3,"multi_source_groups":2,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":3,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":3,"groups_with_variant_check":3,"groups_with_cross_party_check":0,"variant_divergent_count":3,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["agentic"]},{"key":"tau2-telecom","display_name":"Tau2-Telecom","family_id":"tau2-bench","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["telecommunications","signal processing","network optimization","performance analysis"],"languages":["Not specified"],"tasks":["Solving mathematical problems with numerical solutions in the telecom domain"]},"metrics":[{"key":"score","display_name":"Score","sources":["Exgentic"],"models_count":3,"is_primary":true}],"slices":[{"key":"tau-bench-2/telecom","display_name":"tau-bench-2/telecom","is_bare_stem":false,"metrics":[{"key":"score","display_name":"Score","sources":["Exgentic"]}],"derivedTags":["agentic"]}],"summary_eval_ids":["exgentic-open-agent%2Ftau2-telecom"],"reproducibility_summary":{"results_total":3,"has_reproducibility_gap_count":3,"populated_ratio_avg":0.9285714285714285},"provenance_summary":{"total_results":3,"total_groups":3,"multi_source_groups":2,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":3,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":3,"groups_with_variant_check":3,"groups_with_cross_party_check":0,"variant_divergent_count":3,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["agentic"]}],"derivedTags":["agentic"]},{"key":"facts-grounding","display_name":"FACTS Grounding","category":"Knowledge","tags":{"domains":["Not specified"],"languages":["English"],"tasks":["question answering"]},"evals_count":1,"eval_summary_ids":["facts-grounding%2Ffacts-grounding"],"reproducibility_summary":{"results_total":34,"has_reproducibility_gap_count":34,"populated_ratio_avg":0.9285714285714279},"provenance_summary":{"total_results":34,"total_groups":34,"multi_source_groups":7,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":34,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":34,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"standalone_benchmarks":[{"key":"facts-grounding","display_name":"Facts Grounding","family_id":"facts-grounding","is_slice":false,"is_overall":true,"primary_metric_key":"accuracy","has_card":true,"tags":{"domains":["Not specified"],"languages":["English"],"tasks":["question answering"]},"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["Google DeepMind"],"models_count":34,"is_primary":true}],"slices":[{"key":"private","display_name":"Private","is_bare_stem":false,"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["Google DeepMind"]}],"derivedTags":["hallucination"]},{"key":"public","display_name":"Public","is_bare_stem":false,"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["Google DeepMind"]}],"derivedTags":["hallucination"]},{"key":"score","display_name":"Score","is_bare_stem":false,"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["Google DeepMind"]}],"derivedTags":["hallucination"]}],"summary_eval_ids":["facts-grounding%2Ffacts-grounding"],"reproducibility_summary":{"results_total":34,"has_reproducibility_gap_count":34,"populated_ratio_avg":0.9285714285714279},"provenance_summary":{"total_results":34,"total_groups":34,"multi_source_groups":7,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":34,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":34,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":true,"derivedTags":["hallucination"]}],"derivedTags":["hallucination"]},{"key":"fibble-arena","display_name":"Fibble Arena","category":"reasoning","tags":{"domains":["Not specified"],"languages":["Not specified"],"tasks":["Not specified"]},"evals_count":17,"eval_summary_ids":["fibble-arena%2Ffibble-arena","fibble1-arena%2Ffibble1-arena","fibble2-arena%2Ffibble2-arena","fibble3-arena%2Ffibble3-arena","fibble4-arena%2Ffibble4-arena","fibble5-arena%2Ffibble5-arena"],"reproducibility_summary":{"results_total":218,"has_reproducibility_gap_count":162,"populated_ratio_avg":0.27670380078636964},"provenance_summary":{"total_results":218,"total_groups":218,"multi_source_groups":88,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":218,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":218,"groups_with_variant_check":88,"groups_with_cross_party_check":88,"variant_divergent_count":23,"cross_party_divergent_count":22},"composites":[],"standalone_benchmarks":[{"key":"fibble-arena","display_name":"Fibble Arena","family_id":"fibble-arena","is_slice":false,"is_overall":true,"primary_metric_key":"score","has_card":true,"tags":{"domains":["Not specified"],"languages":["Not specified"],"tasks":["Not specified"]},"metrics":[{"key":"mean-response-time","display_name":"Mean Response Time","sources":["Wordle Arena Project"],"models_count":44,"is_primary":false},{"key":"score","display_name":"Score","sources":["Wordle Arena Project"],"models_count":45,"is_primary":true},{"key":"win-rate","display_name":"Win Rate","sources":["Wordle Arena Project"],"models_count":45,"is_primary":false}],"slices":[{"key":"fibble_arena","display_name":"fibble_arena","is_bare_stem":false,"metrics":[{"key":"mean-response-time","display_name":"Mean Response Time","sources":["Wordle Arena Project"]},{"key":"score","display_name":"Score","sources":["Wordle Arena Project"]},{"key":"win-rate","display_name":"Win Rate","sources":["Wordle Arena Project"]}],"derivedTags":["logical_reasoning"]},{"key":"fibble_arena_1lie","display_name":"fibble_arena_1lie","is_bare_stem":false,"metrics":[{"key":"mean-response-time","display_name":"Mean Response Time","sources":["Wordle Arena Project"]},{"key":"score","display_name":"Score","sources":["Wordle Arena Project"]},{"key":"win-rate","display_name":"Win Rate","sources":["Wordle Arena Project"]}],"derivedTags":["logical_reasoning"]},{"key":"fibble_arena_2lies","display_name":"fibble_arena_2lies","is_bare_stem":false,"metrics":[{"key":"mean-response-time","display_name":"Mean Response Time","sources":["Wordle Arena Project"]},{"key":"score","display_name":"Score","sources":["Wordle Arena Project"]},{"key":"win-rate","display_name":"Win Rate","sources":["Wordle Arena Project"]}],"derivedTags":["logical_reasoning"]},{"key":"fibble_arena_3lies","display_name":"fibble_arena_3lies","is_bare_stem":false,"metrics":[{"key":"mean-response-time","display_name":"Mean Response Time","sources":["Wordle Arena Project"]},{"key":"score","display_name":"Score","sources":["Wordle Arena Project"]},{"key":"win-rate","display_name":"Win Rate","sources":["Wordle Arena Project"]}],"derivedTags":["logical_reasoning"]},{"key":"fibble_arena_4lies","display_name":"fibble_arena_4lies","is_bare_stem":false,"metrics":[{"key":"mean-response-time","display_name":"Mean Response Time","sources":["Wordle Arena Project"]},{"key":"score","display_name":"Score","sources":["Wordle Arena Project"]},{"key":"win-rate","display_name":"Win Rate","sources":["Wordle Arena Project"]}],"derivedTags":["logical_reasoning"]},{"key":"fibble_arena_5lies","display_name":"fibble_arena_5lies","is_bare_stem":false,"metrics":[{"key":"mean-response-time","display_name":"Mean Response Time","sources":["Wordle Arena Project"]},{"key":"score","display_name":"Score","sources":["Wordle Arena Project"]},{"key":"win-rate","display_name":"Win Rate","sources":["Wordle Arena Project"]}],"derivedTags":["logical_reasoning"]}],"summary_eval_ids":["fibble-arena%2Ffibble-arena","fibble1-arena%2Ffibble1-arena","fibble2-arena%2Ffibble2-arena","fibble3-arena%2Ffibble3-arena","fibble4-arena%2Ffibble4-arena","fibble5-arena%2Ffibble5-arena"],"reproducibility_summary":{"results_total":45,"has_reproducibility_gap_count":0,"populated_ratio_avg":0.9285714285714286},"provenance_summary":{"total_results":45,"total_groups":45,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":45,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":45,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":true,"derivedTags":["logical_reasoning"]}],"benchmarks":[],"derivedTags":["logical_reasoning"]},{"key":"gdm-intercode-ctf","display_name":"gdm_intercode_ctf","category":"General","tags":{"domains":[],"languages":[],"tasks":[]},"evals_count":1,"eval_summary_ids":["gdm-intercode-ctf%2Fgdm_intercode_ctf"],"reproducibility_summary":{"results_total":8,"has_reproducibility_gap_count":8,"populated_ratio_avg":0.10714285714285712},"provenance_summary":{"total_results":8,"total_groups":8,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":8,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":8,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"standalone_benchmarks":[{"key":"gdm_intercode_ctf","display_name":"Gdm Intercode CTF","family_id":"gdm_intercode_ctf","is_slice":false,"is_overall":false,"primary_metric_key":"accuracy","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["Arcadia Impact"],"models_count":8,"is_primary":true}],"slices":[],"summary_eval_ids":["gdm-intercode-ctf%2Fgdm_intercode_ctf"],"reproducibility_summary":{"results_total":8,"has_reproducibility_gap_count":8,"populated_ratio_avg":0.10714285714285712},"provenance_summary":{"total_results":8,"total_groups":8,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":8,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":8,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":true,"derivedTags":["software_engineering","agentic","safety"]}],"derivedTags":["software_engineering","agentic","safety"]},{"key":"gsm-mc","display_name":"GSM-MC","category":"General","tags":{"domains":[],"languages":[],"tasks":[]},"evals_count":1,"eval_summary_ids":["gsm-mc%2FGSM-MC"],"reproducibility_summary":{"results_total":64,"has_reproducibility_gap_count":48,"populated_ratio_avg":0.10714285714285701},"provenance_summary":{"total_results":64,"total_groups":64,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":64,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":64,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"standalone_benchmarks":[{"key":"gsm-mc","display_name":"GSM-MC","tags":{"domains":[],"languages":[],"tasks":[]},"summary_eval_ids":["gsm-mc%2FGSM-MC"],"slices":[{"key":"gsm-mc","display_name":"GSM-MC","slice_key":null,"is_bare_stem":true,"metrics":[],"derivedTags":["mathematics"]},{"key":"GSM-MC","display_name":"GSM-MC","slice_key":"GSM-MC","is_bare_stem":false,"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["Writer, Inc."],"models_count":64,"is_primary":true}],"derivedTags":["mathematics"]}],"derivedTags":["mathematics"]}],"benchmarks":[],"composites":[],"derivedTags":["mathematics"]},{"key":"gsm8k","display_name":"gsm8k","category":"Reasoning","tags":{"domains":["grade school mathematics","math word problems"],"languages":["English"],"tasks":["Solving grade school math word problems","Text generation for question answering"]},"evals_count":1,"eval_summary_ids":["gsm8k%2Fgsm8k"],"reproducibility_summary":{"results_total":11,"has_reproducibility_gap_count":11,"populated_ratio_avg":0.9285714285714287},"provenance_summary":{"total_results":11,"total_groups":11,"multi_source_groups":6,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":11,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":11,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"standalone_benchmarks":[{"key":"gsm8k","display_name":"GSM8K","family_id":"gsm8k","is_slice":false,"is_overall":true,"primary_metric_key":"accuracy","has_card":true,"tags":{"domains":["grade school mathematics","math word problems"],"languages":["English"],"tasks":["Solving grade school math word problems","Text generation for question answering"]},"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["Arcadia Impact"],"models_count":11,"is_primary":true}],"slices":[{"key":"gsm8k","display_name":"gsm8k","is_bare_stem":true,"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["Arcadia Impact"]}],"derivedTags":["mathematics"]}],"summary_eval_ids":["gsm8k%2Fgsm8k"],"reproducibility_summary":{"results_total":11,"has_reproducibility_gap_count":11,"populated_ratio_avg":0.9285714285714287},"provenance_summary":{"total_results":11,"total_groups":11,"multi_source_groups":6,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":11,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":11,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":true,"derivedTags":["mathematics"]}],"derivedTags":["mathematics"]},{"key":"hal","display_name":"HAL Leaderboard","category":"agentic","tags":{"domains":["Not specified","airline","algorithmic reasoning","autonomous web agents","bioinformatics","biology","chemistry","code generation","commonsense reasoning","competitive programming","computational chemistry","computer science","geographical information science","information seeking","materials science","mathematical reasoning","mathematics","medicine","natural sciences","physics","psychology and cognitive neuroscience","real-world task automation","scientific computing","social science","user tasks (clothing, food, housing, transportation)","web interaction","work automation"],"languages":["English","Not specified","code"],"tasks":["A tool-calling agent interacting with a user simulator within the airline environment","Answering questions about the generated outputs","Answering real-world questions that require reasoning","Code generation for scientific problems","Code generation for scientific tasks","Completing diverse and realistic user tasks on real websites","Data selection","Feature selection","Geospatial analysis","Information-seeking activities requiring planning and information transfer across steps","Installing libraries and dependencies","Machine learning","Map visualization","Molecule visualization","Multi-modality handling","Numerical methods implementation","Performing sequences of operations (Click, Hover, Type, Select) on web page elements","Question answering","Reproducing results from research papers","Running code","Scientific calculation","Solving competitive programming problems by generating correct and efficient code from a problem description","System simulation","Tool-use proficiency","Web browsing","Web navigation"]},"evals_count":12,"eval_summary_ids":["hal%2Fassistantbench","hal%2Fcore-bench-hard","hal%2Fgaia","hal%2Fonline-mind2web","hal%2Fscicode","hal%2Fscienceagentbench","hal%2Fswe-bench-verified-mini-hal","hal%2Ftau-bench-airline","hal%2Fusaco"],"reproducibility_summary":{"results_total":101,"has_reproducibility_gap_count":101,"populated_ratio_avg":0.8309759547383313},"provenance_summary":{"total_results":101,"total_groups":101,"multi_source_groups":19,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":101,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":101,"groups_with_variant_check":3,"groups_with_cross_party_check":3,"variant_divergent_count":3,"cross_party_divergent_count":3},"benchmarks":[{"key":"assistantbench","display_name":"AssistantBench","family_id":"assistantbench","is_slice":false,"is_overall":false,"primary_metric_key":"accuracy","has_card":true,"tags":{"domains":["web interaction","information seeking","real-world task automation"],"languages":["English"],"tasks":["Question answering","Web navigation","Information-seeking activities requiring planning and information transfer across steps"]},"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["Princeton SAgE Team"],"models_count":10,"is_primary":true}],"slices":[],"summary_eval_ids":["hal%2Fassistantbench"],"reproducibility_summary":{"results_total":10,"has_reproducibility_gap_count":10,"populated_ratio_avg":0.9285714285714286},"provenance_summary":{"total_results":10,"total_groups":10,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":10,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":10,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":true,"derivedTags":["agentic","applied_reasoning"]},{"key":"core-bench-hard","display_name":"CORE-Bench Hard","family_id":"core-bench","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["computer science","social science","medicine"],"languages":[],"tasks":["Reproducing results from research papers","Installing libraries and dependencies","Running code","Answering questions about the generated outputs"]},"metrics":[{"key":"score","display_name":"Score","sources":["Princeton SAgE Team"],"models_count":17,"is_primary":true}],"slices":[],"summary_eval_ids":["hal%2Fcore-bench-hard"],"reproducibility_summary":{"results_total":17,"has_reproducibility_gap_count":17,"populated_ratio_avg":0.9285714285714287},"provenance_summary":{"total_results":17,"total_groups":17,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":17,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":17,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["applied_reasoning"]},{"key":"gaia","display_name":"GAIA","family_id":"gaia","is_slice":false,"is_overall":false,"primary_metric_key":"accuracy","has_card":true,"tags":{"domains":["Not specified"],"languages":["Not specified"],"tasks":["Answering real-world questions that require reasoning","Multi-modality handling","Web browsing","Tool-use proficiency"]},"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["Princeton SAgE Team"],"models_count":12,"is_primary":true}],"slices":[{"key":"gaia","display_name":"GAIA","is_bare_stem":true,"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["Princeton SAgE Team"]}],"derivedTags":["applied_reasoning","agentic"]},{"key":"gaia-level-1","display_name":"GAIA Level 1","is_bare_stem":false,"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["Princeton SAgE Team"]}],"derivedTags":["applied_reasoning","agentic"]},{"key":"gaia-level-2","display_name":"GAIA Level 2","is_bare_stem":false,"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["Princeton SAgE Team"]}],"derivedTags":["applied_reasoning","agentic"]},{"key":"gaia-level-3","display_name":"GAIA Level 3","is_bare_stem":false,"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["Princeton SAgE Team"]}],"derivedTags":["applied_reasoning","agentic"]}],"summary_eval_ids":["hal%2Fgaia"],"reproducibility_summary":{"results_total":12,"has_reproducibility_gap_count":12,"populated_ratio_avg":0.9285714285714287},"provenance_summary":{"total_results":12,"total_groups":12,"multi_source_groups":3,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":12,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":12,"groups_with_variant_check":3,"groups_with_cross_party_check":3,"variant_divergent_count":3,"cross_party_divergent_count":3},"is_primary":false,"derivedTags":["applied_reasoning","agentic"]},{"key":"online-mind2web","display_name":"Online-Mind2Web","family_id":"online-mind2web","is_slice":false,"is_overall":false,"primary_metric_key":"Task success rate","has_card":true,"tags":{"domains":["autonomous web agents","work automation","web interaction","user tasks (clothing, food, housing, transportation)"],"languages":["English"],"tasks":["Completing diverse and realistic user tasks on real websites","Performing sequences of operations (Click, Hover, Type, Select) on web page elements"]},"metrics":[{"key":"Task success rate","display_name":"Task success rate","sources":["Princeton SAgE Team"],"models_count":9,"is_primary":true}],"slices":[],"summary_eval_ids":["hal%2Fonline-mind2web"],"reproducibility_summary":{"results_total":9,"has_reproducibility_gap_count":9,"populated_ratio_avg":0.9285714285714286},"provenance_summary":{"total_results":9,"total_groups":9,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":9,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":9,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["agentic"]},{"key":"scicode","display_name":"SciCode","family_id":"scicode","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["natural sciences","code generation","scientific computing","mathematics","physics","chemistry","biology","materials science"],"languages":["code"],"tasks":["Code generation for scientific problems","Numerical methods implementation","System simulation","Scientific calculation"]},"metrics":[{"key":"score","display_name":"Score","sources":["Princeton SAgE Team"],"models_count":11,"is_primary":true}],"slices":[{"key":"scicode","display_name":"Scicode","is_bare_stem":true,"metrics":[{"key":"score","display_name":"Score","sources":["Princeton SAgE Team"]}],"derivedTags":["natural_sciences","software_engineering"]}],"summary_eval_ids":["hal%2Fscicode"],"reproducibility_summary":{"results_total":11,"has_reproducibility_gap_count":11,"populated_ratio_avg":0.9285714285714287},"provenance_summary":{"total_results":11,"total_groups":11,"multi_source_groups":11,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":11,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":11,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["natural_sciences","software_engineering"]},{"key":"scienceagentbench","display_name":"ScienceAgentBench","family_id":"scienceagentbench","is_slice":false,"is_overall":false,"primary_metric_key":"Success rate","has_card":true,"tags":{"domains":["bioinformatics","computational chemistry","geographical information science","psychology and cognitive neuroscience"],"languages":["English"],"tasks":["Code generation for scientific tasks","Feature selection","Data selection","Machine learning","Geospatial analysis","Map visualization","Molecule visualization"]},"metrics":[{"key":"Success rate","display_name":"Success rate","sources":["Princeton SAgE Team"],"models_count":11,"is_primary":true}],"slices":[],"summary_eval_ids":["hal%2Fscienceagentbench"],"reproducibility_summary":{"results_total":11,"has_reproducibility_gap_count":11,"populated_ratio_avg":0.9285714285714287},"provenance_summary":{"total_results":11,"total_groups":11,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":11,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":11,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["natural_sciences","agentic"]},{"key":"swe-bench-verified-mini-hal","display_name":"SWE-Bench-Verified-MINI-HAL","family_id":"swe-bench","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["Princeton SAgE Team"],"models_count":12,"is_primary":true}],"slices":[],"summary_eval_ids":["hal%2Fswe-bench-verified-mini-hal"],"reproducibility_summary":{"results_total":12,"has_reproducibility_gap_count":12,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":12,"total_groups":12,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":12,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":12,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["hallucination"]},{"key":"tau-bench-airline","display_name":"Tau Bench Airline","family_id":"tau-bench","is_slice":false,"is_overall":false,"primary_metric_key":"Task success rate","has_card":true,"tags":{"domains":["airline"],"languages":["Not specified"],"tasks":["A tool-calling agent interacting with a user simulator within the airline environment"]},"metrics":[{"key":"Task success rate","display_name":"Task success rate","sources":["Princeton SAgE Team"],"models_count":10,"is_primary":true}],"slices":[{"key":"tau-bench airline","display_name":"TAU-bench Airline","is_bare_stem":false,"metrics":[{"key":"Task success rate","display_name":"Task success rate","sources":["Princeton SAgE Team"]}],"derivedTags":["agentic"]}],"summary_eval_ids":["hal%2Ftau-bench-airline"],"reproducibility_summary":{"results_total":10,"has_reproducibility_gap_count":10,"populated_ratio_avg":0.9285714285714286},"provenance_summary":{"total_results":10,"total_groups":10,"multi_source_groups":5,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":10,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":10,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["agentic"]},{"key":"usaco","display_name":"USACO","family_id":"usaco","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["algorithmic reasoning","mathematical reasoning","commonsense reasoning","competitive programming"],"languages":["Not specified"],"tasks":["Solving competitive programming problems by generating correct and efficient code from a problem description"]},"metrics":[{"key":"score","display_name":"Score","sources":["Princeton SAgE Team"],"models_count":9,"is_primary":true}],"slices":[],"summary_eval_ids":["hal%2Fusaco"],"reproducibility_summary":{"results_total":9,"has_reproducibility_gap_count":9,"populated_ratio_avg":0.9285714285714286},"provenance_summary":{"total_results":9,"total_groups":9,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":9,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":9,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["mathematics","logical_reasoning"]}],"derivedTags":["hallucination","agentic","applied_reasoning","natural_sciences","software_engineering","mathematics","logical_reasoning"]},{"key":"hellaswag","display_name":"hellaswag","category":"Reasoning","tags":{"domains":["commonsense reasoning","natural language inference"],"languages":["English"],"tasks":["Commonsense inference","Four-way multiple-choice selection for event continuation"]},"evals_count":1,"eval_summary_ids":["hellaswag%2Fhellaswag"],"reproducibility_summary":{"results_total":14,"has_reproducibility_gap_count":14,"populated_ratio_avg":0.9285714285714287},"provenance_summary":{"total_results":14,"total_groups":14,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":14,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":14,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"standalone_benchmarks":[{"key":"hellaswag","display_name":"HellaSwag","family_id":"hellaswag","is_slice":false,"is_overall":true,"primary_metric_key":"accuracy","has_card":true,"tags":{"domains":["commonsense reasoning","natural language inference"],"languages":["English"],"tasks":["Four-way multiple-choice selection for event continuation","Commonsense inference"]},"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["Arcadia Impact"],"models_count":14,"is_primary":true}],"slices":[{"key":"hellaswag","display_name":"hellaswag","is_bare_stem":true,"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["Arcadia Impact"]}],"derivedTags":["commonsense_reasoning"]}],"summary_eval_ids":["hellaswag%2Fhellaswag"],"reproducibility_summary":{"results_total":14,"has_reproducibility_gap_count":14,"populated_ratio_avg":0.9285714285714287},"provenance_summary":{"total_results":14,"total_groups":14,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":14,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":14,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":true,"derivedTags":["commonsense_reasoning"]}],"derivedTags":["commonsense_reasoning"]},{"key":"helm","display_name":"HELM","category":"general","tags":{"domains":["AI alignment","AI safety","ChatGPT","Claude","Creative Tasks","GPT","Gemini","Info Seeking","LLM","Llama","Math & Data","Mistral","Not specified","Reasoning & Planning","STEM","US history","abstract algebra","academic research","anatomy","astronomy","automated red teaming","awesome list","bias detection","bias measurement","biology","business","business ethics","chatbot","chemistry","clinical knowledge","commonsense reasoning","computer science","conversational AI","critical safety risks","customer service","dialogue modeling","economics","elementary mathematics","engineering","explanation generation","finance","formal logic","geography","government and politics","grade school mathematics","health","history","human feedback","humanities","information retrieval","journalism","language generation","language understanding","large language model safety","law","legal","legal analysis","legal documents","machine learning fairness","machine reading comprehension","math","math word problems","mathematical reasoning","mathematics","medical knowledge","medical literature","medicine","morality","multilingual understanding","narrative comprehension","natural language inference","natural language understanding","news articles","news media","olympiads","philosophy","physics","politics","professional medical exams","prompt engineering","prompts","psychology","question answering","reading comprehension","reasoning","robust refusal","sentiment analysis","social media content","social sciences","stories and narratives","summarization","text classification","text generation","toxic comment classification","toxicity detection"],"languages":["Chinese","English","Not specified","multiple"],"tasks":["Analysis of performance across identity subgroups","Answering questions about stories by reading entire books or movie scripts","Answering questions using human-written summaries (summaries only)","Answering questions using the full story text (stories only)","Assessing model compliance with safe prompts","Assessing model refusal of unsafe prompts","Assessing the robustness of LLM refusal mechanisms against harmful behaviors","Binary sentiment classification","Binary toxicity classification (toxic vs. non-toxic)","Commonsense inference","Evaluating automated red teaming methods","Extractive question answering","Fill mask","Following verifiable instructions","Four-way multiple-choice selection for event continuation","Free-form multiple-choice question answering","General knowledge evaluation","Generating a final answer sequence given a math problem","Generating a well-formed answer based on context","Information retrieval","Knowledge evaluation across subjects including mathematics, history, computer science, law, morality, and philosophy","Legal reasoning","Machine translation","Mathematical reasoning","Medical question answering","Multiple-choice question answering","Natural language generation","News article summarization","Not specified","Open-domain question answering","Predicting if a question is answerable given context passages and synthesizing an answer","Preference modeling for comparing model responses","Question answering","Ranking retrieved passages for a question","Reading comprehension","Reasoning","Responding to prompts","Rule-application tasks","Sentiment analysis","Single-document extreme summarization","Solving Olympiad-level mathematical problems","Solving grade school math word problems","Summarization","Text classification","Text generation","Text generation for question answering","Text generation in response to sensitive prompts","Text summarization","Text-pair classification","Toxic comment identification","Toxicity detection","Training RLHF policies","Yes/no question answering"]},"evals_count":106,"eval_summary_ids":["helm-air-bench%2Fair-bench-2024","helm-capabilities%2Fgpqa","helm-capabilities%2Fhelm-capabilities-leaderboard","helm-capabilities%2Fifeval","helm-capabilities%2Fmmlu-pro","helm-capabilities%2Fomni-math","helm-capabilities%2Fwildbench","helm-classic%2Fboolq","helm-classic%2Fcivilcomments","helm-classic%2Fcnn-dailymail","helm-classic%2Fhellaswag","helm-classic%2Fhelm-classic-leaderboard","helm-classic%2Fimdb","helm-classic%2Fmmlu","helm-classic%2Fms-marco-trec","helm-classic%2Fnarrativeqa","helm-classic%2Fnaturalquestions","helm-classic%2Fopenbookqa","helm-classic%2Fquac","helm-classic%2Fraft","helm-classic%2Ftruthfulqa","helm-classic%2Fxsum","helm-instruct%2Fanthropic-rlhf-dataset","helm-instruct%2Fbest-chatgpt-prompts","helm-instruct%2Fhelm-instruct-leaderboard","helm-instruct%2Fkoala-test-dataset","helm-instruct%2Fopen-assistant","helm-instruct%2Fself-instruct","helm-instruct%2Fvicuna","helm-lite%2Fgsm8k","helm-lite%2Fhelm-lite-leaderboard","helm-lite%2Flegalbench","helm-lite%2Fmath","helm-lite%2Fmedqa","helm-lite%2Fmmlu","helm-lite%2Fnarrativeqa","helm-lite%2Fnaturalquestions","helm-lite%2Fopenbookqa","helm-lite%2Fwmt-2014","helm-mmlu%2Fmmlu","helm-safety%2Fanthropic-red-team","helm-safety%2Fbbq","helm-safety%2Fharmbench","helm-safety%2Fhelm-safety","helm-safety%2Fsimplesafetytests","helm-safety%2Fxstest","agentharm%2Fair-bench-2024-13-harassment","agentharm%2Fair-bench-2024-32-fraud","helm-air-bench%2Fair-bench-2024-1-confidentiality","helm-air-bench%2Fair-bench-2024-10-depicting-violence","helm-air-bench%2Fair-bench-2024-11-weapon-usage-development","helm-air-bench%2Fair-bench-2024-12-military-and-warfare","helm-air-bench%2Fair-bench-2024-13-harassment","helm-air-bench%2Fair-bench-2024-14-hate-speech-inciting-promoting-expressing-hatred","helm-air-bench%2Fair-bench-2024-15-perpetuating-harmful-beliefs","helm-air-bench%2Fair-bench-2024-16-offensive-language","helm-air-bench%2Fair-bench-2024-17-adult-content","helm-air-bench%2Fair-bench-2024-18-erotic","helm-air-bench%2Fair-bench-2024-19-non-consensual-nudity","helm-air-bench%2Fair-bench-2024-2-integrity","helm-air-bench%2Fair-bench-2024-20-monetized","helm-air-bench%2Fair-bench-2024-21-endangerment-harm-or-abuse-of-children","helm-air-bench%2Fair-bench-2024-22-child-sexual-abuse","helm-air-bench%2Fair-bench-2024-23-suicidal-and-non-suicidal-self-injury","helm-air-bench%2Fair-bench-2024-24-political-persuasion","helm-air-bench%2Fair-bench-2024-25-influencing-politics","helm-air-bench%2Fair-bench-2024-26-deterring-democratic-participation","helm-air-bench%2Fair-bench-2024-27-disrupting-social-order-china-unique","helm-air-bench%2Fair-bench-2024-28-high-risk-financial-activities","helm-air-bench%2Fair-bench-2024-29-unfair-market-practices","helm-air-bench%2Fair-bench-2024-3-availability","helm-air-bench%2Fair-bench-2024-30-disempowering-workers","helm-air-bench%2Fair-bench-2024-31-fraudulent-schemes","helm-air-bench%2Fair-bench-2024-32-fraud","helm-air-bench%2Fair-bench-2024-33-academic-dishonesty","helm-air-bench%2Fair-bench-2024-34-mis-disinformation","helm-air-bench%2Fair-bench-2024-35-sowing-division","helm-air-bench%2Fair-bench-2024-36-misrepresentation","helm-air-bench%2Fair-bench-2024-37-types-of-defamation","helm-air-bench%2Fair-bench-2024-38-specific-types-of-rights","helm-air-bench%2Fair-bench-2024-39-40-discrimination-protected-characteristics-combinations","helm-air-bench%2Fair-bench-2024-4-automated-decision-making","helm-air-bench%2Fair-bench-2024-41-42-privacy-violations-sensitive-data-combinations","helm-air-bench%2Fair-bench-2024-43-illegal-regulated-substances-goods","helm-air-bench%2Fair-bench-2024-44-illegal-services-exploitation","helm-air-bench%2Fair-bench-2024-45-other-illegal-unlawful-criminal-activities","helm-air-bench%2Fair-bench-2024-5-autonomous-unsafe-operation-of-systems","helm-air-bench%2Fair-bench-2024-6-advice-in-heavily-regulated-industries","helm-air-bench%2Fair-bench-2024-7-supporting-malicious-organized-groups","helm-air-bench%2Fair-bench-2024-8-celebrating-suffering","helm-air-bench%2Fair-bench-2024-9-violent-acts","helm-air-bench%2Fair-bench-2024-child-harm","helm-air-bench%2Fair-bench-2024-criminal-activities","helm-air-bench%2Fair-bench-2024-deception","helm-air-bench%2Fair-bench-2024-defamation","helm-air-bench%2Fair-bench-2024-discrimination-bias","helm-air-bench%2Fair-bench-2024-economic-harm","helm-air-bench%2Fair-bench-2024-fundamental-rights","helm-air-bench%2Fair-bench-2024-hate-toxicity","helm-air-bench%2Fair-bench-2024-manipulation","helm-air-bench%2Fair-bench-2024-operational-misuses","helm-air-bench%2Fair-bench-2024-political-usage","helm-air-bench%2Fair-bench-2024-privacy","helm-air-bench%2Fair-bench-2024-security-risks","helm-air-bench%2Fair-bench-2024-self-harm","helm-air-bench%2Fair-bench-2024-sexual-content","helm-air-bench%2Fair-bench-2024-violence-extremism","helm-air-bench%2Fairbench 2024","helm-air-bench%2Fairbench 2024 - #1.1: network intrusion","helm-air-bench%2Fairbench 2024 - #1.2: vulnerability probing","helm-air-bench%2Fairbench 2024 - #1.3: spoofing","helm-air-bench%2Fairbench 2024 - #1.4: spear phishing","helm-air-bench%2Fairbench 2024 - #1.5: social engineering","helm-air-bench%2Fairbench 2024 - #1.6: unauthorized network entry","helm-air-bench%2Fairbench 2024 - #10.12: bodily destruction","helm-air-bench%2Fairbench 2024 - #10.13: bodily mutilation","helm-air-bench%2Fairbench 2024 - #10.14: torture/abuse","helm-air-bench%2Fairbench 2024 - #10.15: animal abuse","helm-air-bench%2Fairbench 2024 - #10.16: activities meant to kill","helm-air-bench%2Fairbench 2024 - #11.17: guns","helm-air-bench%2Fairbench 2024 - #11.18: explosives/dangerous materials","helm-air-bench%2Fairbench 2024 - #11.19: bioweapons/viruses/gain-of-function","helm-air-bench%2Fairbench 2024 - #11.20: nuclear weapons","helm-air-bench%2Fairbench 2024 - #11.21: chemical weapons","helm-air-bench%2Fairbench 2024 - #11.22: radiological weapons","helm-air-bench%2Fairbench 2024 - #12.23: military","helm-air-bench%2Fairbench 2024 - #12.24: warfare","helm-air-bench%2Fairbench 2024 - #13.10: doxxing","helm-air-bench%2Fairbench 2024 - #13.11: cursing","helm-air-bench%2Fairbench 2024 - #13.1: bullying","helm-air-bench%2Fairbench 2024 - #13.2: threats","helm-air-bench%2Fairbench 2024 - #13.3: intimidation","helm-air-bench%2Fairbench 2024 - #13.4: shaming","helm-air-bench%2Fairbench 2024 - #13.5: humiliation","helm-air-bench%2Fairbench 2024 - #13.6: insults/personal attacks","helm-air-bench%2Fairbench 2024 - #13.7: abuse","helm-air-bench%2Fairbench 2024 - #13.8: provoking","helm-air-bench%2Fairbench 2024 - #13.9: trolling","helm-air-bench%2Fairbench 2024 - #14.12: race","helm-air-bench%2Fairbench 2024 - #14.13: ethnicity","helm-air-bench%2Fairbench 2024 - #14.14: color","helm-air-bench%2Fairbench 2024 - #14.15: gender","helm-air-bench%2Fairbench 2024 - #14.16: sexual orientation","helm-air-bench%2Fairbench 2024 - #14.17: religion","helm-air-bench%2Fairbench 2024 - #14.18: beliefs","helm-air-bench%2Fairbench 2024 - #14.19: nationality","helm-air-bench%2Fairbench 2024 - #14.20: geographic region","helm-air-bench%2Fairbench 2024 - #14.21: caste","helm-air-bench%2Fairbench 2024 - #14.22: social behaviors","helm-air-bench%2Fairbench 2024 - #14.23: physical characteristics","helm-air-bench%2Fairbench 2024 - #14.24: mental characteristics","helm-air-bench%2Fairbench 2024 - #14.25: personality","helm-air-bench%2Fairbench 2024 - #14.26: health conditions","helm-air-bench%2Fairbench 2024 - #14.27: disability","helm-air-bench%2Fairbench 2024 - #14.28: pregnancy status","helm-air-bench%2Fairbench 2024 - #14.29: genetic information","helm-air-bench%2Fairbench 2024 - #14.30: occupation","helm-air-bench%2Fairbench 2024 - #14.31: age","helm-air-bench%2Fairbench 2024 - #15.32: negative stereotyping of any group","helm-air-bench%2Fairbench 2024 - #15.33: perpetuating racism","helm-air-bench%2Fairbench 2024 - #15.34: perpetuating sexism","helm-air-bench%2Fairbench 2024 - #16.35: vulgarity","helm-air-bench%2Fairbench 2024 - #16.36: derogatory comments","helm-air-bench%2Fairbench 2024 - #17.1: obscenity","helm-air-bench%2Fairbench 2024 - #17.2: suggestive","helm-air-bench%2Fairbench 2024 - #17.3: sexual acts","helm-air-bench%2Fairbench 2024 - #17.4: sexual intercourse","helm-air-bench%2Fairbench 2024 - #18.5: erotic chats","helm-air-bench%2Fairbench 2024 - #18.6: fetishes","helm-air-bench%2Fairbench 2024 - #19.7: ncii (non-consensual intimate image)","helm-air-bench%2Fairbench 2024 - #2.10: control override (safety/privacy filters)","helm-air-bench%2Fairbench 2024 - #2.7: malware","helm-air-bench%2Fairbench 2024 - #2.8: packet forgery","helm-air-bench%2Fairbench 2024 - #2.9: data tampering","helm-air-bench%2Fairbench 2024 - #20.8: pornography","helm-air-bench%2Fairbench 2024 - #20.9: promotion of sexual services","helm-air-bench%2Fairbench 2024 - #21.1: grooming","helm-air-bench%2Fairbench 2024 - #21.2: pedophilia","helm-air-bench%2Fairbench 2024 - #21.3: exploiting/harming minors","helm-air-bench%2Fairbench 2024 - #21.4: building services targeting minors/failure to employ age-gating","helm-air-bench%2Fairbench 2024 - #21.5: building services to present a persona of minor","helm-air-bench%2Fairbench 2024 - #22.6: solicitation","helm-air-bench%2Fairbench 2024 - #22.7: csam","helm-air-bench%2Fairbench 2024 - #23.1: suicide","helm-air-bench%2Fairbench 2024 - #23.2: cutting","helm-air-bench%2Fairbench 2024 - #23.3: eating disorders (anorexia/bulimia)","helm-air-bench%2Fairbench 2024 - #24.1: lobbying","helm-air-bench%2Fairbench 2024 - #24.2: generating high-volume campaign materials","helm-air-bench%2Fairbench 2024 - #24.3: personalized or targeted campaign materials","helm-air-bench%2Fairbench 2024 - #24.4: building systems for political campaigning or lobbying","helm-air-bench%2Fairbench 2024 - #24.5: building products for political campaigning or lobbying","helm-air-bench%2Fairbench 2024 - #24.6: political advertisements","helm-air-bench%2Fairbench 2024 - #24.7: propaganda","helm-air-bench%2Fairbench 2024 - #25.8: influence political decisions","helm-air-bench%2Fairbench 2024 - #25.9: influence political opinions","helm-air-bench%2Fairbench 2024 - #26.10: deterring participation in democratic processes","helm-air-bench%2Fairbench 2024 - #26.11: misrepresenting voting processes","helm-air-bench%2Fairbench 2024 - #26.12: misrepresenting voting qualifications","helm-air-bench%2Fairbench 2024 - #26.13: discouraging voting","helm-air-bench%2Fairbench 2024 - #27.14: opposing constitutional principles","helm-air-bench%2Fairbench 2024 - #27.15: subverting state power","helm-air-bench%2Fairbench 2024 - #27.16: undermining national unity","helm-air-bench%2Fairbench 2024 - #27.17: damaging state interests","helm-air-bench%2Fairbench 2024 - #27.18: damaging the state’s honor","helm-air-bench%2Fairbench 2024 - #27.19: inciting unlawful assemblies","helm-air-bench%2Fairbench 2024 - #27.20: inciting unlawful associations","helm-air-bench%2Fairbench 2024 - #27.21: inciting unlawful processions","helm-air-bench%2Fairbench 2024 - #27.22: inciting unlawful demonstrations","helm-air-bench%2Fairbench 2024 - #27.23: undermining religious policies","helm-air-bench%2Fairbench 2024 - #27.24: promoting cults","helm-air-bench%2Fairbench 2024 - #27.25: promoting feudal superstitions","helm-air-bench%2Fairbench 2024 - #28.1: gambling (e.g., sports betting)","helm-air-bench%2Fairbench 2024 - #28.2: payday lending","helm-air-bench%2Fairbench 2024 - #29.3: exploiting advantages for monopolistic practices","helm-air-bench%2Fairbench 2024 - #29.4: anticompetitive practices","helm-air-bench%2Fairbench 2024 - #3.11: system/website impairment","helm-air-bench%2Fairbench 2024 - #3.12: network disruption","helm-air-bench%2Fairbench 2024 - #30.5: undermine workers' rights","helm-air-bench%2Fairbench 2024 - #30.6: worsen job quality","helm-air-bench%2Fairbench 2024 - #30.7: encourage undue worker surveillance","helm-air-bench%2Fairbench 2024 - #30.8: cause harmful labor-force disruptions","helm-air-bench%2Fairbench 2024 - #31.10: pyramid schemes","helm-air-bench%2Fairbench 2024 - #31.9: multi-level marketing","helm-air-bench%2Fairbench 2024 - #32.1: spam","helm-air-bench%2Fairbench 2024 - #32.2: scams","helm-air-bench%2Fairbench 2024 - #32.3: phishing/catfishing","helm-air-bench%2Fairbench 2024 - #32.4: pseudo-pharmaceuticals","helm-air-bench%2Fairbench 2024 - #32.5: impersonating others","helm-air-bench%2Fairbench 2024 - #33.6: plagiarism","helm-air-bench%2Fairbench 2024 - #33.7: promoting academic dishonesty","helm-air-bench%2Fairbench 2024 - #34.8: generating or promoting misinformation","helm-air-bench%2Fairbench 2024 - #34.9: fake online engagement (fake reviews, fake grassroots support)","helm-air-bench%2Fairbench 2024 - #35.1: inducing internal conflict","helm-air-bench%2Fairbench 2024 - #35.2: deflecting scrutiny from harmful actions","helm-air-bench%2Fairbench 2024 - #36.3: automated social media posts","helm-air-bench%2Fairbench 2024 - #36.4: not labeling content as ai-generated (using chatbots to convince people they are communicating with a human)","helm-air-bench%2Fairbench 2024 - #36.5: impersonating humans","helm-air-bench%2Fairbench 2024 - #37.1: disparagement","helm-air-bench%2Fairbench 2024 - #37.2: libel","helm-air-bench%2Fairbench 2024 - #37.3: slander","helm-air-bench%2Fairbench 2024 - #38.1: ip rights/ trade secrets","helm-air-bench%2Fairbench 2024 - #38.2: likeness rights","helm-air-bench%2Fairbench 2024 - #38.3: reputation rights","helm-air-bench%2Fairbench 2024 - #38.4: honor","helm-air-bench%2Fairbench 2024 - #38.5: name rights","helm-air-bench%2Fairbench 2024 - #39-40.10: discrimination in employment, benefits, or services - caste","helm-air-bench%2Fairbench 2024 - #39-40.11: discrimination in employment, benefits, or services - social behaviors","helm-air-bench%2Fairbench 2024 - #39-40.12: discrimination in employment, benefits, or services - physical characteristics","helm-air-bench%2Fairbench 2024 - #39-40.13: discrimination in employment, benefits, or services - mental characteristics","helm-air-bench%2Fairbench 2024 - #39-40.14: discrimination in employment, benefits, or services - predicted personality","helm-air-bench%2Fairbench 2024 - #39-40.15: discrimination in employment, benefits, or services - health conditions","helm-air-bench%2Fairbench 2024 - #39-40.16: discrimination in employment, benefits, or services - disability","helm-air-bench%2Fairbench 2024 - #39-40.17: discrimination in employment, benefits, or services - pregnancy status","helm-air-bench%2Fairbench 2024 - #39-40.18: discrimination in employment, benefits, or services - genetic information","helm-air-bench%2Fairbench 2024 - #39-40.19: discrimination in employment, benefits, or services - occupation","helm-air-bench%2Fairbench 2024 - #39-40.1: discrimination in employment, benefits, or services - race","helm-air-bench%2Fairbench 2024 - #39-40.20: discrimination in employment, benefits, or services - age","helm-air-bench%2Fairbench 2024 - #39-40.21: characterization of identity - race","helm-air-bench%2Fairbench 2024 - #39-40.22: characterization of identity - ethnicity","helm-air-bench%2Fairbench 2024 - #39-40.23: characterization of identity - color","helm-air-bench%2Fairbench 2024 - #39-40.24: characterization of identity - gender","helm-air-bench%2Fairbench 2024 - #39-40.25: characterization of identity - sexual orientation","helm-air-bench%2Fairbench 2024 - #39-40.26: characterization of identity - religion","helm-air-bench%2Fairbench 2024 - #39-40.27: characterization of identity - beliefs","helm-air-bench%2Fairbench 2024 - #39-40.28: characterization of identity - nationality","helm-air-bench%2Fairbench 2024 - #39-40.29: characterization of identity - geographic region","helm-air-bench%2Fairbench 2024 - #39-40.2: discrimination in employment, benefits, or services - ethnicity","helm-air-bench%2Fairbench 2024 - #39-40.30: characterization of identity - caste","helm-air-bench%2Fairbench 2024 - #39-40.31: characterization of identity - social behaviors","helm-air-bench%2Fairbench 2024 - #39-40.32: characterization of identity - physical characteristics","helm-air-bench%2Fairbench 2024 - #39-40.33: characterization of identity - mental characteristics","helm-air-bench%2Fairbench 2024 - #39-40.34: characterization of identity - predicted personality","helm-air-bench%2Fairbench 2024 - #39-40.35: characterization of identity - health conditions","helm-air-bench%2Fairbench 2024 - #39-40.36: characterization of identity - disability","helm-air-bench%2Fairbench 2024 - #39-40.37: characterization of identity - pregnancy status","helm-air-bench%2Fairbench 2024 - #39-40.38: characterization of identity - genetic information","helm-air-bench%2Fairbench 2024 - #39-40.39: characterization of identity - occupation","helm-air-bench%2Fairbench 2024 - #39-40.3: discrimination in employment, benefits, or services - color","helm-air-bench%2Fairbench 2024 - #39-40.40: characterization of identity - age","helm-air-bench%2Fairbench 2024 - #39-40.41: classification of individuals - race","helm-air-bench%2Fairbench 2024 - #39-40.42: classification of individuals - ethnicity","helm-air-bench%2Fairbench 2024 - #39-40.43: classification of individuals - color","helm-air-bench%2Fairbench 2024 - #39-40.44: classification of individuals - gender","helm-air-bench%2Fairbench 2024 - #39-40.45: classification of individuals - sexual orientation","helm-air-bench%2Fairbench 2024 - #39-40.46: classification of individuals - religion","helm-air-bench%2Fairbench 2024 - #39-40.47: classification of individuals - beliefs","helm-air-bench%2Fairbench 2024 - #39-40.48: classification of individuals - nationality","helm-air-bench%2Fairbench 2024 - #39-40.49: classification of individuals - geographic region","helm-air-bench%2Fairbench 2024 - #39-40.4: discrimination in employment, benefits, or services - gender","helm-air-bench%2Fairbench 2024 - #39-40.50: classification of individuals - caste","helm-air-bench%2Fairbench 2024 - #39-40.51: classification of individuals - social behaviors","helm-air-bench%2Fairbench 2024 - #39-40.52: classification of individuals - physical characteristics","helm-air-bench%2Fairbench 2024 - #39-40.53: classification of individuals - mental characteristics","helm-air-bench%2Fairbench 2024 - #39-40.54: classification of individuals - predicted personality","helm-air-bench%2Fairbench 2024 - #39-40.55: classification of individuals - health conditions","helm-air-bench%2Fairbench 2024 - #39-40.56: classification of individuals - disability","helm-air-bench%2Fairbench 2024 - #39-40.57: classification of individuals - pregnancy status","helm-air-bench%2Fairbench 2024 - #39-40.58: classification of individuals - genetic information","helm-air-bench%2Fairbench 2024 - #39-40.59: classification of individuals - occupation","helm-air-bench%2Fairbench 2024 - #39-40.5: discrimination in employment, benefits, or services - sexual orientation","helm-air-bench%2Fairbench 2024 - #39-40.60: classification of individuals - age","helm-air-bench%2Fairbench 2024 - #39-40.6: discrimination in employment, benefits, or services - religion","helm-air-bench%2Fairbench 2024 - #39-40.7: discrimination in employment, benefits, or services - beliefs","helm-air-bench%2Fairbench 2024 - #39-40.8: discrimination in employment, benefits, or services - nationality","helm-air-bench%2Fairbench 2024 - #39-40.9: discrimination in employment, benefits, or services - geographic region","helm-air-bench%2Fairbench 2024 - #4.10: profiling","helm-air-bench%2Fairbench 2024 - #4.1: financing eligibility/creditworthiness","helm-air-bench%2Fairbench 2024 - #4.2: criminal justice/predictive policing","helm-air-bench%2Fairbench 2024 - #4.3: adversely affects legal rights","helm-air-bench%2Fairbench 2024 - #4.4: employment","helm-air-bench%2Fairbench 2024 - #4.5: social scoring","helm-air-bench%2Fairbench 2024 - #4.6: housing eligibility","helm-air-bench%2Fairbench 2024 - #4.7: education eligibility","helm-air-bench%2Fairbench 2024 - #4.8: migration eligibility","helm-air-bench%2Fairbench 2024 - #4.9: insurance eligibility","helm-air-bench%2Fairbench 2024 - #41-42.10: unauthorized disclosure - pii (personal identifiable information)","helm-air-bench%2Fairbench 2024 - #41-42.11: unauthorized disclosure - health data","helm-air-bench%2Fairbench 2024 - #41-42.12: unauthorized disclosure - location data","helm-air-bench%2Fairbench 2024 - #41-42.13: unauthorized disclosure - demographic data","helm-air-bench%2Fairbench 2024 - #41-42.14: unauthorized disclosure - biometric data (facial recognition)","helm-air-bench%2Fairbench 2024 - #41-42.15: unauthorized disclosure - educational records","helm-air-bench%2Fairbench 2024 - #41-42.16: unauthorized disclosure - financial records","helm-air-bench%2Fairbench 2024 - #41-42.17: unauthorized disclosure - behavioral/ preference data","helm-air-bench%2Fairbench 2024 - #41-42.18: unauthorized disclosure - communication records","helm-air-bench%2Fairbench 2024 - #41-42.19: unauthorized distribution - pii (personal identifiable information)","helm-air-bench%2Fairbench 2024 - #41-42.1: unauthorized generation - pii (personal identifiable information)","helm-air-bench%2Fairbench 2024 - #41-42.20: unauthorized distribution - health data","helm-air-bench%2Fairbench 2024 - #41-42.21: unauthorized distribution - location data","helm-air-bench%2Fairbench 2024 - #41-42.22: unauthorized distribution - demographic data","helm-air-bench%2Fairbench 2024 - #41-42.23: unauthorized distribution - biometric data (facial recognition)","helm-air-bench%2Fairbench 2024 - #41-42.24: unauthorized distribution - educational records","helm-air-bench%2Fairbench 2024 - #41-42.25: unauthorized distribution - financial records","helm-air-bench%2Fairbench 2024 - #41-42.26: unauthorized distribution - behavioral/ preference data","helm-air-bench%2Fairbench 2024 - #41-42.27: unauthorized distribution - communication records","helm-air-bench%2Fairbench 2024 - #41-42.28: unauthorized collection/gathering/stealing - pii (personal identifiable information)","helm-air-bench%2Fairbench 2024 - #41-42.29: unauthorized collection/gathering/stealing - health data","helm-air-bench%2Fairbench 2024 - #41-42.2: unauthorized generation - health data","helm-air-bench%2Fairbench 2024 - #41-42.30: unauthorized collection/gathering/stealing - location data","helm-air-bench%2Fairbench 2024 - #41-42.31: unauthorized collection/gathering/stealing - demographic data","helm-air-bench%2Fairbench 2024 - #41-42.32: unauthorized collection/gathering/stealing - biometric data (facial recognition)","helm-air-bench%2Fairbench 2024 - #41-42.33: unauthorized collection/gathering/stealing - educational records","helm-air-bench%2Fairbench 2024 - #41-42.34: unauthorized collection/gathering/stealing - financial records","helm-air-bench%2Fairbench 2024 - #41-42.35: unauthorized collection/gathering/stealing - behavioral/ preference data","helm-air-bench%2Fairbench 2024 - #41-42.36: unauthorized collection/gathering/stealing - communication records","helm-air-bench%2Fairbench 2024 - #41-42.37: unauthorized processing - pii (personal identifiable information)","helm-air-bench%2Fairbench 2024 - #41-42.38: unauthorized processing - health data","helm-air-bench%2Fairbench 2024 - #41-42.39: unauthorized processing - location data","helm-air-bench%2Fairbench 2024 - #41-42.3: unauthorized generation - location data","helm-air-bench%2Fairbench 2024 - #41-42.40: unauthorized processing - demographic data","helm-air-bench%2Fairbench 2024 - #41-42.41: unauthorized processing - biometric data (facial recognition)","helm-air-bench%2Fairbench 2024 - #41-42.42: unauthorized processing - educational records","helm-air-bench%2Fairbench 2024 - #41-42.43: unauthorized processing - financial records","helm-air-bench%2Fairbench 2024 - #41-42.44: unauthorized processing - behavioral/ preference data","helm-air-bench%2Fairbench 2024 - #41-42.45: unauthorized processing - communication records","helm-air-bench%2Fairbench 2024 - #41-42.46: unauthorized inference/synthesis - pii (personal identifiable information)","helm-air-bench%2Fairbench 2024 - #41-42.47: unauthorized inference/synthesis - health data","helm-air-bench%2Fairbench 2024 - #41-42.48: unauthorized inference/synthesis - location data","helm-air-bench%2Fairbench 2024 - #41-42.49: unauthorized inference/synthesis - demographic data","helm-air-bench%2Fairbench 2024 - #41-42.4: unauthorized generation - demographic data","helm-air-bench%2Fairbench 2024 - #41-42.50: unauthorized inference/synthesis - biometric data (facial recognition)","helm-air-bench%2Fairbench 2024 - #41-42.51: unauthorized inference/synthesis - educational records","helm-air-bench%2Fairbench 2024 - #41-42.52: unauthorized inference/synthesis - financial records","helm-air-bench%2Fairbench 2024 - #41-42.53: unauthorized inference/synthesis - behavioral/ preference data","helm-air-bench%2Fairbench 2024 - #41-42.54: unauthorized inference/synthesis - communication records","helm-air-bench%2Fairbench 2024 - #41-42.55: non-consensual tracking/monitoring/stalking/spyware - pii (personal identifiable information)","helm-air-bench%2Fairbench 2024 - #41-42.56: non-consensual tracking/monitoring/stalking/spyware - health data","helm-air-bench%2Fairbench 2024 - #41-42.57: non-consensual tracking/monitoring/stalking/spyware - location data","helm-air-bench%2Fairbench 2024 - #41-42.58: non-consensual tracking/monitoring/stalking/spyware - demographic data","helm-air-bench%2Fairbench 2024 - #41-42.59: non-consensual tracking/monitoring/stalking/spyware - biometric data (facial recognition)","helm-air-bench%2Fairbench 2024 - #41-42.5: unauthorized generation - biometric data (facial recognition)","helm-air-bench%2Fairbench 2024 - #41-42.60: non-consensual tracking/monitoring/stalking/spyware - educational records","helm-air-bench%2Fairbench 2024 - #41-42.61: non-consensual tracking/monitoring/stalking/spyware - financial records","helm-air-bench%2Fairbench 2024 - #41-42.62: non-consensual tracking/monitoring/stalking/spyware - behavioral/ preference data","helm-air-bench%2Fairbench 2024 - #41-42.63: non-consensual tracking/monitoring/stalking/spyware - communication records","helm-air-bench%2Fairbench 2024 - #41-42.64: model attacks (membership inference, model inversion) - pii (personal identifiable information)","helm-air-bench%2Fairbench 2024 - #41-42.65: model attacks (membership inference, model inversion) - health data","helm-air-bench%2Fairbench 2024 - #41-42.66: model attacks (membership inference, model inversion) - location data","helm-air-bench%2Fairbench 2024 - #41-42.67: model attacks (membership inference, model inversion) - demographic data","helm-air-bench%2Fairbench 2024 - #41-42.68: model attacks (membership inference, model inversion) - biometric data (facial recognition)","helm-air-bench%2Fairbench 2024 - #41-42.69: model attacks (membership inference, model inversion) - educational records","helm-air-bench%2Fairbench 2024 - #41-42.6: unauthorized generation - educational records","helm-air-bench%2Fairbench 2024 - #41-42.70: model attacks (membership inference, model inversion) - financial records","helm-air-bench%2Fairbench 2024 - #41-42.71: model attacks (membership inference, model inversion) - behavioral/ preference data","helm-air-bench%2Fairbench 2024 - #41-42.72: model attacks (membership inference, model inversion) - communication records","helm-air-bench%2Fairbench 2024 - #41-42.7: unauthorized generation - financial records","helm-air-bench%2Fairbench 2024 - #41-42.8: unauthorized generation - behavioral/ preference data","helm-air-bench%2Fairbench 2024 - #41-42.9: unauthorized generation - communication records","helm-air-bench%2Fairbench 2024 - #43.1: illegal drugs","helm-air-bench%2Fairbench 2024 - #44.2: human trafficking","helm-air-bench%2Fairbench 2024 - #44.3: sexual exploitation","helm-air-bench%2Fairbench 2024 - #44.4: prostitution","helm-air-bench%2Fairbench 2024 - #45.5: undermining national security or other government interests","helm-air-bench%2Fairbench 2024 - #45.6: undermining social stability","helm-air-bench%2Fairbench 2024 - #45.7: undermining international relations","helm-air-bench%2Fairbench 2024 - #45.8: abetting/furthering activities violating any applicable law","helm-air-bench%2Fairbench 2024 - #5.11: heavy machinery","helm-air-bench%2Fairbench 2024 - #5.12: transportation","helm-air-bench%2Fairbench 2024 - #5.13: energy/electrical grids","helm-air-bench%2Fairbench 2024 - #5.14: nuclear facilities","helm-air-bench%2Fairbench 2024 - #5.15: aircraft navigation/air traffic control","helm-air-bench%2Fairbench 2024 - #5.16: communication systems","helm-air-bench%2Fairbench 2024 - #5.17: water treatment facilities","helm-air-bench%2Fairbench 2024 - #5.18: life support","helm-air-bench%2Fairbench 2024 - #5.19: weapon systems/battlefield management","helm-air-bench%2Fairbench 2024 - #5.20: emergency services","helm-air-bench%2Fairbench 2024 - #5.21: other unauthorized actions on behalf of users","helm-air-bench%2Fairbench 2024 - #6.22: legal","helm-air-bench%2Fairbench 2024 - #6.23: medical/pharmaceutical","helm-air-bench%2Fairbench 2024 - #6.24: accounting","helm-air-bench%2Fairbench 2024 - #6.25: financial","helm-air-bench%2Fairbench 2024 - #6.26: government services","helm-air-bench%2Fairbench 2024 - #7.1: extremism","helm-air-bench%2Fairbench 2024 - #7.2: terrorism","helm-air-bench%2Fairbench 2024 - #7.3: criminal organization","helm-air-bench%2Fairbench 2024 - #8.4: glorifying violence, abuse, or the suffering of others","helm-air-bench%2Fairbench 2024 - #8.5: belittling victimhood or violent events","helm-air-bench%2Fairbench 2024 - #8.6: denying or minimizing well-documented, major violent events or the victimhood of such events/denying the deeds of martyrdom","helm-air-bench%2Fairbench 2024 - #8.7: beautifying and whitewashing acts of war or aggression","helm-air-bench%2Fairbench 2024 - #9.10: property damage","helm-air-bench%2Fairbench 2024 - #9.11: environmental","helm-air-bench%2Fairbench 2024 - #9.8: persons (including murder)","helm-air-bench%2Fairbench 2024 - #9.9: animals"],"reproducibility_summary":{"results_total":2381,"has_reproducibility_gap_count":2381,"populated_ratio_avg":0.878202435951281},"provenance_summary":{"total_results":2381,"total_groups":2381,"multi_source_groups":240,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":2381,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":2381,"groups_with_variant_check":0,"groups_with_cross_party_check":9,"variant_divergent_count":0,"cross_party_divergent_count":6},"composites":[{"key":"helm-air-bench","display_name":"HELM AIR-Bench","category":"Safety","tags":{"domains":[],"languages":[],"tasks":[]},"evals_count":60,"benchmarks":[{"key":"air-bench-2024","display_name":"AIR-Bench 2024","family_id":"air-bench-2024","is_slice":false,"is_overall":false,"primary_metric_key":"Refusal Rate","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"],"models_count":72,"is_primary":true}],"slices":[{"key":"air-bench-2024-1-confidentiality","display_name":"AIR-Bench-2024-1-Confidentiality","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"air-bench-2024-10-depicting-violence","display_name":"AIR-Bench-2024-10-Depicting-Violence","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"air-bench-2024-11-weapon-usage-development","display_name":"AIR-Bench-2024-11-Weapon-Usage-Development","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"air-bench-2024-12-military-and-warfare","display_name":"AIR-Bench-2024-12-Military-AND-Warfare","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"air-bench-2024-13-harassment","display_name":"AIR-Bench-2024-13-Harassment","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"air-bench-2024-14-hate-speech-inciting-promoting-expressing-hatred","display_name":"AIR-Bench-2024-14-HATE-Speech-Inciting-Promoting-Expressing-Hatred","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"air-bench-2024-15-perpetuating-harmful-beliefs","display_name":"AIR-Bench-2024-15-Perpetuating-Harmful-Beliefs","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"air-bench-2024-16-offensive-language","display_name":"AIR-Bench-2024-16-Offensive-Language","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"air-bench-2024-17-adult-content","display_name":"AIR-Bench-2024-17-Adult-Content","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"air-bench-2024-18-erotic","display_name":"AIR-Bench-2024-18-Erotic","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"air-bench-2024-19-non-consensual-nudity","display_name":"AIR-Bench-2024-19-NON-Consensual-Nudity","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"air-bench-2024-2-integrity","display_name":"AIR-Bench-2024-2-Integrity","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"air-bench-2024-20-monetized","display_name":"AIR-Bench-2024-20-Monetized","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"air-bench-2024-21-endangerment-harm-or-abuse-of-children","display_name":"AIR-Bench-2024-21-Endangerment-HARM-OR-Abuse-OF-Children","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"air-bench-2024-22-child-sexual-abuse","display_name":"AIR-Bench-2024-22-Child-Sexual-Abuse","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"air-bench-2024-23-suicidal-and-non-suicidal-self-injury","display_name":"AIR-Bench-2024-23-Suicidal-AND-NON-Suicidal-SELF-Injury","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"air-bench-2024-24-political-persuasion","display_name":"AIR-Bench-2024-24-Political-Persuasion","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"air-bench-2024-25-influencing-politics","display_name":"AIR-Bench-2024-25-Influencing-Politics","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"air-bench-2024-26-deterring-democratic-participation","display_name":"AIR-Bench-2024-26-Deterring-Democratic-Participation","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"air-bench-2024-27-disrupting-social-order-china-unique","display_name":"AIR-Bench-2024-27-Disrupting-Social-Order-China-Unique","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"air-bench-2024-28-high-risk-financial-activities","display_name":"AIR-Bench-2024-28-HIGH-RISK-Financial-Activities","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"air-bench-2024-29-unfair-market-practices","display_name":"AIR-Bench-2024-29-Unfair-Market-Practices","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"air-bench-2024-3-availability","display_name":"AIR-Bench-2024-3-Availability","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"air-bench-2024-30-disempowering-workers","display_name":"AIR-Bench-2024-30-Disempowering-Workers","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"air-bench-2024-31-fraudulent-schemes","display_name":"AIR-Bench-2024-31-Fraudulent-Schemes","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"air-bench-2024-32-fraud","display_name":"AIR-Bench-2024-32-Fraud","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"air-bench-2024-33-academic-dishonesty","display_name":"AIR-Bench-2024-33-Academic-Dishonesty","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"air-bench-2024-34-mis-disinformation","display_name":"AIR-Bench-2024-34-MIS-Disinformation","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"air-bench-2024-35-sowing-division","display_name":"AIR-Bench-2024-35-Sowing-Division","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"air-bench-2024-36-misrepresentation","display_name":"AIR-Bench-2024-36-Misrepresentation","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"air-bench-2024-37-types-of-defamation","display_name":"AIR-Bench-2024-37-Types-OF-Defamation","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"air-bench-2024-38-specific-types-of-rights","display_name":"AIR-Bench-2024-38-Specific-Types-OF-Rights","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"air-bench-2024-39-40-discrimination-protected-characteristics-combinations","display_name":"AIR-Bench-2024-39-40-Discrimination-Protected-Characteristics-Combinations","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"air-bench-2024-4-automated-decision-making","display_name":"AIR-Bench-2024-4-Automated-Decision-Making","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"air-bench-2024-41-42-privacy-violations-sensitive-data-combinations","display_name":"AIR-Bench-2024-41-42-Privacy-Violations-Sensitive-DATA-Combinations","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"air-bench-2024-43-illegal-regulated-substances-goods","display_name":"AIR-Bench-2024-43-Illegal-Regulated-Substances-Goods","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"air-bench-2024-44-illegal-services-exploitation","display_name":"AIR-Bench-2024-44-Illegal-Services-Exploitation","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"air-bench-2024-45-other-illegal-unlawful-criminal-activities","display_name":"AIR-Bench-2024-45-Other-Illegal-Unlawful-Criminal-Activities","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"air-bench-2024-5-autonomous-unsafe-operation-of-systems","display_name":"AIR-Bench-2024-5-Autonomous-Unsafe-Operation-OF-Systems","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"air-bench-2024-6-advice-in-heavily-regulated-industries","display_name":"AIR-Bench-2024-6-Advice-IN-Heavily-Regulated-Industries","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"air-bench-2024-7-supporting-malicious-organized-groups","display_name":"AIR-Bench-2024-7-Supporting-Malicious-Organized-Groups","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"air-bench-2024-8-celebrating-suffering","display_name":"AIR-Bench-2024-8-Celebrating-Suffering","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"air-bench-2024-9-violent-acts","display_name":"AIR-Bench-2024-9-Violent-ACTS","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"air-bench-2024-child-harm","display_name":"AIR-Bench-2024-Child-HARM","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"air-bench-2024-criminal-activities","display_name":"AIR-Bench-2024-Criminal-Activities","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"air-bench-2024-deception","display_name":"AIR-Bench-2024-Deception","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"air-bench-2024-defamation","display_name":"AIR-Bench-2024-Defamation","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"air-bench-2024-discrimination-bias","display_name":"AIR-Bench-2024-Discrimination-BIAS","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"air-bench-2024-economic-harm","display_name":"AIR-Bench-2024-Economic-HARM","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"air-bench-2024-fundamental-rights","display_name":"AIR-Bench-2024-Fundamental-Rights","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"air-bench-2024-hate-toxicity","display_name":"AIR-Bench-2024-HATE-Toxicity","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"air-bench-2024-manipulation","display_name":"AIR-Bench-2024-Manipulation","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"air-bench-2024-operational-misuses","display_name":"AIR-Bench-2024-Operational-Misuses","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"air-bench-2024-political-usage","display_name":"AIR-Bench-2024-Political-Usage","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"air-bench-2024-privacy","display_name":"AIR-Bench-2024-Privacy","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"air-bench-2024-security-risks","display_name":"AIR-Bench-2024-Security-Risks","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"air-bench-2024-self-harm","display_name":"AIR-Bench-2024-SELF-HARM","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"air-bench-2024-sexual-content","display_name":"AIR-Bench-2024-Sexual-Content","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"air-bench-2024-violence-extremism","display_name":"AIR-Bench-2024-Violence-Extremism","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024","display_name":"AIRBench 2024","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #1.1: network intrusion","display_name":"AIRBench 2024 - #1.1: Network intrusion","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #1.2: vulnerability probing","display_name":"AIRBench 2024 - #1.2: Vulnerability probing","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #1.3: spoofing","display_name":"AIRBench 2024 - #1.3: Spoofing","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #1.4: spear phishing","display_name":"AIRBench 2024 - #1.4: Spear phishing","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #1.5: social engineering","display_name":"AIRBench 2024 - #1.5: Social engineering","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #1.6: unauthorized network entry","display_name":"AIRBench 2024 - #1.6: Unauthorized network entry","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #10.12: bodily destruction","display_name":"AIRBench 2024 - #10.12: Bodily destruction","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #10.13: bodily mutilation","display_name":"AIRBench 2024 - #10.13: Bodily mutilation","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #10.14: torture/abuse","display_name":"AIRBench 2024 - #10.14: Torture/abuse","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #10.15: animal abuse","display_name":"AIRBench 2024 - #10.15: Animal abuse","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #10.16: activities meant to kill","display_name":"AIRBench 2024 - #10.16: Activities meant to kill","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #11.17: guns","display_name":"AIRBench 2024 - #11.17: Guns","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #11.18: explosives/dangerous materials","display_name":"AIRBench 2024 - #11.18: Explosives/Dangerous Materials","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #11.19: bioweapons/viruses/gain-of-function","display_name":"AIRBench 2024 - #11.19: Bioweapons/Viruses/Gain-of-function","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #11.20: nuclear weapons","display_name":"AIRBench 2024 - #11.20: Nuclear Weapons","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #11.21: chemical weapons","display_name":"AIRBench 2024 - #11.21: Chemical Weapons","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #11.22: radiological weapons","display_name":"AIRBench 2024 - #11.22: Radiological Weapons","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #12.23: military","display_name":"AIRBench 2024 - #12.23: Military","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #12.24: warfare","display_name":"AIRBench 2024 - #12.24: Warfare","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #13.10: doxxing","display_name":"AIRBench 2024 - #13.10: Doxxing","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #13.11: cursing","display_name":"AIRBench 2024 - #13.11: Cursing","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #13.1: bullying","display_name":"AIRBench 2024 - #13.1: Bullying","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #13.2: threats","display_name":"AIRBench 2024 - #13.2: Threats","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #13.3: intimidation","display_name":"AIRBench 2024 - #13.3: Intimidation","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #13.4: shaming","display_name":"AIRBench 2024 - #13.4: Shaming","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #13.5: humiliation","display_name":"AIRBench 2024 - #13.5: Humiliation","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #13.6: insults/personal attacks","display_name":"AIRBench 2024 - #13.6: Insults/Personal attacks","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #13.7: abuse","display_name":"AIRBench 2024 - #13.7: Abuse","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #13.8: provoking","display_name":"AIRBench 2024 - #13.8: Provoking","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #13.9: trolling","display_name":"AIRBench 2024 - #13.9: Trolling","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #14.12: race","display_name":"AIRBench 2024 - #14.12: Race","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #14.13: ethnicity","display_name":"AIRBench 2024 - #14.13: Ethnicity","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #14.14: color","display_name":"AIRBench 2024 - #14.14: Color","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #14.15: gender","display_name":"AIRBench 2024 - #14.15: Gender","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #14.16: sexual orientation","display_name":"AIRBench 2024 - #14.16: Sexual orientation","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #14.17: religion","display_name":"AIRBench 2024 - #14.17: Religion","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #14.18: beliefs","display_name":"AIRBench 2024 - #14.18: Beliefs","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #14.19: nationality","display_name":"AIRBench 2024 - #14.19: Nationality","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #14.20: geographic region","display_name":"AIRBench 2024 - #14.20: Geographic region","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #14.21: caste","display_name":"AIRBench 2024 - #14.21: Caste","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #14.22: social behaviors","display_name":"AIRBench 2024 - #14.22: Social behaviors","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #14.23: physical characteristics","display_name":"AIRBench 2024 - #14.23: Physical characteristics","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #14.24: mental characteristics","display_name":"AIRBench 2024 - #14.24: Mental characteristics","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #14.25: personality","display_name":"AIRBench 2024 - #14.25: Personality","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #14.26: health conditions","display_name":"AIRBench 2024 - #14.26: Health conditions","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #14.27: disability","display_name":"AIRBench 2024 - #14.27: Disability","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #14.28: pregnancy status","display_name":"AIRBench 2024 - #14.28: Pregnancy status","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #14.29: genetic information","display_name":"AIRBench 2024 - #14.29: Genetic information","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #14.30: occupation","display_name":"AIRBench 2024 - #14.30: Occupation","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #14.31: age","display_name":"AIRBench 2024 - #14.31: Age","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #15.32: negative stereotyping of any group","display_name":"AIRBench 2024 - #15.32: Negative stereotyping of any group","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #15.33: perpetuating racism","display_name":"AIRBench 2024 - #15.33: Perpetuating racism","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #15.34: perpetuating sexism","display_name":"AIRBench 2024 - #15.34: Perpetuating sexism","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #16.35: vulgarity","display_name":"AIRBench 2024 - #16.35: Vulgarity","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #16.36: derogatory comments","display_name":"AIRBench 2024 - #16.36: Derogatory comments","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #17.1: obscenity","display_name":"AIRBench 2024 - #17.1: Obscenity","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #17.2: suggestive","display_name":"AIRBench 2024 - #17.2: Suggestive","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #17.3: sexual acts","display_name":"AIRBench 2024 - #17.3: Sexual acts","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #17.4: sexual intercourse","display_name":"AIRBench 2024 - #17.4: Sexual intercourse","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #18.5: erotic chats","display_name":"AIRBench 2024 - #18.5: Erotic chats","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #18.6: fetishes","display_name":"AIRBench 2024 - #18.6: Fetishes","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #19.7: ncii (non-consensual intimate image)","display_name":"AIRBench 2024 - #19.7: NCII (Non-consensual Intimate Image)","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #2.10: control override (safety/privacy filters)","display_name":"AIRBench 2024 - #2.10: Control override (safety/privacy filters)","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #2.7: malware","display_name":"AIRBench 2024 - #2.7: Malware","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #2.8: packet forgery","display_name":"AIRBench 2024 - #2.8: Packet forgery","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #2.9: data tampering","display_name":"AIRBench 2024 - #2.9: Data tampering","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #20.8: pornography","display_name":"AIRBench 2024 - #20.8: Pornography","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #20.9: promotion of sexual services","display_name":"AIRBench 2024 - #20.9: Promotion of sexual services","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #21.1: grooming","display_name":"AIRBench 2024 - #21.1: Grooming","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #21.2: pedophilia","display_name":"AIRBench 2024 - #21.2: Pedophilia","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #21.3: exploiting/harming minors","display_name":"AIRBench 2024 - #21.3: Exploiting/Harming minors","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #21.4: building services targeting minors/failure to employ age-gating","display_name":"AIRBench 2024 - #21.4: Building services targeting minors/failure to employ age-gating","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #21.5: building services to present a persona of minor","display_name":"AIRBench 2024 - #21.5: Building services to present a persona of minor","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #22.6: solicitation","display_name":"AIRBench 2024 - #22.6: Solicitation","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #22.7: csam","display_name":"AIRBench 2024 - #22.7: CSAM","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #23.1: suicide","display_name":"AIRBench 2024 - #23.1: Suicide","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #23.2: cutting","display_name":"AIRBench 2024 - #23.2: Cutting","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #23.3: eating disorders (anorexia/bulimia)","display_name":"AIRBench 2024 - #23.3: Eating Disorders (anorexia/bulimia)","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #24.1: lobbying","display_name":"AIRBench 2024 - #24.1: Lobbying","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #24.2: generating high-volume campaign materials","display_name":"AIRBench 2024 - #24.2: Generating high-volume campaign materials","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #24.3: personalized or targeted campaign materials","display_name":"AIRBench 2024 - #24.3: Personalized or targeted campaign materials","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #24.4: building systems for political campaigning or lobbying","display_name":"AIRBench 2024 - #24.4: Building systems for political campaigning or lobbying","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #24.5: building products for political campaigning or lobbying","display_name":"AIRBench 2024 - #24.5: Building products for political campaigning or lobbying","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #24.6: political advertisements","display_name":"AIRBench 2024 - #24.6: Political advertisements","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #24.7: propaganda","display_name":"AIRBench 2024 - #24.7: Propaganda","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #25.8: influence political decisions","display_name":"AIRBench 2024 - #25.8: Influence political decisions","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #25.9: influence political opinions","display_name":"AIRBench 2024 - #25.9: Influence political opinions","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #26.10: deterring participation in democratic processes","display_name":"AIRBench 2024 - #26.10: Deterring participation in democratic processes","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #26.11: misrepresenting voting processes","display_name":"AIRBench 2024 - #26.11: Misrepresenting voting processes","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #26.12: misrepresenting voting qualifications","display_name":"AIRBench 2024 - #26.12: Misrepresenting voting qualifications","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #26.13: discouraging voting","display_name":"AIRBench 2024 - #26.13: Discouraging voting","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #27.14: opposing constitutional principles","display_name":"AIRBench 2024 - #27.14: Opposing constitutional principles","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #27.15: subverting state power","display_name":"AIRBench 2024 - #27.15: Subverting state power","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #27.16: undermining national unity","display_name":"AIRBench 2024 - #27.16: Undermining national unity","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #27.17: damaging state interests","display_name":"AIRBench 2024 - #27.17: Damaging state interests","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #27.18: damaging the state’s honor","display_name":"AIRBench 2024 - #27.18: Damaging the state’s honor","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #27.19: inciting unlawful assemblies","display_name":"AIRBench 2024 - #27.19: Inciting unlawful assemblies","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #27.20: inciting unlawful associations","display_name":"AIRBench 2024 - #27.20: Inciting unlawful associations","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #27.21: inciting unlawful processions","display_name":"AIRBench 2024 - #27.21: Inciting unlawful processions","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #27.22: inciting unlawful demonstrations","display_name":"AIRBench 2024 - #27.22: Inciting unlawful demonstrations","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #27.23: undermining religious policies","display_name":"AIRBench 2024 - #27.23: Undermining religious policies","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #27.24: promoting cults","display_name":"AIRBench 2024 - #27.24: Promoting cults","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #27.25: promoting feudal superstitions","display_name":"AIRBench 2024 - #27.25: Promoting feudal superstitions","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #28.1: gambling (e.g., sports betting)","display_name":"AIRBench 2024 - #28.1: Gambling (e.g., sports betting)","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #28.2: payday lending","display_name":"AIRBench 2024 - #28.2: Payday lending","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #29.3: exploiting advantages for monopolistic practices","display_name":"AIRBench 2024 - #29.3: Exploiting advantages for monopolistic practices","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #29.4: anticompetitive practices","display_name":"AIRBench 2024 - #29.4: Anticompetitive practices","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #3.11: system/website impairment","display_name":"AIRBench 2024 - #3.11: System/Website impairment","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #3.12: network disruption","display_name":"AIRBench 2024 - #3.12: Network disruption","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #30.5: undermine workers' rights","display_name":"AIRBench 2024 - #30.5: Undermine workers' rights","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #30.6: worsen job quality","display_name":"AIRBench 2024 - #30.6: Worsen job quality","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #30.7: encourage undue worker surveillance","display_name":"AIRBench 2024 - #30.7: Encourage undue worker surveillance","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #30.8: cause harmful labor-force disruptions","display_name":"AIRBench 2024 - #30.8: Cause harmful labor-force disruptions","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #31.10: pyramid schemes","display_name":"AIRBench 2024 - #31.10: Pyramid schemes","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #31.9: multi-level marketing","display_name":"AIRBench 2024 - #31.9: Multi-level marketing","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #32.1: spam","display_name":"AIRBench 2024 - #32.1: Spam","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #32.2: scams","display_name":"AIRBench 2024 - #32.2: Scams","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #32.3: phishing/catfishing","display_name":"AIRBench 2024 - #32.3: Phishing/Catfishing","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #32.4: pseudo-pharmaceuticals","display_name":"AIRBench 2024 - #32.4: Pseudo-pharmaceuticals","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #32.5: impersonating others","display_name":"AIRBench 2024 - #32.5: Impersonating others","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #33.6: plagiarism","display_name":"AIRBench 2024 - #33.6: Plagiarism","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #33.7: promoting academic dishonesty","display_name":"AIRBench 2024 - #33.7: Promoting academic dishonesty","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #34.8: generating or promoting misinformation","display_name":"AIRBench 2024 - #34.8: Generating or promoting misinformation","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #34.9: fake online engagement (fake reviews, fake grassroots support)","display_name":"AIRBench 2024 - #34.9: Fake online engagement (fake reviews, fake grassroots support)","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #35.1: inducing internal conflict","display_name":"AIRBench 2024 - #35.1: Inducing internal conflict","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #35.2: deflecting scrutiny from harmful actions","display_name":"AIRBench 2024 - #35.2: Deflecting scrutiny from harmful actions","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #36.3: automated social media posts","display_name":"AIRBench 2024 - #36.3: Automated social media posts","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #36.4: not labeling content as ai-generated (using chatbots to convince people they are communicating with a human)","display_name":"AIRBench 2024 - #36.4: Not labeling content as AI-generated (Using chatbots to convince people they are communicating with a human)","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #36.5: impersonating humans","display_name":"AIRBench 2024 - #36.5: Impersonating humans","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #37.1: disparagement","display_name":"AIRBench 2024 - #37.1: Disparagement","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #37.2: libel","display_name":"AIRBench 2024 - #37.2: Libel","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #37.3: slander","display_name":"AIRBench 2024 - #37.3: Slander","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #38.1: ip rights/ trade secrets","display_name":"AIRBench 2024 - #38.1: IP rights/ Trade secrets","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #38.2: likeness rights","display_name":"AIRBench 2024 - #38.2: Likeness rights","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #38.3: reputation rights","display_name":"AIRBench 2024 - #38.3: Reputation rights","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #38.4: honor","display_name":"AIRBench 2024 - #38.4: Honor","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #38.5: name rights","display_name":"AIRBench 2024 - #38.5: Name rights","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #39-40.10: discrimination in employment, benefits, or services - caste","display_name":"AIRBench 2024 - #39-40.10: Discrimination in employment, benefits, or services - Caste","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #39-40.11: discrimination in employment, benefits, or services - social behaviors","display_name":"AIRBench 2024 - #39-40.11: Discrimination in employment, benefits, or services - Social behaviors","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #39-40.12: discrimination in employment, benefits, or services - physical characteristics","display_name":"AIRBench 2024 - #39-40.12: Discrimination in employment, benefits, or services - Physical Characteristics","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #39-40.13: discrimination in employment, benefits, or services - mental characteristics","display_name":"AIRBench 2024 - #39-40.13: Discrimination in employment, benefits, or services - Mental Characteristics","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #39-40.14: discrimination in employment, benefits, or services - predicted personality","display_name":"AIRBench 2024 - #39-40.14: Discrimination in employment, benefits, or services - Predicted Personality","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #39-40.15: discrimination in employment, benefits, or services - health conditions","display_name":"AIRBench 2024 - #39-40.15: Discrimination in employment, benefits, or services - Health conditions","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #39-40.16: discrimination in employment, benefits, or services - disability","display_name":"AIRBench 2024 - #39-40.16: Discrimination in employment, benefits, or services - Disability","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #39-40.17: discrimination in employment, benefits, or services - pregnancy status","display_name":"AIRBench 2024 - #39-40.17: Discrimination in employment, benefits, or services - Pregnancy Status","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #39-40.18: discrimination in employment, benefits, or services - genetic information","display_name":"AIRBench 2024 - #39-40.18: Discrimination in employment, benefits, or services - Genetic Information","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #39-40.19: discrimination in employment, benefits, or services - occupation","display_name":"AIRBench 2024 - #39-40.19: Discrimination in employment, benefits, or services - Occupation","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #39-40.1: discrimination in employment, benefits, or services - race","display_name":"AIRBench 2024 - #39-40.1: Discrimination in employment, benefits, or services - Race","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #39-40.20: discrimination in employment, benefits, or services - age","display_name":"AIRBench 2024 - #39-40.20: Discrimination in employment, benefits, or services - Age","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #39-40.21: characterization of identity - race","display_name":"AIRBench 2024 - #39-40.21: Characterization of identity - Race","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #39-40.22: characterization of identity - ethnicity","display_name":"AIRBench 2024 - #39-40.22: Characterization of identity - Ethnicity","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #39-40.23: characterization of identity - color","display_name":"AIRBench 2024 - #39-40.23: Characterization of identity - Color","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #39-40.24: characterization of identity - gender","display_name":"AIRBench 2024 - #39-40.24: Characterization of identity - Gender","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #39-40.25: characterization of identity - sexual orientation","display_name":"AIRBench 2024 - #39-40.25: Characterization of identity - Sexual orientation","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #39-40.26: characterization of identity - religion","display_name":"AIRBench 2024 - #39-40.26: Characterization of identity - Religion","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #39-40.27: characterization of identity - beliefs","display_name":"AIRBench 2024 - #39-40.27: Characterization of identity - Beliefs","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #39-40.28: characterization of identity - nationality","display_name":"AIRBench 2024 - #39-40.28: Characterization of identity - Nationality","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #39-40.29: characterization of identity - geographic region","display_name":"AIRBench 2024 - #39-40.29: Characterization of identity - Geographic region","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #39-40.2: discrimination in employment, benefits, or services - ethnicity","display_name":"AIRBench 2024 - #39-40.2: Discrimination in employment, benefits, or services - Ethnicity","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #39-40.30: characterization of identity - caste","display_name":"AIRBench 2024 - #39-40.30: Characterization of identity - Caste","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #39-40.31: characterization of identity - social behaviors","display_name":"AIRBench 2024 - #39-40.31: Characterization of identity - Social behaviors","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #39-40.32: characterization of identity - physical characteristics","display_name":"AIRBench 2024 - #39-40.32: Characterization of identity - Physical Characteristics","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #39-40.33: characterization of identity - mental characteristics","display_name":"AIRBench 2024 - #39-40.33: Characterization of identity - Mental Characteristics","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #39-40.34: characterization of identity - predicted personality","display_name":"AIRBench 2024 - #39-40.34: Characterization of identity - Predicted Personality","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #39-40.35: characterization of identity - health conditions","display_name":"AIRBench 2024 - #39-40.35: Characterization of identity - Health conditions","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #39-40.36: characterization of identity - disability","display_name":"AIRBench 2024 - #39-40.36: Characterization of identity - Disability","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #39-40.37: characterization of identity - pregnancy status","display_name":"AIRBench 2024 - #39-40.37: Characterization of identity - Pregnancy Status","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #39-40.38: characterization of identity - genetic information","display_name":"AIRBench 2024 - #39-40.38: Characterization of identity - Genetic Information","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #39-40.39: characterization of identity - occupation","display_name":"AIRBench 2024 - #39-40.39: Characterization of identity - Occupation","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #39-40.3: discrimination in employment, benefits, or services - color","display_name":"AIRBench 2024 - #39-40.3: Discrimination in employment, benefits, or services - Color","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #39-40.40: characterization of identity - age","display_name":"AIRBench 2024 - #39-40.40: Characterization of identity - Age","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #39-40.41: classification of individuals - race","display_name":"AIRBench 2024 - #39-40.41: Classification of individuals - Race","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #39-40.42: classification of individuals - ethnicity","display_name":"AIRBench 2024 - #39-40.42: Classification of individuals - Ethnicity","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #39-40.43: classification of individuals - color","display_name":"AIRBench 2024 - #39-40.43: Classification of individuals - Color","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #39-40.44: classification of individuals - gender","display_name":"AIRBench 2024 - #39-40.44: Classification of individuals - Gender","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #39-40.45: classification of individuals - sexual orientation","display_name":"AIRBench 2024 - #39-40.45: Classification of individuals - Sexual orientation","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #39-40.46: classification of individuals - religion","display_name":"AIRBench 2024 - #39-40.46: Classification of individuals - Religion","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #39-40.47: classification of individuals - beliefs","display_name":"AIRBench 2024 - #39-40.47: Classification of individuals - Beliefs","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #39-40.48: classification of individuals - nationality","display_name":"AIRBench 2024 - #39-40.48: Classification of individuals - Nationality","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #39-40.49: classification of individuals - geographic region","display_name":"AIRBench 2024 - #39-40.49: Classification of individuals - Geographic region","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #39-40.4: discrimination in employment, benefits, or services - gender","display_name":"AIRBench 2024 - #39-40.4: Discrimination in employment, benefits, or services - Gender","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #39-40.50: classification of individuals - caste","display_name":"AIRBench 2024 - #39-40.50: Classification of individuals - Caste","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #39-40.51: classification of individuals - social behaviors","display_name":"AIRBench 2024 - #39-40.51: Classification of individuals - Social behaviors","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #39-40.52: classification of individuals - physical characteristics","display_name":"AIRBench 2024 - #39-40.52: Classification of individuals - Physical Characteristics","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #39-40.53: classification of individuals - mental characteristics","display_name":"AIRBench 2024 - #39-40.53: Classification of individuals - Mental Characteristics","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #39-40.54: classification of individuals - predicted personality","display_name":"AIRBench 2024 - #39-40.54: Classification of individuals - Predicted Personality","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #39-40.55: classification of individuals - health conditions","display_name":"AIRBench 2024 - #39-40.55: Classification of individuals - Health conditions","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #39-40.56: classification of individuals - disability","display_name":"AIRBench 2024 - #39-40.56: Classification of individuals - Disability","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #39-40.57: classification of individuals - pregnancy status","display_name":"AIRBench 2024 - #39-40.57: Classification of individuals - Pregnancy Status","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #39-40.58: classification of individuals - genetic information","display_name":"AIRBench 2024 - #39-40.58: Classification of individuals - Genetic Information","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #39-40.59: classification of individuals - occupation","display_name":"AIRBench 2024 - #39-40.59: Classification of individuals - Occupation","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #39-40.5: discrimination in employment, benefits, or services - sexual orientation","display_name":"AIRBench 2024 - #39-40.5: Discrimination in employment, benefits, or services - Sexual orientation","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #39-40.60: classification of individuals - age","display_name":"AIRBench 2024 - #39-40.60: Classification of individuals - Age","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #39-40.6: discrimination in employment, benefits, or services - religion","display_name":"AIRBench 2024 - #39-40.6: Discrimination in employment, benefits, or services - Religion","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #39-40.7: discrimination in employment, benefits, or services - beliefs","display_name":"AIRBench 2024 - #39-40.7: Discrimination in employment, benefits, or services - Beliefs","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #39-40.8: discrimination in employment, benefits, or services - nationality","display_name":"AIRBench 2024 - #39-40.8: Discrimination in employment, benefits, or services - Nationality","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #39-40.9: discrimination in employment, benefits, or services - geographic region","display_name":"AIRBench 2024 - #39-40.9: Discrimination in employment, benefits, or services - Geographic region","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #4.10: profiling","display_name":"AIRBench 2024 - #4.10: Profiling","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #4.1: financing eligibility/creditworthiness","display_name":"AIRBench 2024 - #4.1: Financing eligibility/Creditworthiness","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #4.2: criminal justice/predictive policing","display_name":"AIRBench 2024 - #4.2: Criminal justice/Predictive policing","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #4.3: adversely affects legal rights","display_name":"AIRBench 2024 - #4.3: Adversely affects legal rights","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #4.4: employment","display_name":"AIRBench 2024 - #4.4: Employment","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #4.5: social scoring","display_name":"AIRBench 2024 - #4.5: Social scoring","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #4.6: housing eligibility","display_name":"AIRBench 2024 - #4.6: Housing eligibility","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #4.7: education eligibility","display_name":"AIRBench 2024 - #4.7: Education eligibility","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #4.8: migration eligibility","display_name":"AIRBench 2024 - #4.8: Migration eligibility","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #4.9: insurance eligibility","display_name":"AIRBench 2024 - #4.9: Insurance eligibility","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #41-42.10: unauthorized disclosure - pii (personal identifiable information)","display_name":"AIRBench 2024 - #41-42.10: Unauthorized disclosure - PII (Personal Identifiable Information)","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #41-42.11: unauthorized disclosure - health data","display_name":"AIRBench 2024 - #41-42.11: Unauthorized disclosure - Health data","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #41-42.12: unauthorized disclosure - location data","display_name":"AIRBench 2024 - #41-42.12: Unauthorized disclosure - Location data","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #41-42.13: unauthorized disclosure - demographic data","display_name":"AIRBench 2024 - #41-42.13: Unauthorized disclosure - Demographic data","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #41-42.14: unauthorized disclosure - biometric data (facial recognition)","display_name":"AIRBench 2024 - #41-42.14: Unauthorized disclosure - Biometric data (facial recognition)","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #41-42.15: unauthorized disclosure - educational records","display_name":"AIRBench 2024 - #41-42.15: Unauthorized disclosure - Educational records","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #41-42.16: unauthorized disclosure - financial records","display_name":"AIRBench 2024 - #41-42.16: Unauthorized disclosure - Financial records","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #41-42.17: unauthorized disclosure - behavioral/ preference data","display_name":"AIRBench 2024 - #41-42.17: Unauthorized disclosure - Behavioral/ Preference data","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #41-42.18: unauthorized disclosure - communication records","display_name":"AIRBench 2024 - #41-42.18: Unauthorized disclosure - Communication Records","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #41-42.19: unauthorized distribution - pii (personal identifiable information)","display_name":"AIRBench 2024 - #41-42.19: Unauthorized distribution - PII (Personal Identifiable Information)","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #41-42.1: unauthorized generation - pii (personal identifiable information)","display_name":"AIRBench 2024 - #41-42.1: Unauthorized generation - PII (Personal Identifiable Information)","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #41-42.20: unauthorized distribution - health data","display_name":"AIRBench 2024 - #41-42.20: Unauthorized distribution - Health data","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #41-42.21: unauthorized distribution - location data","display_name":"AIRBench 2024 - #41-42.21: Unauthorized distribution - Location data","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #41-42.22: unauthorized distribution - demographic data","display_name":"AIRBench 2024 - #41-42.22: Unauthorized distribution - Demographic data","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #41-42.23: unauthorized distribution - biometric data (facial recognition)","display_name":"AIRBench 2024 - #41-42.23: Unauthorized distribution - Biometric data (facial recognition)","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #41-42.24: unauthorized distribution - educational records","display_name":"AIRBench 2024 - #41-42.24: Unauthorized distribution - Educational records","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #41-42.25: unauthorized distribution - financial records","display_name":"AIRBench 2024 - #41-42.25: Unauthorized distribution - Financial records","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #41-42.26: unauthorized distribution - behavioral/ preference data","display_name":"AIRBench 2024 - #41-42.26: Unauthorized distribution - Behavioral/ Preference data","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #41-42.27: unauthorized distribution - communication records","display_name":"AIRBench 2024 - #41-42.27: Unauthorized distribution - Communication Records","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #41-42.28: unauthorized collection/gathering/stealing - pii (personal identifiable information)","display_name":"AIRBench 2024 - #41-42.28: Unauthorized collection/gathering/stealing - PII (Personal Identifiable Information)","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #41-42.29: unauthorized collection/gathering/stealing - health data","display_name":"AIRBench 2024 - #41-42.29: Unauthorized collection/gathering/stealing - Health data","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #41-42.2: unauthorized generation - health data","display_name":"AIRBench 2024 - #41-42.2: Unauthorized generation - Health data","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #41-42.30: unauthorized collection/gathering/stealing - location data","display_name":"AIRBench 2024 - #41-42.30: Unauthorized collection/gathering/stealing - Location data","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #41-42.31: unauthorized collection/gathering/stealing - demographic data","display_name":"AIRBench 2024 - #41-42.31: Unauthorized collection/gathering/stealing - Demographic data","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #41-42.32: unauthorized collection/gathering/stealing - biometric data (facial recognition)","display_name":"AIRBench 2024 - #41-42.32: Unauthorized collection/gathering/stealing - Biometric data (facial recognition)","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #41-42.33: unauthorized collection/gathering/stealing - educational records","display_name":"AIRBench 2024 - #41-42.33: Unauthorized collection/gathering/stealing - Educational records","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #41-42.34: unauthorized collection/gathering/stealing - financial records","display_name":"AIRBench 2024 - #41-42.34: Unauthorized collection/gathering/stealing - Financial records","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #41-42.35: unauthorized collection/gathering/stealing - behavioral/ preference data","display_name":"AIRBench 2024 - #41-42.35: Unauthorized collection/gathering/stealing - Behavioral/ Preference data","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #41-42.36: unauthorized collection/gathering/stealing - communication records","display_name":"AIRBench 2024 - #41-42.36: Unauthorized collection/gathering/stealing - Communication Records","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #41-42.37: unauthorized processing - pii (personal identifiable information)","display_name":"AIRBench 2024 - #41-42.37: Unauthorized processing - PII (Personal Identifiable Information)","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #41-42.38: unauthorized processing - health data","display_name":"AIRBench 2024 - #41-42.38: Unauthorized processing - Health data","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #41-42.39: unauthorized processing - location data","display_name":"AIRBench 2024 - #41-42.39: Unauthorized processing - Location data","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #41-42.3: unauthorized generation - location data","display_name":"AIRBench 2024 - #41-42.3: Unauthorized generation - Location data","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #41-42.40: unauthorized processing - demographic data","display_name":"AIRBench 2024 - #41-42.40: Unauthorized processing - Demographic data","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #41-42.41: unauthorized processing - biometric data (facial recognition)","display_name":"AIRBench 2024 - #41-42.41: Unauthorized processing - Biometric data (facial recognition)","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #41-42.42: unauthorized processing - educational records","display_name":"AIRBench 2024 - #41-42.42: Unauthorized processing - Educational records","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #41-42.43: unauthorized processing - financial records","display_name":"AIRBench 2024 - #41-42.43: Unauthorized processing - Financial records","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #41-42.44: unauthorized processing - behavioral/ preference data","display_name":"AIRBench 2024 - #41-42.44: Unauthorized processing - Behavioral/ Preference data","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #41-42.45: unauthorized processing - communication records","display_name":"AIRBench 2024 - #41-42.45: Unauthorized processing - Communication Records","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #41-42.46: unauthorized inference/synthesis - pii (personal identifiable information)","display_name":"AIRBench 2024 - #41-42.46: Unauthorized inference/synthesis - PII (Personal Identifiable Information)","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #41-42.47: unauthorized inference/synthesis - health data","display_name":"AIRBench 2024 - #41-42.47: Unauthorized inference/synthesis - Health data","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #41-42.48: unauthorized inference/synthesis - location data","display_name":"AIRBench 2024 - #41-42.48: Unauthorized inference/synthesis - Location data","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #41-42.49: unauthorized inference/synthesis - demographic data","display_name":"AIRBench 2024 - #41-42.49: Unauthorized inference/synthesis - Demographic data","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #41-42.4: unauthorized generation - demographic data","display_name":"AIRBench 2024 - #41-42.4: Unauthorized generation - Demographic data","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #41-42.50: unauthorized inference/synthesis - biometric data (facial recognition)","display_name":"AIRBench 2024 - #41-42.50: Unauthorized inference/synthesis - Biometric data (facial recognition)","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #41-42.51: unauthorized inference/synthesis - educational records","display_name":"AIRBench 2024 - #41-42.51: Unauthorized inference/synthesis - Educational records","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #41-42.52: unauthorized inference/synthesis - financial records","display_name":"AIRBench 2024 - #41-42.52: Unauthorized inference/synthesis - Financial records","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #41-42.53: unauthorized inference/synthesis - behavioral/ preference data","display_name":"AIRBench 2024 - #41-42.53: Unauthorized inference/synthesis - Behavioral/ Preference data","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #41-42.54: unauthorized inference/synthesis - communication records","display_name":"AIRBench 2024 - #41-42.54: Unauthorized inference/synthesis - Communication Records","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #41-42.55: non-consensual tracking/monitoring/stalking/spyware - pii (personal identifiable information)","display_name":"AIRBench 2024 - #41-42.55: Non-consensual tracking/monitoring/stalking/spyware - PII (Personal Identifiable Information)","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #41-42.56: non-consensual tracking/monitoring/stalking/spyware - health data","display_name":"AIRBench 2024 - #41-42.56: Non-consensual tracking/monitoring/stalking/spyware - Health data","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #41-42.57: non-consensual tracking/monitoring/stalking/spyware - location data","display_name":"AIRBench 2024 - #41-42.57: Non-consensual tracking/monitoring/stalking/spyware - Location data","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #41-42.58: non-consensual tracking/monitoring/stalking/spyware - demographic data","display_name":"AIRBench 2024 - #41-42.58: Non-consensual tracking/monitoring/stalking/spyware - Demographic data","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #41-42.59: non-consensual tracking/monitoring/stalking/spyware - biometric data (facial recognition)","display_name":"AIRBench 2024 - #41-42.59: Non-consensual tracking/monitoring/stalking/spyware - Biometric data (facial recognition)","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #41-42.5: unauthorized generation - biometric data (facial recognition)","display_name":"AIRBench 2024 - #41-42.5: Unauthorized generation - Biometric data (facial recognition)","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #41-42.60: non-consensual tracking/monitoring/stalking/spyware - educational records","display_name":"AIRBench 2024 - #41-42.60: Non-consensual tracking/monitoring/stalking/spyware - Educational records","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #41-42.61: non-consensual tracking/monitoring/stalking/spyware - financial records","display_name":"AIRBench 2024 - #41-42.61: Non-consensual tracking/monitoring/stalking/spyware - Financial records","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #41-42.62: non-consensual tracking/monitoring/stalking/spyware - behavioral/ preference data","display_name":"AIRBench 2024 - #41-42.62: Non-consensual tracking/monitoring/stalking/spyware - Behavioral/ Preference data","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #41-42.63: non-consensual tracking/monitoring/stalking/spyware - communication records","display_name":"AIRBench 2024 - #41-42.63: Non-consensual tracking/monitoring/stalking/spyware - Communication Records","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #41-42.64: model attacks (membership inference, model inversion) - pii (personal identifiable information)","display_name":"AIRBench 2024 - #41-42.64: Model Attacks (membership inference, model inversion) - PII (Personal Identifiable Information)","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #41-42.65: model attacks (membership inference, model inversion) - health data","display_name":"AIRBench 2024 - #41-42.65: Model Attacks (membership inference, model inversion) - Health data","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #41-42.66: model attacks (membership inference, model inversion) - location data","display_name":"AIRBench 2024 - #41-42.66: Model Attacks (membership inference, model inversion) - Location data","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #41-42.67: model attacks (membership inference, model inversion) - demographic data","display_name":"AIRBench 2024 - #41-42.67: Model Attacks (membership inference, model inversion) - Demographic data","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #41-42.68: model attacks (membership inference, model inversion) - biometric data (facial recognition)","display_name":"AIRBench 2024 - #41-42.68: Model Attacks (membership inference, model inversion) - Biometric data (facial recognition)","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #41-42.69: model attacks (membership inference, model inversion) - educational records","display_name":"AIRBench 2024 - #41-42.69: Model Attacks (membership inference, model inversion) - Educational records","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #41-42.6: unauthorized generation - educational records","display_name":"AIRBench 2024 - #41-42.6: Unauthorized generation - Educational records","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #41-42.70: model attacks (membership inference, model inversion) - financial records","display_name":"AIRBench 2024 - #41-42.70: Model Attacks (membership inference, model inversion) - Financial records","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #41-42.71: model attacks (membership inference, model inversion) - behavioral/ preference data","display_name":"AIRBench 2024 - #41-42.71: Model Attacks (membership inference, model inversion) - Behavioral/ Preference data","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #41-42.72: model attacks (membership inference, model inversion) - communication records","display_name":"AIRBench 2024 - #41-42.72: Model Attacks (membership inference, model inversion) - Communication Records","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #41-42.7: unauthorized generation - financial records","display_name":"AIRBench 2024 - #41-42.7: Unauthorized generation - Financial records","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #41-42.8: unauthorized generation - behavioral/ preference data","display_name":"AIRBench 2024 - #41-42.8: Unauthorized generation - Behavioral/ Preference data","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #41-42.9: unauthorized generation - communication records","display_name":"AIRBench 2024 - #41-42.9: Unauthorized generation - Communication Records","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #43.1: illegal drugs","display_name":"AIRBench 2024 - #43.1: Illegal drugs","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #44.2: human trafficking","display_name":"AIRBench 2024 - #44.2: Human trafficking","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #44.3: sexual exploitation","display_name":"AIRBench 2024 - #44.3: Sexual exploitation","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #44.4: prostitution","display_name":"AIRBench 2024 - #44.4: Prostitution","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #45.5: undermining national security or other government interests","display_name":"AIRBench 2024 - #45.5: Undermining national security or other government interests","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #45.6: undermining social stability","display_name":"AIRBench 2024 - #45.6: Undermining social stability","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #45.7: undermining international relations","display_name":"AIRBench 2024 - #45.7: Undermining international relations","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #45.8: abetting/furthering activities violating any applicable law","display_name":"AIRBench 2024 - #45.8: Abetting/Furthering activities violating any applicable law","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #5.11: heavy machinery","display_name":"AIRBench 2024 - #5.11: Heavy machinery","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #5.12: transportation","display_name":"AIRBench 2024 - #5.12: Transportation","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #5.13: energy/electrical grids","display_name":"AIRBench 2024 - #5.13: Energy/Electrical grids","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #5.14: nuclear facilities","display_name":"AIRBench 2024 - #5.14: Nuclear facilities","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #5.15: aircraft navigation/air traffic control","display_name":"AIRBench 2024 - #5.15: Aircraft navigation/Air traffic control","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #5.16: communication systems","display_name":"AIRBench 2024 - #5.16: Communication systems","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #5.17: water treatment facilities","display_name":"AIRBench 2024 - #5.17: Water treatment facilities","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #5.18: life support","display_name":"AIRBench 2024 - #5.18: Life support","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #5.19: weapon systems/battlefield management","display_name":"AIRBench 2024 - #5.19: Weapon systems/Battlefield management","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #5.20: emergency services","display_name":"AIRBench 2024 - #5.20: Emergency services","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #5.21: other unauthorized actions on behalf of users","display_name":"AIRBench 2024 - #5.21: Other unauthorized actions on behalf of users","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #6.22: legal","display_name":"AIRBench 2024 - #6.22: Legal","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #6.23: medical/pharmaceutical","display_name":"AIRBench 2024 - #6.23: Medical/Pharmaceutical","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #6.24: accounting","display_name":"AIRBench 2024 - #6.24: Accounting","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #6.25: financial","display_name":"AIRBench 2024 - #6.25: Financial","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #6.26: government services","display_name":"AIRBench 2024 - #6.26: Government services","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #7.1: extremism","display_name":"AIRBench 2024 - #7.1: Extremism","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #7.2: terrorism","display_name":"AIRBench 2024 - #7.2: Terrorism","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #7.3: criminal organization","display_name":"AIRBench 2024 - #7.3: Criminal organization","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #8.4: glorifying violence, abuse, or the suffering of others","display_name":"AIRBench 2024 - #8.4: Glorifying violence, abuse, or the suffering of others","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #8.5: belittling victimhood or violent events","display_name":"AIRBench 2024 - #8.5: Belittling victimhood or violent events","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #8.6: denying or minimizing well-documented, major violent events or the victimhood of such events/denying the deeds of martyrdom","display_name":"AIRBench 2024 - #8.6: Denying or minimizing well-documented, major violent events or the victimhood of such events/Denying the deeds of martyrdom","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #8.7: beautifying and whitewashing acts of war or aggression","display_name":"AIRBench 2024 - #8.7: Beautifying and whitewashing acts of war or aggression","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #9.10: property damage","display_name":"AIRBench 2024 - #9.10: Property damage","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #9.11: environmental","display_name":"AIRBench 2024 - #9.11: Environmental","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #9.8: persons (including murder)","display_name":"AIRBench 2024 - #9.8: Persons (including murder)","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]},{"key":"airbench 2024 - #9.9: animals","display_name":"AIRBench 2024 - #9.9: Animals","is_bare_stem":false,"metrics":[{"key":"Refusal Rate","display_name":"Refusal Rate","sources":["crfm"]}],"derivedTags":["safety"]}],"summary_eval_ids":["helm-air-bench%2Fair-bench-2024","agentharm%2Fair-bench-2024-13-harassment","agentharm%2Fair-bench-2024-32-fraud","helm-air-bench%2Fair-bench-2024-1-confidentiality","helm-air-bench%2Fair-bench-2024-10-depicting-violence","helm-air-bench%2Fair-bench-2024-11-weapon-usage-development","helm-air-bench%2Fair-bench-2024-12-military-and-warfare","helm-air-bench%2Fair-bench-2024-13-harassment","helm-air-bench%2Fair-bench-2024-14-hate-speech-inciting-promoting-expressing-hatred","helm-air-bench%2Fair-bench-2024-15-perpetuating-harmful-beliefs","helm-air-bench%2Fair-bench-2024-16-offensive-language","helm-air-bench%2Fair-bench-2024-17-adult-content","helm-air-bench%2Fair-bench-2024-18-erotic","helm-air-bench%2Fair-bench-2024-19-non-consensual-nudity","helm-air-bench%2Fair-bench-2024-2-integrity","helm-air-bench%2Fair-bench-2024-20-monetized","helm-air-bench%2Fair-bench-2024-21-endangerment-harm-or-abuse-of-children","helm-air-bench%2Fair-bench-2024-22-child-sexual-abuse","helm-air-bench%2Fair-bench-2024-23-suicidal-and-non-suicidal-self-injury","helm-air-bench%2Fair-bench-2024-24-political-persuasion","helm-air-bench%2Fair-bench-2024-25-influencing-politics","helm-air-bench%2Fair-bench-2024-26-deterring-democratic-participation","helm-air-bench%2Fair-bench-2024-27-disrupting-social-order-china-unique","helm-air-bench%2Fair-bench-2024-28-high-risk-financial-activities","helm-air-bench%2Fair-bench-2024-29-unfair-market-practices","helm-air-bench%2Fair-bench-2024-3-availability","helm-air-bench%2Fair-bench-2024-30-disempowering-workers","helm-air-bench%2Fair-bench-2024-31-fraudulent-schemes","helm-air-bench%2Fair-bench-2024-32-fraud","helm-air-bench%2Fair-bench-2024-33-academic-dishonesty","helm-air-bench%2Fair-bench-2024-34-mis-disinformation","helm-air-bench%2Fair-bench-2024-35-sowing-division","helm-air-bench%2Fair-bench-2024-36-misrepresentation","helm-air-bench%2Fair-bench-2024-37-types-of-defamation","helm-air-bench%2Fair-bench-2024-38-specific-types-of-rights","helm-air-bench%2Fair-bench-2024-39-40-discrimination-protected-characteristics-combinations","helm-air-bench%2Fair-bench-2024-4-automated-decision-making","helm-air-bench%2Fair-bench-2024-41-42-privacy-violations-sensitive-data-combinations","helm-air-bench%2Fair-bench-2024-43-illegal-regulated-substances-goods","helm-air-bench%2Fair-bench-2024-44-illegal-services-exploitation","helm-air-bench%2Fair-bench-2024-45-other-illegal-unlawful-criminal-activities","helm-air-bench%2Fair-bench-2024-5-autonomous-unsafe-operation-of-systems","helm-air-bench%2Fair-bench-2024-6-advice-in-heavily-regulated-industries","helm-air-bench%2Fair-bench-2024-7-supporting-malicious-organized-groups","helm-air-bench%2Fair-bench-2024-8-celebrating-suffering","helm-air-bench%2Fair-bench-2024-9-violent-acts","helm-air-bench%2Fair-bench-2024-child-harm","helm-air-bench%2Fair-bench-2024-criminal-activities","helm-air-bench%2Fair-bench-2024-deception","helm-air-bench%2Fair-bench-2024-defamation","helm-air-bench%2Fair-bench-2024-discrimination-bias","helm-air-bench%2Fair-bench-2024-economic-harm","helm-air-bench%2Fair-bench-2024-fundamental-rights","helm-air-bench%2Fair-bench-2024-hate-toxicity","helm-air-bench%2Fair-bench-2024-manipulation","helm-air-bench%2Fair-bench-2024-operational-misuses","helm-air-bench%2Fair-bench-2024-political-usage","helm-air-bench%2Fair-bench-2024-privacy","helm-air-bench%2Fair-bench-2024-security-risks","helm-air-bench%2Fair-bench-2024-self-harm","helm-air-bench%2Fair-bench-2024-sexual-content","helm-air-bench%2Fair-bench-2024-violence-extremism","helm-air-bench%2Fairbench 2024","helm-air-bench%2Fairbench 2024 - #1.1: network intrusion","helm-air-bench%2Fairbench 2024 - #1.2: vulnerability probing","helm-air-bench%2Fairbench 2024 - #1.3: spoofing","helm-air-bench%2Fairbench 2024 - #1.4: spear phishing","helm-air-bench%2Fairbench 2024 - #1.5: social engineering","helm-air-bench%2Fairbench 2024 - #1.6: unauthorized network entry","helm-air-bench%2Fairbench 2024 - #10.12: bodily destruction","helm-air-bench%2Fairbench 2024 - #10.13: bodily mutilation","helm-air-bench%2Fairbench 2024 - #10.14: torture/abuse","helm-air-bench%2Fairbench 2024 - #10.15: animal abuse","helm-air-bench%2Fairbench 2024 - #10.16: activities meant to kill","helm-air-bench%2Fairbench 2024 - #11.17: guns","helm-air-bench%2Fairbench 2024 - #11.18: explosives/dangerous materials","helm-air-bench%2Fairbench 2024 - #11.19: bioweapons/viruses/gain-of-function","helm-air-bench%2Fairbench 2024 - #11.20: nuclear weapons","helm-air-bench%2Fairbench 2024 - #11.21: chemical weapons","helm-air-bench%2Fairbench 2024 - #11.22: radiological weapons","helm-air-bench%2Fairbench 2024 - #12.23: military","helm-air-bench%2Fairbench 2024 - #12.24: warfare","helm-air-bench%2Fairbench 2024 - #13.10: doxxing","helm-air-bench%2Fairbench 2024 - #13.11: cursing","helm-air-bench%2Fairbench 2024 - #13.1: bullying","helm-air-bench%2Fairbench 2024 - #13.2: threats","helm-air-bench%2Fairbench 2024 - #13.3: intimidation","helm-air-bench%2Fairbench 2024 - #13.4: shaming","helm-air-bench%2Fairbench 2024 - #13.5: humiliation","helm-air-bench%2Fairbench 2024 - #13.6: insults/personal attacks","helm-air-bench%2Fairbench 2024 - #13.7: abuse","helm-air-bench%2Fairbench 2024 - #13.8: provoking","helm-air-bench%2Fairbench 2024 - #13.9: trolling","helm-air-bench%2Fairbench 2024 - #14.12: race","helm-air-bench%2Fairbench 2024 - #14.13: ethnicity","helm-air-bench%2Fairbench 2024 - #14.14: color","helm-air-bench%2Fairbench 2024 - #14.15: gender","helm-air-bench%2Fairbench 2024 - #14.16: sexual orientation","helm-air-bench%2Fairbench 2024 - #14.17: religion","helm-air-bench%2Fairbench 2024 - #14.18: beliefs","helm-air-bench%2Fairbench 2024 - #14.19: nationality","helm-air-bench%2Fairbench 2024 - #14.20: geographic region","helm-air-bench%2Fairbench 2024 - #14.21: caste","helm-air-bench%2Fairbench 2024 - #14.22: social behaviors","helm-air-bench%2Fairbench 2024 - #14.23: physical characteristics","helm-air-bench%2Fairbench 2024 - #14.24: mental characteristics","helm-air-bench%2Fairbench 2024 - #14.25: personality","helm-air-bench%2Fairbench 2024 - #14.26: health conditions","helm-air-bench%2Fairbench 2024 - #14.27: disability","helm-air-bench%2Fairbench 2024 - #14.28: pregnancy status","helm-air-bench%2Fairbench 2024 - #14.29: genetic information","helm-air-bench%2Fairbench 2024 - #14.30: occupation","helm-air-bench%2Fairbench 2024 - #14.31: age","helm-air-bench%2Fairbench 2024 - #15.32: negative stereotyping of any group","helm-air-bench%2Fairbench 2024 - #15.33: perpetuating racism","helm-air-bench%2Fairbench 2024 - #15.34: perpetuating sexism","helm-air-bench%2Fairbench 2024 - #16.35: vulgarity","helm-air-bench%2Fairbench 2024 - #16.36: derogatory comments","helm-air-bench%2Fairbench 2024 - #17.1: obscenity","helm-air-bench%2Fairbench 2024 - #17.2: suggestive","helm-air-bench%2Fairbench 2024 - #17.3: sexual acts","helm-air-bench%2Fairbench 2024 - #17.4: sexual intercourse","helm-air-bench%2Fairbench 2024 - #18.5: erotic chats","helm-air-bench%2Fairbench 2024 - #18.6: fetishes","helm-air-bench%2Fairbench 2024 - #19.7: ncii (non-consensual intimate image)","helm-air-bench%2Fairbench 2024 - #2.10: control override (safety/privacy filters)","helm-air-bench%2Fairbench 2024 - #2.7: malware","helm-air-bench%2Fairbench 2024 - #2.8: packet forgery","helm-air-bench%2Fairbench 2024 - #2.9: data tampering","helm-air-bench%2Fairbench 2024 - #20.8: pornography","helm-air-bench%2Fairbench 2024 - #20.9: promotion of sexual services","helm-air-bench%2Fairbench 2024 - #21.1: grooming","helm-air-bench%2Fairbench 2024 - #21.2: pedophilia","helm-air-bench%2Fairbench 2024 - #21.3: exploiting/harming minors","helm-air-bench%2Fairbench 2024 - #21.4: building services targeting minors/failure to employ age-gating","helm-air-bench%2Fairbench 2024 - #21.5: building services to present a persona of minor","helm-air-bench%2Fairbench 2024 - #22.6: solicitation","helm-air-bench%2Fairbench 2024 - #22.7: csam","helm-air-bench%2Fairbench 2024 - #23.1: suicide","helm-air-bench%2Fairbench 2024 - #23.2: cutting","helm-air-bench%2Fairbench 2024 - #23.3: eating disorders (anorexia/bulimia)","helm-air-bench%2Fairbench 2024 - #24.1: lobbying","helm-air-bench%2Fairbench 2024 - #24.2: generating high-volume campaign materials","helm-air-bench%2Fairbench 2024 - #24.3: personalized or targeted campaign materials","helm-air-bench%2Fairbench 2024 - #24.4: building systems for political campaigning or lobbying","helm-air-bench%2Fairbench 2024 - #24.5: building products for political campaigning or lobbying","helm-air-bench%2Fairbench 2024 - #24.6: political advertisements","helm-air-bench%2Fairbench 2024 - #24.7: propaganda","helm-air-bench%2Fairbench 2024 - #25.8: influence political decisions","helm-air-bench%2Fairbench 2024 - #25.9: influence political opinions","helm-air-bench%2Fairbench 2024 - #26.10: deterring participation in democratic processes","helm-air-bench%2Fairbench 2024 - #26.11: misrepresenting voting processes","helm-air-bench%2Fairbench 2024 - #26.12: misrepresenting voting qualifications","helm-air-bench%2Fairbench 2024 - #26.13: discouraging voting","helm-air-bench%2Fairbench 2024 - #27.14: opposing constitutional principles","helm-air-bench%2Fairbench 2024 - #27.15: subverting state power","helm-air-bench%2Fairbench 2024 - #27.16: undermining national unity","helm-air-bench%2Fairbench 2024 - #27.17: damaging state interests","helm-air-bench%2Fairbench 2024 - #27.18: damaging the state’s honor","helm-air-bench%2Fairbench 2024 - #27.19: inciting unlawful assemblies","helm-air-bench%2Fairbench 2024 - #27.20: inciting unlawful associations","helm-air-bench%2Fairbench 2024 - #27.21: inciting unlawful processions","helm-air-bench%2Fairbench 2024 - #27.22: inciting unlawful demonstrations","helm-air-bench%2Fairbench 2024 - #27.23: undermining religious policies","helm-air-bench%2Fairbench 2024 - #27.24: promoting cults","helm-air-bench%2Fairbench 2024 - #27.25: promoting feudal superstitions","helm-air-bench%2Fairbench 2024 - #28.1: gambling (e.g., sports betting)","helm-air-bench%2Fairbench 2024 - #28.2: payday lending","helm-air-bench%2Fairbench 2024 - #29.3: exploiting advantages for monopolistic practices","helm-air-bench%2Fairbench 2024 - #29.4: anticompetitive practices","helm-air-bench%2Fairbench 2024 - #3.11: system/website impairment","helm-air-bench%2Fairbench 2024 - #3.12: network disruption","helm-air-bench%2Fairbench 2024 - #30.5: undermine workers' rights","helm-air-bench%2Fairbench 2024 - #30.6: worsen job quality","helm-air-bench%2Fairbench 2024 - #30.7: encourage undue worker surveillance","helm-air-bench%2Fairbench 2024 - #30.8: cause harmful labor-force disruptions","helm-air-bench%2Fairbench 2024 - #31.10: pyramid schemes","helm-air-bench%2Fairbench 2024 - #31.9: multi-level marketing","helm-air-bench%2Fairbench 2024 - #32.1: spam","helm-air-bench%2Fairbench 2024 - #32.2: scams","helm-air-bench%2Fairbench 2024 - #32.3: phishing/catfishing","helm-air-bench%2Fairbench 2024 - #32.4: pseudo-pharmaceuticals","helm-air-bench%2Fairbench 2024 - #32.5: impersonating others","helm-air-bench%2Fairbench 2024 - #33.6: plagiarism","helm-air-bench%2Fairbench 2024 - #33.7: promoting academic dishonesty","helm-air-bench%2Fairbench 2024 - #34.8: generating or promoting misinformation","helm-air-bench%2Fairbench 2024 - #34.9: fake online engagement (fake reviews, fake grassroots support)","helm-air-bench%2Fairbench 2024 - #35.1: inducing internal conflict","helm-air-bench%2Fairbench 2024 - #35.2: deflecting scrutiny from harmful actions","helm-air-bench%2Fairbench 2024 - #36.3: automated social media posts","helm-air-bench%2Fairbench 2024 - #36.4: not labeling content as ai-generated (using chatbots to convince people they are communicating with a human)","helm-air-bench%2Fairbench 2024 - #36.5: impersonating humans","helm-air-bench%2Fairbench 2024 - #37.1: disparagement","helm-air-bench%2Fairbench 2024 - #37.2: libel","helm-air-bench%2Fairbench 2024 - #37.3: slander","helm-air-bench%2Fairbench 2024 - #38.1: ip rights/ trade secrets","helm-air-bench%2Fairbench 2024 - #38.2: likeness rights","helm-air-bench%2Fairbench 2024 - #38.3: reputation rights","helm-air-bench%2Fairbench 2024 - #38.4: honor","helm-air-bench%2Fairbench 2024 - #38.5: name rights","helm-air-bench%2Fairbench 2024 - #39-40.10: discrimination in employment, benefits, or services - caste","helm-air-bench%2Fairbench 2024 - #39-40.11: discrimination in employment, benefits, or services - social behaviors","helm-air-bench%2Fairbench 2024 - #39-40.12: discrimination in employment, benefits, or services - physical characteristics","helm-air-bench%2Fairbench 2024 - #39-40.13: discrimination in employment, benefits, or services - mental characteristics","helm-air-bench%2Fairbench 2024 - #39-40.14: discrimination in employment, benefits, or services - predicted personality","helm-air-bench%2Fairbench 2024 - #39-40.15: discrimination in employment, benefits, or services - health conditions","helm-air-bench%2Fairbench 2024 - #39-40.16: discrimination in employment, benefits, or services - disability","helm-air-bench%2Fairbench 2024 - #39-40.17: discrimination in employment, benefits, or services - pregnancy status","helm-air-bench%2Fairbench 2024 - #39-40.18: discrimination in employment, benefits, or services - genetic information","helm-air-bench%2Fairbench 2024 - #39-40.19: discrimination in employment, benefits, or services - occupation","helm-air-bench%2Fairbench 2024 - #39-40.1: discrimination in employment, benefits, or services - race","helm-air-bench%2Fairbench 2024 - #39-40.20: discrimination in employment, benefits, or services - age","helm-air-bench%2Fairbench 2024 - #39-40.21: characterization of identity - race","helm-air-bench%2Fairbench 2024 - #39-40.22: characterization of identity - ethnicity","helm-air-bench%2Fairbench 2024 - #39-40.23: characterization of identity - color","helm-air-bench%2Fairbench 2024 - #39-40.24: characterization of identity - gender","helm-air-bench%2Fairbench 2024 - #39-40.25: characterization of identity - sexual orientation","helm-air-bench%2Fairbench 2024 - #39-40.26: characterization of identity - religion","helm-air-bench%2Fairbench 2024 - #39-40.27: characterization of identity - beliefs","helm-air-bench%2Fairbench 2024 - #39-40.28: characterization of identity - nationality","helm-air-bench%2Fairbench 2024 - #39-40.29: characterization of identity - geographic region","helm-air-bench%2Fairbench 2024 - #39-40.2: discrimination in employment, benefits, or services - ethnicity","helm-air-bench%2Fairbench 2024 - #39-40.30: characterization of identity - caste","helm-air-bench%2Fairbench 2024 - #39-40.31: characterization of identity - social behaviors","helm-air-bench%2Fairbench 2024 - #39-40.32: characterization of identity - physical characteristics","helm-air-bench%2Fairbench 2024 - #39-40.33: characterization of identity - mental characteristics","helm-air-bench%2Fairbench 2024 - #39-40.34: characterization of identity - predicted personality","helm-air-bench%2Fairbench 2024 - #39-40.35: characterization of identity - health conditions","helm-air-bench%2Fairbench 2024 - #39-40.36: characterization of identity - disability","helm-air-bench%2Fairbench 2024 - #39-40.37: characterization of identity - pregnancy status","helm-air-bench%2Fairbench 2024 - #39-40.38: characterization of identity - genetic information","helm-air-bench%2Fairbench 2024 - #39-40.39: characterization of identity - occupation","helm-air-bench%2Fairbench 2024 - #39-40.3: discrimination in employment, benefits, or services - color","helm-air-bench%2Fairbench 2024 - #39-40.40: characterization of identity - age","helm-air-bench%2Fairbench 2024 - #39-40.41: classification of individuals - race","helm-air-bench%2Fairbench 2024 - #39-40.42: classification of individuals - ethnicity","helm-air-bench%2Fairbench 2024 - #39-40.43: classification of individuals - color","helm-air-bench%2Fairbench 2024 - #39-40.44: classification of individuals - gender","helm-air-bench%2Fairbench 2024 - #39-40.45: classification of individuals - sexual orientation","helm-air-bench%2Fairbench 2024 - #39-40.46: classification of individuals - religion","helm-air-bench%2Fairbench 2024 - #39-40.47: classification of individuals - beliefs","helm-air-bench%2Fairbench 2024 - #39-40.48: classification of individuals - nationality","helm-air-bench%2Fairbench 2024 - #39-40.49: classification of individuals - geographic region","helm-air-bench%2Fairbench 2024 - #39-40.4: discrimination in employment, benefits, or services - gender","helm-air-bench%2Fairbench 2024 - #39-40.50: classification of individuals - caste","helm-air-bench%2Fairbench 2024 - #39-40.51: classification of individuals - social behaviors","helm-air-bench%2Fairbench 2024 - #39-40.52: classification of individuals - physical characteristics","helm-air-bench%2Fairbench 2024 - #39-40.53: classification of individuals - mental characteristics","helm-air-bench%2Fairbench 2024 - #39-40.54: classification of individuals - predicted personality","helm-air-bench%2Fairbench 2024 - #39-40.55: classification of individuals - health conditions","helm-air-bench%2Fairbench 2024 - #39-40.56: classification of individuals - disability","helm-air-bench%2Fairbench 2024 - #39-40.57: classification of individuals - pregnancy status","helm-air-bench%2Fairbench 2024 - #39-40.58: classification of individuals - genetic information","helm-air-bench%2Fairbench 2024 - #39-40.59: classification of individuals - occupation","helm-air-bench%2Fairbench 2024 - #39-40.5: discrimination in employment, benefits, or services - sexual orientation","helm-air-bench%2Fairbench 2024 - #39-40.60: classification of individuals - age","helm-air-bench%2Fairbench 2024 - #39-40.6: discrimination in employment, benefits, or services - religion","helm-air-bench%2Fairbench 2024 - #39-40.7: discrimination in employment, benefits, or services - beliefs","helm-air-bench%2Fairbench 2024 - #39-40.8: discrimination in employment, benefits, or services - nationality","helm-air-bench%2Fairbench 2024 - #39-40.9: discrimination in employment, benefits, or services - geographic region","helm-air-bench%2Fairbench 2024 - #4.10: profiling","helm-air-bench%2Fairbench 2024 - #4.1: financing eligibility/creditworthiness","helm-air-bench%2Fairbench 2024 - #4.2: criminal justice/predictive policing","helm-air-bench%2Fairbench 2024 - #4.3: adversely affects legal rights","helm-air-bench%2Fairbench 2024 - #4.4: employment","helm-air-bench%2Fairbench 2024 - #4.5: social scoring","helm-air-bench%2Fairbench 2024 - #4.6: housing eligibility","helm-air-bench%2Fairbench 2024 - #4.7: education eligibility","helm-air-bench%2Fairbench 2024 - #4.8: migration eligibility","helm-air-bench%2Fairbench 2024 - #4.9: insurance eligibility","helm-air-bench%2Fairbench 2024 - #41-42.10: unauthorized disclosure - pii (personal identifiable information)","helm-air-bench%2Fairbench 2024 - #41-42.11: unauthorized disclosure - health data","helm-air-bench%2Fairbench 2024 - #41-42.12: unauthorized disclosure - location data","helm-air-bench%2Fairbench 2024 - #41-42.13: unauthorized disclosure - demographic data","helm-air-bench%2Fairbench 2024 - #41-42.14: unauthorized disclosure - biometric data (facial recognition)","helm-air-bench%2Fairbench 2024 - #41-42.15: unauthorized disclosure - educational records","helm-air-bench%2Fairbench 2024 - #41-42.16: unauthorized disclosure - financial records","helm-air-bench%2Fairbench 2024 - #41-42.17: unauthorized disclosure - behavioral/ preference data","helm-air-bench%2Fairbench 2024 - #41-42.18: unauthorized disclosure - communication records","helm-air-bench%2Fairbench 2024 - #41-42.19: unauthorized distribution - pii (personal identifiable information)","helm-air-bench%2Fairbench 2024 - #41-42.1: unauthorized generation - pii (personal identifiable information)","helm-air-bench%2Fairbench 2024 - #41-42.20: unauthorized distribution - health data","helm-air-bench%2Fairbench 2024 - #41-42.21: unauthorized distribution - location data","helm-air-bench%2Fairbench 2024 - #41-42.22: unauthorized distribution - demographic data","helm-air-bench%2Fairbench 2024 - #41-42.23: unauthorized distribution - biometric data (facial recognition)","helm-air-bench%2Fairbench 2024 - #41-42.24: unauthorized distribution - educational records","helm-air-bench%2Fairbench 2024 - #41-42.25: unauthorized distribution - financial records","helm-air-bench%2Fairbench 2024 - #41-42.26: unauthorized distribution - behavioral/ preference data","helm-air-bench%2Fairbench 2024 - #41-42.27: unauthorized distribution - communication records","helm-air-bench%2Fairbench 2024 - #41-42.28: unauthorized collection/gathering/stealing - pii (personal identifiable information)","helm-air-bench%2Fairbench 2024 - #41-42.29: unauthorized collection/gathering/stealing - health data","helm-air-bench%2Fairbench 2024 - #41-42.2: unauthorized generation - health data","helm-air-bench%2Fairbench 2024 - #41-42.30: unauthorized collection/gathering/stealing - location data","helm-air-bench%2Fairbench 2024 - #41-42.31: unauthorized collection/gathering/stealing - demographic data","helm-air-bench%2Fairbench 2024 - #41-42.32: unauthorized collection/gathering/stealing - biometric data (facial recognition)","helm-air-bench%2Fairbench 2024 - #41-42.33: unauthorized collection/gathering/stealing - educational records","helm-air-bench%2Fairbench 2024 - #41-42.34: unauthorized collection/gathering/stealing - financial records","helm-air-bench%2Fairbench 2024 - #41-42.35: unauthorized collection/gathering/stealing - behavioral/ preference data","helm-air-bench%2Fairbench 2024 - #41-42.36: unauthorized collection/gathering/stealing - communication records","helm-air-bench%2Fairbench 2024 - #41-42.37: unauthorized processing - pii (personal identifiable information)","helm-air-bench%2Fairbench 2024 - #41-42.38: unauthorized processing - health data","helm-air-bench%2Fairbench 2024 - #41-42.39: unauthorized processing - location data","helm-air-bench%2Fairbench 2024 - #41-42.3: unauthorized generation - location data","helm-air-bench%2Fairbench 2024 - #41-42.40: unauthorized processing - demographic data","helm-air-bench%2Fairbench 2024 - #41-42.41: unauthorized processing - biometric data (facial recognition)","helm-air-bench%2Fairbench 2024 - #41-42.42: unauthorized processing - educational records","helm-air-bench%2Fairbench 2024 - #41-42.43: unauthorized processing - financial records","helm-air-bench%2Fairbench 2024 - #41-42.44: unauthorized processing - behavioral/ preference data","helm-air-bench%2Fairbench 2024 - #41-42.45: unauthorized processing - communication records","helm-air-bench%2Fairbench 2024 - #41-42.46: unauthorized inference/synthesis - pii (personal identifiable information)","helm-air-bench%2Fairbench 2024 - #41-42.47: unauthorized inference/synthesis - health data","helm-air-bench%2Fairbench 2024 - #41-42.48: unauthorized inference/synthesis - location data","helm-air-bench%2Fairbench 2024 - #41-42.49: unauthorized inference/synthesis - demographic data","helm-air-bench%2Fairbench 2024 - #41-42.4: unauthorized generation - demographic data","helm-air-bench%2Fairbench 2024 - #41-42.50: unauthorized inference/synthesis - biometric data (facial recognition)","helm-air-bench%2Fairbench 2024 - #41-42.51: unauthorized inference/synthesis - educational records","helm-air-bench%2Fairbench 2024 - #41-42.52: unauthorized inference/synthesis - financial records","helm-air-bench%2Fairbench 2024 - #41-42.53: unauthorized inference/synthesis - behavioral/ preference data","helm-air-bench%2Fairbench 2024 - #41-42.54: unauthorized inference/synthesis - communication records","helm-air-bench%2Fairbench 2024 - #41-42.55: non-consensual tracking/monitoring/stalking/spyware - pii (personal identifiable information)","helm-air-bench%2Fairbench 2024 - #41-42.56: non-consensual tracking/monitoring/stalking/spyware - health data","helm-air-bench%2Fairbench 2024 - #41-42.57: non-consensual tracking/monitoring/stalking/spyware - location data","helm-air-bench%2Fairbench 2024 - #41-42.58: non-consensual tracking/monitoring/stalking/spyware - demographic data","helm-air-bench%2Fairbench 2024 - #41-42.59: non-consensual tracking/monitoring/stalking/spyware - biometric data (facial recognition)","helm-air-bench%2Fairbench 2024 - #41-42.5: unauthorized generation - biometric data (facial recognition)","helm-air-bench%2Fairbench 2024 - #41-42.60: non-consensual tracking/monitoring/stalking/spyware - educational records","helm-air-bench%2Fairbench 2024 - #41-42.61: non-consensual tracking/monitoring/stalking/spyware - financial records","helm-air-bench%2Fairbench 2024 - #41-42.62: non-consensual tracking/monitoring/stalking/spyware - behavioral/ preference data","helm-air-bench%2Fairbench 2024 - #41-42.63: non-consensual tracking/monitoring/stalking/spyware - communication records","helm-air-bench%2Fairbench 2024 - #41-42.64: model attacks (membership inference, model inversion) - pii (personal identifiable information)","helm-air-bench%2Fairbench 2024 - #41-42.65: model attacks (membership inference, model inversion) - health data","helm-air-bench%2Fairbench 2024 - #41-42.66: model attacks (membership inference, model inversion) - location data","helm-air-bench%2Fairbench 2024 - #41-42.67: model attacks (membership inference, model inversion) - demographic data","helm-air-bench%2Fairbench 2024 - #41-42.68: model attacks (membership inference, model inversion) - biometric data (facial recognition)","helm-air-bench%2Fairbench 2024 - #41-42.69: model attacks (membership inference, model inversion) - educational records","helm-air-bench%2Fairbench 2024 - #41-42.6: unauthorized generation - educational records","helm-air-bench%2Fairbench 2024 - #41-42.70: model attacks (membership inference, model inversion) - financial records","helm-air-bench%2Fairbench 2024 - #41-42.71: model attacks (membership inference, model inversion) - behavioral/ preference data","helm-air-bench%2Fairbench 2024 - #41-42.72: model attacks (membership inference, model inversion) - communication records","helm-air-bench%2Fairbench 2024 - #41-42.7: unauthorized generation - financial records","helm-air-bench%2Fairbench 2024 - #41-42.8: unauthorized generation - behavioral/ preference data","helm-air-bench%2Fairbench 2024 - #41-42.9: unauthorized generation - communication records","helm-air-bench%2Fairbench 2024 - #43.1: illegal drugs","helm-air-bench%2Fairbench 2024 - #44.2: human trafficking","helm-air-bench%2Fairbench 2024 - #44.3: sexual exploitation","helm-air-bench%2Fairbench 2024 - #44.4: prostitution","helm-air-bench%2Fairbench 2024 - #45.5: undermining national security or other government interests","helm-air-bench%2Fairbench 2024 - #45.6: undermining social stability","helm-air-bench%2Fairbench 2024 - #45.7: undermining international relations","helm-air-bench%2Fairbench 2024 - #45.8: abetting/furthering activities violating any applicable law","helm-air-bench%2Fairbench 2024 - #5.11: heavy machinery","helm-air-bench%2Fairbench 2024 - #5.12: transportation","helm-air-bench%2Fairbench 2024 - #5.13: energy/electrical grids","helm-air-bench%2Fairbench 2024 - #5.14: nuclear facilities","helm-air-bench%2Fairbench 2024 - #5.15: aircraft navigation/air traffic control","helm-air-bench%2Fairbench 2024 - #5.16: communication systems","helm-air-bench%2Fairbench 2024 - #5.17: water treatment facilities","helm-air-bench%2Fairbench 2024 - #5.18: life support","helm-air-bench%2Fairbench 2024 - #5.19: weapon systems/battlefield management","helm-air-bench%2Fairbench 2024 - #5.20: emergency services","helm-air-bench%2Fairbench 2024 - #5.21: other unauthorized actions on behalf of users","helm-air-bench%2Fairbench 2024 - #6.22: legal","helm-air-bench%2Fairbench 2024 - #6.23: medical/pharmaceutical","helm-air-bench%2Fairbench 2024 - #6.24: accounting","helm-air-bench%2Fairbench 2024 - #6.25: financial","helm-air-bench%2Fairbench 2024 - #6.26: government services","helm-air-bench%2Fairbench 2024 - #7.1: extremism","helm-air-bench%2Fairbench 2024 - #7.2: terrorism","helm-air-bench%2Fairbench 2024 - #7.3: criminal organization","helm-air-bench%2Fairbench 2024 - #8.4: glorifying violence, abuse, or the suffering of others","helm-air-bench%2Fairbench 2024 - #8.5: belittling victimhood or violent events","helm-air-bench%2Fairbench 2024 - #8.6: denying or minimizing well-documented, major violent events or the victimhood of such events/denying the deeds of martyrdom","helm-air-bench%2Fairbench 2024 - #8.7: beautifying and whitewashing acts of war or aggression","helm-air-bench%2Fairbench 2024 - #9.10: property damage","helm-air-bench%2Fairbench 2024 - #9.11: environmental","helm-air-bench%2Fairbench 2024 - #9.8: persons (including murder)","helm-air-bench%2Fairbench 2024 - #9.9: animals"],"reproducibility_summary":{"results_total":72,"has_reproducibility_gap_count":72,"populated_ratio_avg":0.10714285714285698},"provenance_summary":{"total_results":72,"total_groups":72,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":72,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":72,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":true,"derivedTags":["safety"]}],"reproducibility_summary":{"results_total":72,"has_reproducibility_gap_count":72,"populated_ratio_avg":0.10714285714285698},"provenance_summary":{"total_results":72,"total_groups":72,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":72,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":72,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":true,"derivedTags":["general","safety"]},{"key":"helm-capabilities","display_name":"HELM Capabilities","category":"Reasoning","tags":{"domains":["Creative Tasks","Info Seeking","Math & Data","Reasoning & Planning","STEM","biology","business","chemistry","computer science","economics","engineering","health","history","humanities","information retrieval","law","math","mathematics","olympiads","philosophy","physics","psychology","question answering","sentiment analysis","social sciences","summarization","toxicity detection"],"languages":["English"],"tasks":["Following verifiable instructions","Information retrieval","Multiple-choice question answering","Question answering","Sentiment analysis","Solving Olympiad-level mathematical problems","Summarization","Text generation","Toxicity detection"]},"evals_count":6,"benchmarks":[{"key":"gpqa","display_name":"GPQA","family_id":"gpqa","is_slice":false,"is_overall":false,"primary_metric_key":"cot-correct","has_card":true,"tags":{"domains":["biology","physics","chemistry"],"languages":["English"],"tasks":["Multiple-choice question answering"]},"metrics":[{"key":"cot-correct","display_name":"COT Correct","sources":["crfm"],"models_count":62,"is_primary":true}],"slices":[{"key":"gpqa","display_name":"GPQA","is_bare_stem":true,"metrics":[{"key":"cot-correct","display_name":"COT Correct","sources":["crfm"]}],"derivedTags":["natural_sciences"]}],"summary_eval_ids":["helm-capabilities%2Fgpqa"],"reproducibility_summary":{"results_total":62,"has_reproducibility_gap_count":62,"populated_ratio_avg":0.928571428571429},"provenance_summary":{"total_results":62,"total_groups":62,"multi_source_groups":49,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":62,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":62,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["natural_sciences"]},{"key":"helm-capabilities-leaderboard","display_name":"HELM-Capabilities-Leaderboard","family_id":"helm-capabilities-leaderboard","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["question answering","information retrieval","summarization","sentiment analysis","toxicity detection"],"languages":["English"],"tasks":["Question answering","Information retrieval","Summarization","Sentiment analysis","Toxicity detection"]},"metrics":[{"key":"score","display_name":"Score","sources":["crfm"],"models_count":62,"is_primary":true}],"slices":[],"summary_eval_ids":["helm-capabilities%2Fhelm-capabilities-leaderboard"],"reproducibility_summary":{"results_total":62,"has_reproducibility_gap_count":62,"populated_ratio_avg":0.9285714285714292},"provenance_summary":{"total_results":62,"total_groups":62,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":62,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":62,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["general"]},{"key":"omni-math","display_name":"Omni-MATH","family_id":"omni-math","is_slice":false,"is_overall":false,"primary_metric_key":"accuracy","has_card":true,"tags":{"domains":["mathematics","olympiads"],"languages":["English"],"tasks":["Solving Olympiad-level mathematical problems"]},"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["crfm"],"models_count":62,"is_primary":true}],"slices":[{"key":"omni-math","display_name":"Omni-MATH","is_bare_stem":true,"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["crfm"]}],"derivedTags":["mathematics"]}],"summary_eval_ids":["helm-capabilities%2Fomni-math"],"reproducibility_summary":{"results_total":62,"has_reproducibility_gap_count":62,"populated_ratio_avg":0.9285714285714292},"provenance_summary":{"total_results":62,"total_groups":62,"multi_source_groups":3,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":62,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":62,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["mathematics"]},{"key":"wildbench","display_name":"WildBench","family_id":"wildbench","is_slice":false,"is_overall":false,"primary_metric_key":"WB Score","has_card":true,"tags":{"domains":["Info Seeking","Math & Data","Reasoning & Planning","Creative Tasks"],"languages":["English"],"tasks":["Text generation"]},"metrics":[{"key":"WB Score","display_name":"WB Score","sources":["crfm"],"models_count":62,"is_primary":true}],"slices":[],"summary_eval_ids":["helm-capabilities%2Fwildbench"],"reproducibility_summary":{"results_total":62,"has_reproducibility_gap_count":62,"populated_ratio_avg":0.9285714285714292},"provenance_summary":{"total_results":62,"total_groups":62,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":62,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":62,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["general"]}],"reproducibility_summary":{"results_total":372,"has_reproducibility_gap_count":372,"populated_ratio_avg":0.9285714285714289},"provenance_summary":{"total_results":372,"total_groups":372,"multi_source_groups":126,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":372,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":372,"groups_with_variant_check":0,"groups_with_cross_party_check":9,"variant_divergent_count":0,"cross_party_divergent_count":6},"is_primary":false,"derivedTags":["general","natural_sciences","mathematics"]},{"key":"helm-classic","display_name":"HELM Classic","category":"Knowledge","tags":{"domains":["Not specified","STEM","US history","abstract algebra","academic research","anatomy","astronomy","bias detection","bias measurement","biology","business ethics","chemistry","clinical knowledge","commonsense reasoning","computer science","customer service","dialogue modeling","economics","elementary mathematics","engineering","finance","formal logic","geography","government and politics","health","history","humanities","information retrieval","journalism","language generation","language understanding","law","legal documents","machine learning fairness","machine reading comprehension","mathematics","medical literature","medicine","morality","natural language inference","natural language understanding","news articles","news media","philosophy","physics","politics","question answering","reading comprehension","reasoning","sentiment analysis","social media content","social sciences","stories and narratives","summarization","text classification","text generation","toxic comment classification"],"languages":["English"],"tasks":["Analysis of performance across identity subgroups","Answering questions about stories by reading entire books or movie scripts","Answering questions using human-written summaries (summaries only)","Answering questions using the full story text (stories only)","Binary sentiment classification","Binary toxicity classification (toxic vs. non-toxic)","Commonsense inference","Extractive question answering","Fill mask","Four-way multiple-choice selection for event continuation","Generating a well-formed answer based on context","Knowledge evaluation across subjects including mathematics, history, computer science, law, morality, and philosophy","Multiple-choice question answering","Natural language generation","News article summarization","Open-domain question answering","Predicting if a question is answerable given context passages and synthesizing an answer","Question answering","Ranking retrieved passages for a question","Reasoning","Single-document extreme summarization","Summarization","Text classification","Text generation","Text summarization","Text-pair classification","Toxic comment identification","Yes/no question answering"]},"evals_count":15,"benchmarks":[{"key":"boolq","display_name":"BoolQ","family_id":"boolq","is_slice":false,"is_overall":false,"primary_metric_key":"exact-match","has_card":true,"tags":{"domains":["natural language understanding","reading comprehension","natural language inference"],"languages":["English"],"tasks":["Yes/no question answering","Text-pair classification"]},"metrics":[{"key":"exact-match","display_name":"Exact Match","sources":["crfm"],"models_count":50,"is_primary":true}],"slices":[{"key":"boolq","display_name":"BoolQ","is_bare_stem":true,"metrics":[{"key":"exact-match","display_name":"Exact Match","sources":["crfm"]}],"derivedTags":["commonsense_reasoning"]}],"summary_eval_ids":["helm-classic%2Fboolq"],"reproducibility_summary":{"results_total":50,"has_reproducibility_gap_count":50,"populated_ratio_avg":0.9285714285714287},"provenance_summary":{"total_results":50,"total_groups":50,"multi_source_groups":15,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":50,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":50,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["commonsense_reasoning"]},{"key":"civilcomments","display_name":"CivilComments","family_id":"civilcomments","is_slice":false,"is_overall":false,"primary_metric_key":"exact-match","has_card":true,"tags":{"domains":["machine learning fairness","bias measurement","toxic comment classification","text classification"],"languages":["English"],"tasks":["Binary toxicity classification (toxic vs. non-toxic)","Analysis of performance across identity subgroups"]},"metrics":[{"key":"exact-match","display_name":"Exact Match","sources":["crfm"],"models_count":50,"is_primary":true}],"slices":[],"summary_eval_ids":["helm-classic%2Fcivilcomments"],"reproducibility_summary":{"results_total":50,"has_reproducibility_gap_count":50,"populated_ratio_avg":0.9285714285714287},"provenance_summary":{"total_results":50,"total_groups":50,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":50,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":50,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["safety"]},{"key":"cnn-dailymail","display_name":"CNN/DailyMail","family_id":"cnn-dailymail","is_slice":false,"is_overall":false,"primary_metric_key":"ROUGE-2","has_card":true,"tags":{"domains":["summarization","journalism","news media"],"languages":["English"],"tasks":["Summarization"]},"metrics":[{"key":"ROUGE-2","display_name":"ROUGE-2","sources":["crfm"],"models_count":33,"is_primary":true}],"slices":[{"key":"cnn/dailymail","display_name":"CNN/DailyMail","is_bare_stem":false,"metrics":[{"key":"ROUGE-2","display_name":"ROUGE-2","sources":["crfm"]}],"derivedTags":["linguistic_core"]}],"summary_eval_ids":["helm-classic%2Fcnn-dailymail"],"reproducibility_summary":{"results_total":33,"has_reproducibility_gap_count":33,"populated_ratio_avg":0.9285714285714279},"provenance_summary":{"total_results":33,"total_groups":33,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":33,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":33,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["linguistic_core"]},{"key":"hellaswag","display_name":"HellaSwag","family_id":"hellaswag","is_slice":false,"is_overall":false,"primary_metric_key":"exact-match","has_card":true,"tags":{"domains":["commonsense reasoning","natural language inference"],"languages":["English"],"tasks":["Four-way multiple-choice selection for event continuation","Commonsense inference"]},"metrics":[{"key":"exact-match","display_name":"Exact Match","sources":["crfm"],"models_count":23,"is_primary":true}],"slices":[{"key":"hellaswag","display_name":"HellaSwag","is_bare_stem":true,"metrics":[{"key":"exact-match","display_name":"Exact Match","sources":["crfm"]}],"derivedTags":["commonsense_reasoning"]}],"summary_eval_ids":["helm-classic%2Fhellaswag"],"reproducibility_summary":{"results_total":23,"has_reproducibility_gap_count":23,"populated_ratio_avg":0.9285714285714283},"provenance_summary":{"total_results":23,"total_groups":23,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":23,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":23,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["commonsense_reasoning"]},{"key":"helm-classic-leaderboard","display_name":"HELM-Classic-Leaderboard","family_id":"helm-classic-leaderboard","is_slice":false,"is_overall":false,"primary_metric_key":"rank","has_card":true,"tags":{"domains":["question answering","summarization","reasoning","bias detection","language understanding","language generation"],"languages":["English"],"tasks":["Question answering","Text summarization","Text classification","Reasoning","Toxic comment identification"]},"metrics":[{"key":"rank","display_name":"Rank","sources":["crfm"],"models_count":50,"is_primary":true}],"slices":[],"summary_eval_ids":["helm-classic%2Fhelm-classic-leaderboard"],"reproducibility_summary":{"results_total":50,"has_reproducibility_gap_count":50,"populated_ratio_avg":0.9285714285714287},"provenance_summary":{"total_results":50,"total_groups":50,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":50,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":50,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["general"]},{"key":"imdb","display_name":"IMDb","family_id":"imdb","is_slice":false,"is_overall":false,"primary_metric_key":"exact-match","has_card":true,"tags":{"domains":["sentiment analysis"],"languages":["English"],"tasks":["Binary sentiment classification","Text classification"]},"metrics":[{"key":"exact-match","display_name":"Exact Match","sources":["crfm"],"models_count":50,"is_primary":true}],"slices":[{"key":"imdb","display_name":"IMDB","is_bare_stem":true,"metrics":[{"key":"exact-match","display_name":"Exact Match","sources":["crfm"]}],"derivedTags":["linguistic_core"]}],"summary_eval_ids":["helm-classic%2Fimdb"],"reproducibility_summary":{"results_total":50,"has_reproducibility_gap_count":50,"populated_ratio_avg":0.9285714285714287},"provenance_summary":{"total_results":50,"total_groups":50,"multi_source_groups":15,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":50,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":50,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["linguistic_core"]},{"key":"ms-marco-trec","display_name":"MS MARCO TREC","family_id":"ms-marco-trec","is_slice":false,"is_overall":false,"primary_metric_key":"NDCG@10","has_card":true,"tags":{"domains":["information retrieval","question answering","machine reading comprehension"],"languages":["English"],"tasks":["Predicting if a question is answerable given context passages and synthesizing an answer","Generating a well-formed answer based on context","Ranking retrieved passages for a question","Question answering","Natural language generation"]},"metrics":[{"key":"NDCG@10","display_name":"NDCG@10","sources":["crfm"],"models_count":22,"is_primary":true}],"slices":[],"summary_eval_ids":["helm-classic%2Fms-marco-trec"],"reproducibility_summary":{"results_total":22,"has_reproducibility_gap_count":22,"populated_ratio_avg":0.9285714285714284},"provenance_summary":{"total_results":22,"total_groups":22,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":22,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":22,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["linguistic_core","applied_reasoning"]},{"key":"narrativeqa","display_name":"NarrativeQA","family_id":"narrativeqa","is_slice":false,"is_overall":false,"primary_metric_key":"f1","has_card":true,"tags":{"domains":["reading comprehension","natural language understanding","stories and narratives"],"languages":["English"],"tasks":["Answering questions about stories by reading entire books or movie scripts","Answering questions using human-written summaries (summaries only)","Answering questions using the full story text (stories only)"]},"metrics":[{"key":"f1","display_name":"F1 Score","sources":["crfm"],"models_count":50,"is_primary":true}],"slices":[],"summary_eval_ids":["helm-classic%2Fnarrativeqa"],"reproducibility_summary":{"results_total":50,"has_reproducibility_gap_count":50,"populated_ratio_avg":0.9285714285714287},"provenance_summary":{"total_results":50,"total_groups":50,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":50,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":50,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["linguistic_core"]},{"key":"naturalquestions","display_name":"NaturalQuestions","family_id":"naturalquestions","is_slice":false,"is_overall":false,"primary_metric_key":"f1","has_card":true,"tags":{"domains":["Not specified"],"languages":["English"],"tasks":["Question answering"]},"metrics":[{"key":"f1","display_name":"F1 Score","sources":["crfm"],"models_count":49,"is_primary":true}],"slices":[{"key":"naturalquestions (open-book)","display_name":"NaturalQuestions (open-book)","is_bare_stem":false,"metrics":[{"key":"f1","display_name":"F1 Score","sources":["crfm"]}],"derivedTags":["knowledge"]}],"summary_eval_ids":["helm-classic%2Fnaturalquestions"],"reproducibility_summary":{"results_total":49,"has_reproducibility_gap_count":49,"populated_ratio_avg":0.9285714285714283},"provenance_summary":{"total_results":49,"total_groups":49,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":49,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":49,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["knowledge"]},{"key":"openbookqa","display_name":"OpenbookQA","family_id":"openbookqa","is_slice":false,"is_overall":false,"primary_metric_key":"exact-match","has_card":true,"tags":{"domains":["Not specified"],"languages":["English"],"tasks":["Open-domain question answering"]},"metrics":[{"key":"exact-match","display_name":"Exact Match","sources":["crfm"],"models_count":23,"is_primary":true}],"slices":[{"key":"openbookqa","display_name":"OpenbookQA","is_bare_stem":true,"metrics":[{"key":"exact-match","display_name":"Exact Match","sources":["crfm"]}],"derivedTags":["commonsense_reasoning"]}],"summary_eval_ids":["helm-classic%2Fopenbookqa"],"reproducibility_summary":{"results_total":23,"has_reproducibility_gap_count":23,"populated_ratio_avg":0.9285714285714283},"provenance_summary":{"total_results":23,"total_groups":23,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":23,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":23,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["commonsense_reasoning"]},{"key":"quac","display_name":"QuAC","family_id":"quac","is_slice":false,"is_overall":false,"primary_metric_key":"f1","has_card":true,"tags":{"domains":["question answering","dialogue modeling","text generation"],"languages":["English"],"tasks":["Extractive question answering","Text generation","Fill mask"]},"metrics":[{"key":"f1","display_name":"F1 Score","sources":["crfm"],"models_count":50,"is_primary":true}],"slices":[],"summary_eval_ids":["helm-classic%2Fquac"],"reproducibility_summary":{"results_total":50,"has_reproducibility_gap_count":50,"populated_ratio_avg":0.9285714285714287},"provenance_summary":{"total_results":50,"total_groups":50,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":50,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":50,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["linguistic_core"]},{"key":"raft","display_name":"RAFT","family_id":"raft","is_slice":false,"is_overall":false,"primary_metric_key":"exact-match","has_card":true,"tags":{"domains":["text classification","medical literature","customer service","academic research","legal documents","news articles","social media content"],"languages":["English"],"tasks":["Text classification"]},"metrics":[{"key":"exact-match","display_name":"Exact Match","sources":["crfm"],"models_count":50,"is_primary":true}],"slices":[],"summary_eval_ids":["helm-classic%2Fraft"],"reproducibility_summary":{"results_total":50,"has_reproducibility_gap_count":50,"populated_ratio_avg":0.9285714285714287},"provenance_summary":{"total_results":50,"total_groups":50,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":50,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":50,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["linguistic_core","applied_reasoning"]},{"key":"truthfulqa","display_name":"TruthfulQA","family_id":"truthfulqa","is_slice":false,"is_overall":false,"primary_metric_key":"exact-match","has_card":true,"tags":{"domains":["health","law","finance","politics"],"languages":["English"],"tasks":["Text generation","Multiple-choice question answering"]},"metrics":[{"key":"exact-match","display_name":"Exact Match","sources":["crfm"],"models_count":50,"is_primary":true}],"slices":[{"key":"truthfulqa","display_name":"TruthfulQA","is_bare_stem":true,"metrics":[{"key":"exact-match","display_name":"Exact Match","sources":["crfm"]}],"derivedTags":["hallucination"]}],"summary_eval_ids":["helm-classic%2Ftruthfulqa"],"reproducibility_summary":{"results_total":50,"has_reproducibility_gap_count":50,"populated_ratio_avg":0.9285714285714287},"provenance_summary":{"total_results":50,"total_groups":50,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":50,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":50,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["hallucination"]},{"key":"xsum","display_name":"XSum","family_id":"xsum","is_slice":false,"is_overall":false,"primary_metric_key":"ROUGE-2","has_card":true,"tags":{"domains":["news articles","summarization"],"languages":["English"],"tasks":["Single-document extreme summarization","News article summarization"]},"metrics":[{"key":"ROUGE-2","display_name":"ROUGE-2","sources":["crfm"],"models_count":33,"is_primary":true}],"slices":[{"key":"xsum","display_name":"XSUM","is_bare_stem":true,"metrics":[{"key":"ROUGE-2","display_name":"ROUGE-2","sources":["crfm"]}],"derivedTags":["linguistic_core"]}],"summary_eval_ids":["helm-classic%2Fxsum"],"reproducibility_summary":{"results_total":33,"has_reproducibility_gap_count":33,"populated_ratio_avg":0.9285714285714279},"provenance_summary":{"total_results":33,"total_groups":33,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":33,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":33,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["linguistic_core"]}],"reproducibility_summary":{"results_total":633,"has_reproducibility_gap_count":633,"populated_ratio_avg":0.9285714285714286},"provenance_summary":{"total_results":633,"total_groups":633,"multi_source_groups":30,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":633,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":633,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["general","commonsense_reasoning","safety","linguistic_core","applied_reasoning","knowledge","hallucination"]},{"key":"helm-instruct","display_name":"HELM Instruct","category":"General","tags":{"domains":["AI alignment","AI safety","ChatGPT","Claude","GPT","Gemini","LLM","Llama","Mistral","Not specified","awesome list","chatbot","conversational AI","human feedback","information retrieval","prompt engineering","prompts","question answering","sentiment analysis","summarization","text generation","toxicity detection"],"languages":["English","Not specified"],"tasks":["Information retrieval","Not specified","Preference modeling for comparing model responses","Question answering","Sentiment analysis","Summarization","Text generation","Toxicity detection","Training RLHF policies"]},"evals_count":7,"benchmarks":[{"key":"anthropic-rlhf-dataset","display_name":"Anthropic RLHF Dataset","family_id":"anthropic-rlhf-dataset","is_slice":false,"is_overall":false,"primary_metric_key":"harmlessness","has_card":true,"tags":{"domains":["AI alignment","AI safety","human feedback"],"languages":["Not specified"],"tasks":["Preference modeling for comparing model responses","Training RLHF policies"]},"metrics":[{"key":"harmlessness","display_name":"Harmlessness","sources":["crfm"],"models_count":4,"is_primary":true}],"slices":[],"summary_eval_ids":["helm-instruct%2Fanthropic-rlhf-dataset"],"reproducibility_summary":{"results_total":4,"has_reproducibility_gap_count":4,"populated_ratio_avg":0.9285714285714286},"provenance_summary":{"total_results":4,"total_groups":4,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":4,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":4,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["general"]},{"key":"best-chatgpt-prompts","display_name":"Best ChatGPT Prompts","family_id":"best-chatgpt-prompts","is_slice":false,"is_overall":false,"primary_metric_key":"harmlessness","has_card":true,"tags":{"domains":["ChatGPT","prompts","GPT","Claude","Gemini","Llama","Mistral","LLM","prompt engineering","conversational AI","text generation","chatbot","awesome list"],"languages":["Not specified"],"tasks":["Question answering","Text generation"]},"metrics":[{"key":"harmlessness","display_name":"Harmlessness","sources":["crfm"],"models_count":4,"is_primary":true}],"slices":[],"summary_eval_ids":["helm-instruct%2Fbest-chatgpt-prompts"],"reproducibility_summary":{"results_total":4,"has_reproducibility_gap_count":4,"populated_ratio_avg":0.9285714285714286},"provenance_summary":{"total_results":4,"total_groups":4,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":4,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":4,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["general"]},{"key":"helm-instruct-leaderboard","display_name":"HELM-Instruct-Leaderboard","family_id":"helm-instruct-leaderboard","is_slice":false,"is_overall":false,"primary_metric_key":"rank","has_card":true,"tags":{"domains":["question answering","information retrieval","summarization","sentiment analysis","toxicity detection"],"languages":["Not specified"],"tasks":["Question answering","Information retrieval","Summarization","Sentiment analysis","Toxicity detection"]},"metrics":[{"key":"rank","display_name":"Rank","sources":["crfm"],"models_count":4,"is_primary":true}],"slices":[],"summary_eval_ids":["helm-instruct%2Fhelm-instruct-leaderboard"],"reproducibility_summary":{"results_total":4,"has_reproducibility_gap_count":4,"populated_ratio_avg":0.9285714285714286},"provenance_summary":{"total_results":4,"total_groups":4,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":4,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":4,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["general"]},{"key":"koala-test-dataset","display_name":"Koala Test Dataset","family_id":"koala-test-dataset","is_slice":false,"is_overall":false,"primary_metric_key":"harmlessness","has_card":true,"tags":{"domains":["Not specified"],"languages":["Not specified"],"tasks":["Not specified"]},"metrics":[{"key":"harmlessness","display_name":"Harmlessness","sources":["crfm"],"models_count":4,"is_primary":true}],"slices":[],"summary_eval_ids":["helm-instruct%2Fkoala-test-dataset"],"reproducibility_summary":{"results_total":4,"has_reproducibility_gap_count":4,"populated_ratio_avg":0.9285714285714286},"provenance_summary":{"total_results":4,"total_groups":4,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":4,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":4,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["general"]},{"key":"open-assistant","display_name":"OpenAssistant","family_id":"open-assistant","is_slice":false,"is_overall":false,"primary_metric_key":"harmlessness","has_card":true,"tags":{"domains":["Not specified"],"languages":["Not specified"],"tasks":["Not specified"]},"metrics":[{"key":"harmlessness","display_name":"Harmlessness","sources":["crfm"],"models_count":4,"is_primary":true}],"slices":[],"summary_eval_ids":["helm-instruct%2Fopen-assistant"],"reproducibility_summary":{"results_total":4,"has_reproducibility_gap_count":4,"populated_ratio_avg":0.9285714285714286},"provenance_summary":{"total_results":4,"total_groups":4,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":4,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":4,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["general"]},{"key":"self-instruct","display_name":"Self-Instruct","family_id":"self-instruct","is_slice":false,"is_overall":false,"primary_metric_key":"harmlessness","has_card":true,"tags":{"domains":["Not specified"],"languages":["Not specified"],"tasks":["Not specified"]},"metrics":[{"key":"harmlessness","display_name":"Harmlessness","sources":["crfm"],"models_count":4,"is_primary":true}],"slices":[],"summary_eval_ids":["helm-instruct%2Fself-instruct"],"reproducibility_summary":{"results_total":4,"has_reproducibility_gap_count":4,"populated_ratio_avg":0.9285714285714286},"provenance_summary":{"total_results":4,"total_groups":4,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":4,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":4,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["general"]},{"key":"vicuna","display_name":"Vicuna","family_id":"vicuna","is_slice":false,"is_overall":false,"primary_metric_key":"harmlessness","has_card":true,"tags":{"domains":["Not specified"],"languages":["English"],"tasks":["Not specified"]},"metrics":[{"key":"harmlessness","display_name":"Harmlessness","sources":["crfm"],"models_count":4,"is_primary":true}],"slices":[],"summary_eval_ids":["helm-instruct%2Fvicuna"],"reproducibility_summary":{"results_total":4,"has_reproducibility_gap_count":4,"populated_ratio_avg":0.9285714285714286},"provenance_summary":{"total_results":4,"total_groups":4,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":4,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":4,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["general"]}],"reproducibility_summary":{"results_total":28,"has_reproducibility_gap_count":28,"populated_ratio_avg":0.9285714285714287},"provenance_summary":{"total_results":28,"total_groups":28,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":28,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":28,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["general"]},{"key":"helm-lite","display_name":"HELM Lite","category":"Knowledge","tags":{"domains":["Not specified","STEM","US history","abstract algebra","anatomy","astronomy","biology","business ethics","chemistry","clinical knowledge","computer science","economics","elementary mathematics","engineering","explanation generation","finance","formal logic","geography","government and politics","grade school mathematics","history","humanities","law","legal","legal analysis","math word problems","mathematical reasoning","mathematics","medical knowledge","medicine","morality","multilingual understanding","narrative comprehension","natural language understanding","philosophy","physics","professional medical exams","question answering","reading comprehension","social sciences","stories and narratives"],"languages":["English","Not specified","multiple"],"tasks":["Answering questions about stories by reading entire books or movie scripts","Answering questions using human-written summaries (summaries only)","Answering questions using the full story text (stories only)","Free-form multiple-choice question answering","General knowledge evaluation","Generating a final answer sequence given a math problem","Knowledge evaluation across subjects including mathematics, history, computer science, law, morality, and philosophy","Legal reasoning","Machine translation","Mathematical reasoning","Medical question answering","Multiple-choice question answering","Not specified","Open-domain question answering","Question answering","Reading comprehension","Rule-application tasks","Solving grade school math word problems","Text classification","Text generation","Text generation for question answering"]},"evals_count":10,"benchmarks":[{"key":"gsm8k","display_name":"GSM8K","family_id":"gsm8k","is_slice":false,"is_overall":false,"primary_metric_key":"exact-match","has_card":true,"tags":{"domains":["grade school mathematics","math word problems"],"languages":["English"],"tasks":["Solving grade school math word problems","Text generation for question answering"]},"metrics":[{"key":"exact-match","display_name":"Exact Match","sources":["crfm"],"models_count":76,"is_primary":true}],"slices":[{"key":"gsm8k","display_name":"GSM8K","is_bare_stem":true,"metrics":[{"key":"exact-match","display_name":"Exact Match","sources":["crfm"]}],"derivedTags":["mathematics"]}],"summary_eval_ids":["helm-lite%2Fgsm8k"],"reproducibility_summary":{"results_total":76,"has_reproducibility_gap_count":76,"populated_ratio_avg":0.9285714285714293},"provenance_summary":{"total_results":76,"total_groups":76,"multi_source_groups":14,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":76,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":76,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["mathematics"]},{"key":"helm-lite-leaderboard","display_name":"HELM-LITE-Leaderboard","family_id":"helm-lite-leaderboard","is_slice":false,"is_overall":false,"primary_metric_key":"rank","has_card":true,"tags":{"domains":["mathematical reasoning","legal analysis","multilingual understanding","medical knowledge","narrative comprehension","question answering"],"languages":["English","multiple"],"tasks":["Mathematical reasoning","Legal reasoning","Medical question answering","General knowledge evaluation","Reading comprehension","Machine translation"]},"metrics":[{"key":"rank","display_name":"Rank","sources":["crfm"],"models_count":77,"is_primary":true}],"slices":[],"summary_eval_ids":["helm-lite%2Fhelm-lite-leaderboard"],"reproducibility_summary":{"results_total":77,"has_reproducibility_gap_count":77,"populated_ratio_avg":0.9285714285714293},"provenance_summary":{"total_results":77,"total_groups":77,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":77,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":77,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["general"]},{"key":"legalbench","display_name":"LegalBench","family_id":"legalbench","is_slice":false,"is_overall":false,"primary_metric_key":"exact-match","has_card":true,"tags":{"domains":["legal","law","finance"],"languages":["English"],"tasks":["Text classification","Question answering","Text generation","Rule-application tasks"]},"metrics":[{"key":"exact-match","display_name":"Exact Match","sources":["crfm"],"models_count":77,"is_primary":true}],"slices":[{"key":"legalbench","display_name":"LegalBench","is_bare_stem":true,"metrics":[{"key":"exact-match","display_name":"Exact Match","sources":["crfm"]}],"derivedTags":["law"]}],"summary_eval_ids":["helm-lite%2Flegalbench"],"reproducibility_summary":{"results_total":77,"has_reproducibility_gap_count":77,"populated_ratio_avg":0.9285714285714293},"provenance_summary":{"total_results":77,"total_groups":77,"multi_source_groups":27,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":77,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":77,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["law"]},{"key":"medqa","display_name":"MedQA","family_id":"medqa","is_slice":false,"is_overall":false,"primary_metric_key":"exact-match","has_card":true,"tags":{"domains":["medical knowledge","professional medical exams"],"languages":["English"],"tasks":["Free-form multiple-choice question answering","Open-domain question answering"]},"metrics":[{"key":"exact-match","display_name":"Exact Match","sources":["crfm"],"models_count":77,"is_primary":true}],"slices":[{"key":"medqa","display_name":"MedQA","is_bare_stem":true,"metrics":[{"key":"exact-match","display_name":"Exact Match","sources":["crfm"]}],"derivedTags":["knowledge","natural_sciences"]}],"summary_eval_ids":["helm-lite%2Fmedqa"],"reproducibility_summary":{"results_total":77,"has_reproducibility_gap_count":77,"populated_ratio_avg":0.9285714285714293},"provenance_summary":{"total_results":77,"total_groups":77,"multi_source_groups":16,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":77,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":77,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["knowledge","natural_sciences"]},{"key":"narrativeqa","display_name":"NarrativeQA","family_id":"narrativeqa","is_slice":false,"is_overall":false,"primary_metric_key":"f1","has_card":true,"tags":{"domains":["reading comprehension","natural language understanding","stories and narratives"],"languages":["English"],"tasks":["Answering questions about stories by reading entire books or movie scripts","Answering questions using human-written summaries (summaries only)","Answering questions using the full story text (stories only)"]},"metrics":[{"key":"f1","display_name":"F1 Score","sources":["crfm"],"models_count":77,"is_primary":true}],"slices":[],"summary_eval_ids":["helm-lite%2Fnarrativeqa"],"reproducibility_summary":{"results_total":77,"has_reproducibility_gap_count":77,"populated_ratio_avg":0.9285714285714293},"provenance_summary":{"total_results":77,"total_groups":77,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":77,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":77,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["linguistic_core"]},{"key":"naturalquestions","display_name":"NaturalQuestions","family_id":"naturalquestions","is_slice":false,"is_overall":false,"primary_metric_key":"f1","has_card":true,"tags":{"domains":["Not specified"],"languages":["English"],"tasks":["Question answering"]},"metrics":[{"key":"f1","display_name":"F1 Score","sources":["crfm"],"models_count":77,"is_primary":true}],"slices":[{"key":"naturalquestions (closed-book)","display_name":"NaturalQuestions (closed-book)","is_bare_stem":false,"metrics":[{"key":"f1","display_name":"F1 Score","sources":["crfm"]}],"derivedTags":["knowledge"]}],"summary_eval_ids":["helm-lite%2Fnaturalquestions"],"reproducibility_summary":{"results_total":77,"has_reproducibility_gap_count":77,"populated_ratio_avg":0.9285714285714292},"provenance_summary":{"total_results":77,"total_groups":77,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":77,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":77,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["knowledge"]},{"key":"openbookqa","display_name":"OpenbookQA","family_id":"openbookqa","is_slice":false,"is_overall":false,"primary_metric_key":"exact-match","has_card":true,"tags":{"domains":["Not specified"],"languages":["English"],"tasks":["Open-domain question answering"]},"metrics":[{"key":"exact-match","display_name":"Exact Match","sources":["crfm"],"models_count":77,"is_primary":true}],"slices":[{"key":"openbookqa","display_name":"OpenbookQA","is_bare_stem":true,"metrics":[{"key":"exact-match","display_name":"Exact Match","sources":["crfm"]}],"derivedTags":["commonsense_reasoning"]}],"summary_eval_ids":["helm-lite%2Fopenbookqa"],"reproducibility_summary":{"results_total":77,"has_reproducibility_gap_count":77,"populated_ratio_avg":0.9285714285714293},"provenance_summary":{"total_results":77,"total_groups":77,"multi_source_groups":1,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":77,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":77,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["commonsense_reasoning"]},{"key":"wmt-2014","display_name":"WMT 2014","family_id":"wmt-2014","is_slice":false,"is_overall":false,"primary_metric_key":"bleu-4","has_card":true,"tags":{"domains":["Not specified"],"languages":["Not specified"],"tasks":["Not specified"]},"metrics":[{"key":"bleu-4","display_name":"BLEU-4","sources":["crfm"],"models_count":77,"is_primary":true}],"slices":[],"summary_eval_ids":["helm-lite%2Fwmt-2014"],"reproducibility_summary":{"results_total":77,"has_reproducibility_gap_count":77,"populated_ratio_avg":0.9285714285714293},"provenance_summary":{"total_results":77,"total_groups":77,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":77,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":77,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["linguistic_core"]}],"reproducibility_summary":{"results_total":769,"has_reproducibility_gap_count":769,"populated_ratio_avg":0.928571428571429},"provenance_summary":{"total_results":769,"total_groups":769,"multi_source_groups":75,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":769,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":769,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["general","mathematics","law","knowledge","natural_sciences","linguistic_core","commonsense_reasoning"]},{"key":"helm-mmlu","display_name":"HELM MMLU","category":"Safety","tags":{"domains":["STEM","US history","abstract algebra","anatomy","astronomy","biology","business ethics","chemistry","clinical knowledge","computer science","economics","elementary mathematics","engineering","formal logic","geography","government and politics","history","humanities","law","mathematics","medicine","morality","philosophy","physics","social sciences"],"languages":["English"],"tasks":["Knowledge evaluation across subjects including mathematics, history, computer science, law, morality, and philosophy","Multiple-choice question answering"]},"evals_count":2,"benchmarks":[{"key":"mmlu","display_name":"MMLU","family_id":"mmlu","is_slice":false,"is_overall":false,"primary_metric_key":"exact-match","has_card":true,"tags":{"domains":["STEM","humanities","social sciences","elementary mathematics","US history","computer science","law","morality","philosophy","abstract algebra","anatomy","astronomy","business ethics","clinical knowledge","biology","chemistry","mathematics","medicine","physics","economics","engineering","formal logic","history","geography","government and politics"],"languages":["English"],"tasks":["Multiple-choice question answering","Knowledge evaluation across subjects including mathematics, history, computer science, law, morality, and philosophy"]},"metrics":[{"key":"exact-match","display_name":"Exact Match","sources":["crfm"],"models_count":63,"is_primary":true},{"key":"rank","display_name":"Rank","sources":["crfm"],"models_count":63,"is_primary":false}],"slices":[{"key":"abstract algebra","display_name":"Abstract Algebra","is_bare_stem":false,"metrics":[{"key":"exact-match","display_name":"Exact Match","sources":["crfm"]}],"derivedTags":["general","knowledge"]},{"key":"anatomy","display_name":"Anatomy","is_bare_stem":false,"metrics":[{"key":"exact-match","display_name":"Exact Match","sources":["crfm"]}],"derivedTags":["general","knowledge"]},{"key":"astronomy","display_name":"Astronomy","is_bare_stem":false,"metrics":[{"key":"exact-match","display_name":"Exact Match","sources":["crfm"]}],"derivedTags":["general","knowledge"]},{"key":"business ethics","display_name":"Business Ethics","is_bare_stem":false,"metrics":[{"key":"exact-match","display_name":"Exact Match","sources":["crfm"]}],"derivedTags":["general","knowledge"]},{"key":"clinical knowledge","display_name":"Clinical Knowledge","is_bare_stem":false,"metrics":[{"key":"exact-match","display_name":"Exact Match","sources":["crfm"]}],"derivedTags":["general","knowledge"]},{"key":"college physics","display_name":"College Physics","is_bare_stem":false,"metrics":[{"key":"exact-match","display_name":"Exact Match","sources":["crfm"]}],"derivedTags":["general","knowledge"]},{"key":"computer security","display_name":"Computer Security","is_bare_stem":false,"metrics":[{"key":"exact-match","display_name":"Exact Match","sources":["crfm"]}],"derivedTags":["general","knowledge"]},{"key":"conceptual physics","display_name":"Conceptual Physics","is_bare_stem":false,"metrics":[{"key":"exact-match","display_name":"Exact Match","sources":["crfm"]}],"derivedTags":["general","knowledge"]},{"key":"econometrics","display_name":"Econometrics","is_bare_stem":false,"metrics":[{"key":"exact-match","display_name":"Exact Match","sources":["crfm"]}],"derivedTags":["general","knowledge"]},{"key":"electrical engineering","display_name":"Electrical Engineering","is_bare_stem":false,"metrics":[{"key":"exact-match","display_name":"Exact Match","sources":["crfm"]}],"derivedTags":["general","knowledge"]},{"key":"elementary mathematics","display_name":"Elementary Mathematics","is_bare_stem":false,"metrics":[{"key":"exact-match","display_name":"Exact Match","sources":["crfm"]}],"derivedTags":["general","knowledge"]},{"key":"formal logic","display_name":"Formal Logic","is_bare_stem":false,"metrics":[{"key":"exact-match","display_name":"Exact Match","sources":["crfm"]}],"derivedTags":["general","knowledge"]},{"key":"global facts","display_name":"Global Facts","is_bare_stem":false,"metrics":[{"key":"exact-match","display_name":"Exact Match","sources":["crfm"]}],"derivedTags":["general","knowledge"]},{"key":"high school world history","display_name":"High School World History","is_bare_stem":false,"metrics":[{"key":"exact-match","display_name":"Exact Match","sources":["crfm"]}],"derivedTags":["general","knowledge"]},{"key":"human sexuality","display_name":"Human Sexuality","is_bare_stem":false,"metrics":[{"key":"exact-match","display_name":"Exact Match","sources":["crfm"]}],"derivedTags":["general","knowledge"]},{"key":"international law","display_name":"International Law","is_bare_stem":false,"metrics":[{"key":"exact-match","display_name":"Exact Match","sources":["crfm"]}],"derivedTags":["general","knowledge"]},{"key":"jurisprudence","display_name":"Jurisprudence","is_bare_stem":false,"metrics":[{"key":"exact-match","display_name":"Exact Match","sources":["crfm"]}],"derivedTags":["general","knowledge"]},{"key":"logical fallacies","display_name":"Logical Fallacies","is_bare_stem":false,"metrics":[{"key":"exact-match","display_name":"Exact Match","sources":["crfm"]}],"derivedTags":["general","knowledge"]},{"key":"machine learning","display_name":"Machine Learning","is_bare_stem":false,"metrics":[{"key":"exact-match","display_name":"Exact Match","sources":["crfm"]}],"derivedTags":["general","knowledge"]},{"key":"management","display_name":"Management","is_bare_stem":false,"metrics":[{"key":"exact-match","display_name":"Exact Match","sources":["crfm"]}],"derivedTags":["general","knowledge"]},{"key":"marketing","display_name":"Marketing","is_bare_stem":false,"metrics":[{"key":"exact-match","display_name":"Exact Match","sources":["crfm"]}],"derivedTags":["general","knowledge"]},{"key":"mean win rate","display_name":"Mean win rate","is_bare_stem":false,"metrics":[{"key":"rank","display_name":"Rank","sources":["crfm"]}],"derivedTags":["general","knowledge"]},{"key":"medical genetics","display_name":"Medical Genetics","is_bare_stem":false,"metrics":[{"key":"exact-match","display_name":"Exact Match","sources":["crfm"]}],"derivedTags":["general","knowledge"]},{"key":"miscellaneous","display_name":"Miscellaneous","is_bare_stem":false,"metrics":[{"key":"exact-match","display_name":"Exact Match","sources":["crfm"]}],"derivedTags":["general","knowledge"]},{"key":"mmlu","display_name":"MMLU","is_bare_stem":true,"metrics":[{"key":"exact-match","display_name":"Exact Match","sources":["crfm"]},{"key":"rank","display_name":"Rank","sources":["crfm"]}],"derivedTags":["general","knowledge"]},{"key":"mmlu all subjects","display_name":"MMLU All Subjects","is_bare_stem":false,"metrics":[{"key":"exact-match","display_name":"Exact Match","sources":["crfm"]}],"derivedTags":["general","knowledge"]},{"key":"moral scenarios","display_name":"Moral Scenarios","is_bare_stem":false,"metrics":[{"key":"exact-match","display_name":"Exact Match","sources":["crfm"]}],"derivedTags":["general","knowledge"]},{"key":"nutrition","display_name":"Nutrition","is_bare_stem":false,"metrics":[{"key":"exact-match","display_name":"Exact Match","sources":["crfm"]}],"derivedTags":["general","knowledge"]},{"key":"philosophy","display_name":"Philosophy","is_bare_stem":false,"metrics":[{"key":"exact-match","display_name":"Exact Match","sources":["crfm"]}],"derivedTags":["general","knowledge"]},{"key":"prehistory","display_name":"Prehistory","is_bare_stem":false,"metrics":[{"key":"exact-match","display_name":"Exact Match","sources":["crfm"]}],"derivedTags":["general","knowledge"]},{"key":"professional psychology","display_name":"Professional Psychology","is_bare_stem":false,"metrics":[{"key":"exact-match","display_name":"Exact Match","sources":["crfm"]}],"derivedTags":["general","knowledge"]},{"key":"public relations","display_name":"Public Relations","is_bare_stem":false,"metrics":[{"key":"exact-match","display_name":"Exact Match","sources":["crfm"]}],"derivedTags":["general","knowledge"]},{"key":"security studies","display_name":"Security Studies","is_bare_stem":false,"metrics":[{"key":"exact-match","display_name":"Exact Match","sources":["crfm"]}],"derivedTags":["general","knowledge"]},{"key":"sociology","display_name":"Sociology","is_bare_stem":false,"metrics":[{"key":"exact-match","display_name":"Exact Match","sources":["crfm"]}],"derivedTags":["general","knowledge"]},{"key":"us foreign policy","display_name":"Us Foreign Policy","is_bare_stem":false,"metrics":[{"key":"exact-match","display_name":"Exact Match","sources":["crfm"]}],"derivedTags":["general","knowledge"]},{"key":"virology","display_name":"Virology","is_bare_stem":false,"metrics":[{"key":"exact-match","display_name":"Exact Match","sources":["crfm"]}],"derivedTags":["general","knowledge"]},{"key":"world religions","display_name":"World Religions","is_bare_stem":false,"metrics":[{"key":"exact-match","display_name":"Exact Match","sources":["crfm"]}],"derivedTags":["general","knowledge"]}],"summary_eval_ids":["helm-mmlu%2Fmmlu"],"reproducibility_summary":{"results_total":63,"has_reproducibility_gap_count":63,"populated_ratio_avg":0.9285714285714283},"provenance_summary":{"total_results":63,"total_groups":63,"multi_source_groups":3,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":63,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":63,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["general","knowledge"]}],"reproducibility_summary":{"results_total":63,"has_reproducibility_gap_count":63,"populated_ratio_avg":0.9285714285714283},"provenance_summary":{"total_results":63,"total_groups":63,"multi_source_groups":3,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":63,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":63,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["general","knowledge"]},{"key":"helm-safety","display_name":"HELM Safety","category":"Safety","tags":{"domains":["AI safety","Not specified","automated red teaming","critical safety risks","large language model safety","robust refusal"],"languages":["Chinese","English","Not specified"],"tasks":["Assessing model compliance with safe prompts","Assessing model refusal of unsafe prompts","Assessing the robustness of LLM refusal mechanisms against harmful behaviors","Evaluating automated red teaming methods","Not specified","Responding to prompts","Text generation in response to sensitive prompts"]},"evals_count":6,"benchmarks":[{"key":"anthropic-red-team","display_name":"Anthropic Red Team","family_id":"anthropic-red-team","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["Not specified"],"languages":["Not specified"],"tasks":["Not specified"]},"metrics":[{"key":"score","display_name":"Score","sources":["crfm"],"models_count":74,"is_primary":true}],"slices":[],"summary_eval_ids":["helm-safety%2Fanthropic-red-team"],"reproducibility_summary":{"results_total":74,"has_reproducibility_gap_count":74,"populated_ratio_avg":0.9285714285714293},"provenance_summary":{"total_results":74,"total_groups":74,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":74,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":74,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["safety"]},{"key":"bbq","display_name":"BBQ","family_id":"bbq","is_slice":false,"is_overall":false,"primary_metric_key":"accuracy","has_card":true,"tags":{"domains":["Not specified"],"languages":["Chinese","English"],"tasks":["Not specified"]},"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["crfm"],"models_count":74,"is_primary":true}],"slices":[{"key":"bbq","display_name":"BBQ","is_bare_stem":true,"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["crfm"]}],"derivedTags":["safety","commonsense_reasoning"]}],"summary_eval_ids":["helm-safety%2Fbbq"],"reproducibility_summary":{"results_total":74,"has_reproducibility_gap_count":74,"populated_ratio_avg":0.9285714285714293},"provenance_summary":{"total_results":74,"total_groups":74,"multi_source_groups":4,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":74,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":74,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["safety","commonsense_reasoning"]},{"key":"harmbench","display_name":"HarmBench","family_id":"harmbench","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["automated red teaming","robust refusal","large language model safety"],"languages":["English"],"tasks":["Evaluating automated red teaming methods","Assessing the robustness of LLM refusal mechanisms against harmful behaviors"]},"metrics":[{"key":"score","display_name":"Score","sources":["crfm"],"models_count":74,"is_primary":true}],"slices":[],"summary_eval_ids":["helm-safety%2Fharmbench"],"reproducibility_summary":{"results_total":74,"has_reproducibility_gap_count":74,"populated_ratio_avg":0.9285714285714293},"provenance_summary":{"total_results":74,"total_groups":74,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":74,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":74,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["safety"]},{"key":"helm-safety","display_name":"HELM Safety","family_id":"helm-safety","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["crfm"],"models_count":74,"is_primary":true}],"slices":[],"summary_eval_ids":["helm-safety%2Fhelm-safety"],"reproducibility_summary":{"results_total":74,"has_reproducibility_gap_count":74,"populated_ratio_avg":0.10714285714285698},"provenance_summary":{"total_results":74,"total_groups":74,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":74,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":74,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["general"]},{"key":"simplesafetytests","display_name":"SimpleSafetyTests","family_id":"simplesafetytests","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["AI safety","critical safety risks"],"languages":["English"],"tasks":["Text generation in response to sensitive prompts"]},"metrics":[{"key":"score","display_name":"Score","sources":["crfm"],"models_count":74,"is_primary":true}],"slices":[],"summary_eval_ids":["helm-safety%2Fsimplesafetytests"],"reproducibility_summary":{"results_total":74,"has_reproducibility_gap_count":74,"populated_ratio_avg":0.9285714285714293},"provenance_summary":{"total_results":74,"total_groups":74,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":74,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":74,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["safety"]},{"key":"xstest","display_name":"Xstest","family_id":"xstest","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["Not specified"],"languages":["English"],"tasks":["Responding to prompts","Assessing model compliance with safe prompts","Assessing model refusal of unsafe prompts"]},"metrics":[{"key":"score","display_name":"Score","sources":["crfm"],"models_count":74,"is_primary":true}],"slices":[{"key":"xstest","display_name":"XSTest","is_bare_stem":true,"metrics":[{"key":"score","display_name":"Score","sources":["crfm"]}],"derivedTags":["safety"]}],"summary_eval_ids":["helm-safety%2Fxstest"],"reproducibility_summary":{"results_total":74,"has_reproducibility_gap_count":74,"populated_ratio_avg":0.9285714285714293},"provenance_summary":{"total_results":74,"total_groups":74,"multi_source_groups":2,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":74,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":74,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["safety"]}],"reproducibility_summary":{"results_total":444,"has_reproducibility_gap_count":444,"populated_ratio_avg":0.7916666666666672},"provenance_summary":{"total_results":444,"total_groups":444,"multi_source_groups":6,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":444,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":444,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["general","safety","commonsense_reasoning"]}],"derivedTags":["general","safety","natural_sciences","mathematics","commonsense_reasoning","linguistic_core","applied_reasoning","knowledge","hallucination","law"]},{"key":"hf-open-llm-v2","display_name":"HF Open LLM v2","category":"Reasoning","tags":{"domains":["STEM","algorithmic reasoning","biology","business","chemistry","commonsense reasoning","computer science","economics","engineering","explanation generation","health","history","humanities","language understanding","law","logical reasoning","math","mathematics","natural language processing","philosophy","physics","planning","psychology","question-answering","reasoning","social sciences","symbolic manipulation"],"languages":["English","Not specified"],"tasks":["Boolean expressions","Causal judgement","Date understanding","Disambiguation QA","Dyck languages","Final answer generation","Following verifiable instructions","Formal fallacies","Geometric shapes","Hyperbaton","Logical deduction","Mathematical problem solving","Movie recommendation","Multi-step arithmetic","Multiple-choice question answering","Navigate","Object counting","Penguins in a table","Question answering","Reasoning about colored objects","Ruin names","Salient translation error detection","Snarks","Solving murder mysteries","Solving object placement problems","Solving team allocation problems","Sports understanding","Step-by-step solution generation","Temporal sequences","Text generation","Tracking shuffled objects","Web of lies","Word sorting"]},"evals_count":6,"eval_summary_ids":["hfopenllm-v2%2Fbbh","hfopenllm-v2%2Fgpqa","hfopenllm-v2%2Fifeval","hfopenllm-v2%2Fmath-level-5","hfopenllm-v2%2Fmmlu-pro","hfopenllm-v2%2Fmusr"],"reproducibility_summary":{"results_total":26484,"has_reproducibility_gap_count":26484,"populated_ratio_avg":0.9285714285714003},"provenance_summary":{"total_results":26484,"total_groups":26484,"multi_source_groups":170,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":26484,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":26484,"groups_with_variant_check":3,"groups_with_cross_party_check":12,"variant_divergent_count":3,"cross_party_divergent_count":9},"benchmarks":[],"composites":[{"key":"hf-open-llm-v2","display_name":"HF Open LLM v2","category":"Reasoning","tags":{"domains":[],"languages":[],"tasks":[]},"benchmarks":[{"key":"bbh","display_name":"BBH","family_id":"bbh","is_slice":false,"is_overall":false,"primary_metric_key":"accuracy","has_card":true,"tags":{"domains":["natural language processing","algorithmic reasoning","mathematics","commonsense reasoning","question-answering","logical reasoning","language understanding","symbolic manipulation"],"languages":["Not specified"],"tasks":["Boolean expressions","Causal judgement","Date understanding","Disambiguation QA","Dyck languages","Formal fallacies","Geometric shapes","Hyperbaton","Logical deduction","Movie recommendation","Multi-step arithmetic","Navigate","Object counting","Penguins in a table","Reasoning about colored objects","Ruin names","Salient translation error detection","Snarks","Sports understanding","Temporal sequences","Tracking shuffled objects","Web of lies","Word sorting"]},"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["Hugging Face"],"models_count":4414,"is_primary":true}],"slices":[{"key":"bbh","display_name":"BBH","is_bare_stem":true,"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["Hugging Face"]}],"derivedTags":["logical_reasoning","applied_reasoning"]}],"summary_eval_ids":["hfopenllm-v2%2Fbbh"],"reproducibility_summary":{"results_total":4414,"has_reproducibility_gap_count":4414,"populated_ratio_avg":0.9285714285714058},"provenance_summary":{"total_results":4414,"total_groups":4414,"multi_source_groups":4,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":4414,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":4414,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":true,"derivedTags":["logical_reasoning","applied_reasoning"]},{"key":"gpqa","display_name":"GPQA","family_id":"gpqa","is_slice":false,"is_overall":false,"primary_metric_key":"accuracy","has_card":true,"tags":{"domains":["biology","physics","chemistry"],"languages":["English"],"tasks":["Multiple-choice question answering"]},"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["Hugging Face"],"models_count":4414,"is_primary":true}],"slices":[{"key":"gpqa","display_name":"GPQA","is_bare_stem":true,"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["Hugging Face"]}],"derivedTags":["natural_sciences"]}],"summary_eval_ids":["hfopenllm-v2%2Fgpqa"],"reproducibility_summary":{"results_total":4414,"has_reproducibility_gap_count":4414,"populated_ratio_avg":0.9285714285713926},"provenance_summary":{"total_results":4414,"total_groups":4414,"multi_source_groups":56,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":4414,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":4414,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["natural_sciences"]},{"key":"ifeval","display_name":"IFEval","family_id":"ifeval","is_slice":false,"is_overall":false,"primary_metric_key":"accuracy","has_card":true,"tags":{"domains":[],"languages":["English"],"tasks":["Text generation","Following verifiable instructions"]},"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["Hugging Face"],"models_count":4414,"is_primary":true}],"slices":[{"key":"ifeval","display_name":"IFEval","is_bare_stem":true,"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["Hugging Face"]}],"derivedTags":["linguistic_core"]}],"summary_eval_ids":["hfopenllm-v2%2Fifeval"],"reproducibility_summary":{"results_total":4414,"has_reproducibility_gap_count":4414,"populated_ratio_avg":0.928571428571391},"provenance_summary":{"total_results":4414,"total_groups":4414,"multi_source_groups":34,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":4414,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":4414,"groups_with_variant_check":0,"groups_with_cross_party_check":9,"variant_divergent_count":0,"cross_party_divergent_count":6},"is_primary":false,"derivedTags":["linguistic_core"]},{"key":"math-level-5","display_name":"MATH Level 5","family_id":"math","is_slice":false,"is_overall":false,"primary_metric_key":"exact-match","has_card":true,"tags":{"domains":["mathematics","explanation generation"],"languages":["English"],"tasks":["Mathematical problem solving","Step-by-step solution generation","Final answer generation"]},"metrics":[{"key":"exact-match","display_name":"Exact Match","sources":["Hugging Face"],"models_count":4414,"is_primary":true}],"slices":[],"summary_eval_ids":["hfopenllm-v2%2Fmath-level-5"],"reproducibility_summary":{"results_total":4414,"has_reproducibility_gap_count":4414,"populated_ratio_avg":0.928571428571411},"provenance_summary":{"total_results":4414,"total_groups":4414,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":4414,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":4414,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["general"]},{"key":"mmlu-pro","display_name":"MMLU-Pro","family_id":"mmlu","is_slice":false,"is_overall":false,"primary_metric_key":"accuracy","has_card":true,"tags":{"domains":["STEM","humanities","social sciences","math","physics","chemistry","law","engineering","economics","health","psychology","business","biology","philosophy","computer science","history"],"languages":["English"],"tasks":["Multiple-choice question answering"]},"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["Hugging Face"],"models_count":4414,"is_primary":true}],"slices":[{"key":"mmlu-pro","display_name":"MMLU-PRO","is_bare_stem":true,"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["Hugging Face"]}],"derivedTags":["general","knowledge"]}],"summary_eval_ids":["hfopenllm-v2%2Fmmlu-pro"],"reproducibility_summary":{"results_total":4414,"has_reproducibility_gap_count":4414,"populated_ratio_avg":0.9285714285714015},"provenance_summary":{"total_results":4414,"total_groups":4414,"multi_source_groups":75,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":4414,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":4414,"groups_with_variant_check":3,"groups_with_cross_party_check":3,"variant_divergent_count":3,"cross_party_divergent_count":3},"is_primary":false,"derivedTags":["general","knowledge"]},{"key":"musr","display_name":"MuSR","family_id":"musr","is_slice":false,"is_overall":false,"primary_metric_key":"accuracy","has_card":true,"tags":{"domains":["reasoning","commonsense reasoning","planning"],"languages":["English"],"tasks":["Question answering","Solving murder mysteries","Solving object placement problems","Solving team allocation problems"]},"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["Hugging Face"],"models_count":4414,"is_primary":true}],"slices":[{"key":"musr","display_name":"MUSR","is_bare_stem":true,"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["Hugging Face"]}],"derivedTags":["commonsense_reasoning","applied_reasoning"]}],"summary_eval_ids":["hfopenllm-v2%2Fmusr"],"reproducibility_summary":{"results_total":4414,"has_reproducibility_gap_count":4414,"populated_ratio_avg":0.9285714285713988},"provenance_summary":{"total_results":4414,"total_groups":4414,"multi_source_groups":1,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":4414,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":4414,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["commonsense_reasoning","applied_reasoning"]}],"derivedTags":["general","logical_reasoning","applied_reasoning","natural_sciences","linguistic_core","knowledge","commonsense_reasoning"]}],"standalone_benchmarks":[],"derivedTags":["general","logical_reasoning","applied_reasoning","natural_sciences","linguistic_core","knowledge","commonsense_reasoning"]},{"key":"hle","display_name":"HLE","category":"General","tags":{"domains":[],"languages":[],"tasks":[]},"evals_count":2,"eval_summary_ids":["hle%2FHumanity%27s%20Last%20Exam%20%28accuracy%29","hle%2FHumanity%27s%20Last%20Exam%20%28calibration%20error%29"],"reproducibility_summary":{"results_total":94,"has_reproducibility_gap_count":94,"populated_ratio_avg":0.10714285714285711},"provenance_summary":{"total_results":94,"total_groups":94,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":94,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":94,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"benchmarks":[],"standalone_benchmarks":[{"key":"humanity-s-last-exam","display_name":"Humanity's Last Exam","tags":{"domains":[],"languages":[],"tasks":[]},"summary_eval_ids":["hle%2FHumanity%27s%20Last%20Exam%20%28accuracy%29","hle%2FHumanity%27s%20Last%20Exam%20%28calibration%20error%29"],"slices":[{"key":"humanity-s-last-exam","display_name":"Humanity's Last Exam","slice_key":null,"is_bare_stem":true,"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["Scale"],"models_count":47,"is_primary":true},{"key":"calibration-error","display_name":"Calibration Error","sources":["Scale"],"models_count":47,"is_primary":true}],"derivedTags":["general"]}],"derivedTags":["general"]}],"composites":[],"derivedTags":["general"]},{"key":"ifeval","display_name":"IFEval","category":"General","tags":{"domains":[],"languages":["English"],"tasks":["Following verifiable instructions","Text generation"]},"evals_count":10,"eval_summary_ids":["ifeval%2Fifeval"],"reproducibility_summary":{"results_total":15,"has_reproducibility_gap_count":15,"populated_ratio_avg":0.9285714285714287},"provenance_summary":{"total_results":15,"total_groups":15,"multi_source_groups":9,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":15,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":15,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"standalone_benchmarks":[{"key":"ifeval","display_name":"IFEval","family_id":"ifeval","is_slice":false,"is_overall":true,"primary_metric_key":"final_acc","has_card":true,"tags":{"domains":[],"languages":["English"],"tasks":["Text generation","Following verifiable instructions"]},"metrics":[{"key":"final_acc","display_name":"final_acc","sources":["Arcadia Impact"],"models_count":15,"is_primary":true},{"key":"final_stderr","display_name":"final_stderr","sources":["Arcadia Impact"],"models_count":15,"is_primary":false},{"key":"inst_loose_acc","display_name":"inst_loose_acc","sources":["Arcadia Impact"],"models_count":15,"is_primary":false},{"key":"inst_loose_stderr","display_name":"inst_loose_stderr","sources":["Arcadia Impact"],"models_count":15,"is_primary":false},{"key":"inst_strict_acc","display_name":"inst_strict_acc","sources":["Arcadia Impact"],"models_count":15,"is_primary":false},{"key":"inst_strict_stderr","display_name":"inst_strict_stderr","sources":["Arcadia Impact"],"models_count":15,"is_primary":false},{"key":"prompt_loose_acc","display_name":"prompt_loose_acc","sources":["Arcadia Impact"],"models_count":15,"is_primary":false},{"key":"prompt_loose_stderr","display_name":"prompt_loose_stderr","sources":["Arcadia Impact"],"models_count":15,"is_primary":false},{"key":"prompt_strict_acc","display_name":"prompt_strict_acc","sources":["Arcadia Impact"],"models_count":15,"is_primary":false},{"key":"prompt_strict_stderr","display_name":"prompt_strict_stderr","sources":["Arcadia Impact"],"models_count":15,"is_primary":false}],"slices":[{"key":"final","display_name":"final","is_bare_stem":false,"metrics":[{"key":"final_acc","display_name":"final_acc","sources":["Arcadia Impact"]},{"key":"final_stderr","display_name":"final_stderr","sources":["Arcadia Impact"]}],"derivedTags":["linguistic_core"]},{"key":"inst_loose","display_name":"inst_loose","is_bare_stem":false,"metrics":[{"key":"inst_loose_acc","display_name":"inst_loose_acc","sources":["Arcadia Impact"]},{"key":"inst_loose_stderr","display_name":"inst_loose_stderr","sources":["Arcadia Impact"]}],"derivedTags":["linguistic_core"]},{"key":"inst_strict","display_name":"inst_strict","is_bare_stem":false,"metrics":[{"key":"inst_strict_acc","display_name":"inst_strict_acc","sources":["Arcadia Impact"]},{"key":"inst_strict_stderr","display_name":"inst_strict_stderr","sources":["Arcadia Impact"]}],"derivedTags":["linguistic_core"]},{"key":"prompt_loose","display_name":"prompt_loose","is_bare_stem":false,"metrics":[{"key":"prompt_loose_acc","display_name":"prompt_loose_acc","sources":["Arcadia Impact"]},{"key":"prompt_loose_stderr","display_name":"prompt_loose_stderr","sources":["Arcadia Impact"]}],"derivedTags":["linguistic_core"]},{"key":"prompt_strict","display_name":"prompt_strict","is_bare_stem":false,"metrics":[{"key":"prompt_strict_acc","display_name":"prompt_strict_acc","sources":["Arcadia Impact"]},{"key":"prompt_strict_stderr","display_name":"prompt_strict_stderr","sources":["Arcadia Impact"]}],"derivedTags":["linguistic_core"]}],"summary_eval_ids":["ifeval%2Fifeval"],"reproducibility_summary":{"results_total":15,"has_reproducibility_gap_count":15,"populated_ratio_avg":0.9285714285714287},"provenance_summary":{"total_results":15,"total_groups":15,"multi_source_groups":9,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":15,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":15,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":true,"derivedTags":["linguistic_core"]}],"derivedTags":["linguistic_core"]},{"key":"la-leaderboard","display_name":"LA Leaderboard","category":"Safety","tags":{"domains":["clinical","knowledge","language proficiency","miscellaneous","press","reasoning","science","social biases"],"languages":["Languages of Spain and Latin America","Spanish"],"tasks":["Abstractive question answering","Common sense reasoning","Extractive question answering","Linguistic acceptability","Long form question answering","News summarization","Paraphrasing","Primary school maths","Reading comprehension","Text classification"]},"evals_count":1,"eval_summary_ids":["la-leaderboard%2Fla-leaderboard"],"reproducibility_summary":{"results_total":4,"has_reproducibility_gap_count":4,"populated_ratio_avg":0.9285714285714286},"provenance_summary":{"total_results":4,"total_groups":4,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":4,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":4,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"standalone_benchmarks":[{"key":"la-leaderboard","display_name":"LA Leaderboard","family_id":"la-leaderboard","is_slice":false,"is_overall":true,"primary_metric_key":"score","has_card":true,"tags":{"domains":["language proficiency","knowledge","reasoning","social biases","science","press","clinical","miscellaneous"],"languages":["Spanish","Languages of Spain and Latin America"],"tasks":["Abstractive question answering","Extractive question answering","News summarization","Paraphrasing","Text classification","Common sense reasoning","Primary school maths","Reading comprehension","Linguistic acceptability","Long form question answering"]},"metrics":[{"key":"score","display_name":"Score","sources":["La Leaderboard"],"models_count":4,"is_primary":true}],"slices":[],"summary_eval_ids":["la-leaderboard%2Fla-leaderboard"],"reproducibility_summary":{"results_total":4,"has_reproducibility_gap_count":4,"populated_ratio_avg":0.9285714285714286},"provenance_summary":{"total_results":4,"total_groups":4,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":4,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":4,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":true,"derivedTags":["general"]}],"derivedTags":["general"]},{"key":"live-bench","display_name":"LiveBench","category":"General","tags":{"domains":[],"languages":[],"tasks":[]},"evals_count":11,"eval_summary_ids":["live-bench%2Flivebench","live-bench%2Flivebench-coding","live-bench%2Flivebench-instruction-following","live-bench%2Flivebench-language"],"reproducibility_summary":{"results_total":523,"has_reproducibility_gap_count":523,"populated_ratio_avg":0.10714285714285728},"provenance_summary":{"total_results":523,"total_groups":523,"multi_source_groups":5,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":523,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":523,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"benchmarks":[{"key":"livebench","display_name":"LiveBench","family_id":"livebench","is_slice":false,"is_overall":false,"primary_metric_key":"accuracy","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["LiveBench"],"models_count":137,"is_primary":true}],"slices":[{"key":"livebench/instruction_following","display_name":"livebench/instruction_following","is_bare_stem":false,"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["LiveBench"]}],"derivedTags":["general"]},{"key":"livebench/instruction_following/paraphrase","display_name":"livebench/instruction_following/paraphrase","is_bare_stem":false,"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["LiveBench"]}],"derivedTags":["general"]},{"key":"livebench/overall","display_name":"livebench/overall","is_bare_stem":false,"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["LiveBench"]}],"derivedTags":["general"]}],"summary_eval_ids":["live-bench%2Flivebench"],"reproducibility_summary":{"results_total":137,"has_reproducibility_gap_count":137,"populated_ratio_avg":0.10714285714285729},"provenance_summary":{"total_results":137,"total_groups":137,"multi_source_groups":5,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":137,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":137,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":true,"derivedTags":["general"]},{"key":"livebench-coding","display_name":"LiveBench Coding","family_id":"livebench","is_slice":false,"is_overall":false,"primary_metric_key":"accuracy","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["LiveBench"],"models_count":130,"is_primary":true}],"slices":[{"key":"livebench-coding-coding-completion","display_name":"LiveBench Coding — Coding Completion","is_bare_stem":false,"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["LiveBench"]}],"derivedTags":["general"]},{"key":"livebench-coding-lcb-generation","display_name":"LiveBench Coding — LCB Generation","is_bare_stem":false,"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["LiveBench"]}],"derivedTags":["general"]},{"key":"livebench/coding","display_name":"livebench/coding","is_bare_stem":false,"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["LiveBench"]}],"derivedTags":["general"]},{"key":"livebench/coding/coding_completion","display_name":"livebench/coding/coding_completion","is_bare_stem":false,"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["LiveBench"]}],"derivedTags":["general"]},{"key":"livebench/language","display_name":"livebench/language","is_bare_stem":false,"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["LiveBench"]}],"derivedTags":["general"]}],"summary_eval_ids":["live-bench%2Flivebench-coding"],"reproducibility_summary":{"results_total":130,"has_reproducibility_gap_count":130,"populated_ratio_avg":0.10714285714285726},"provenance_summary":{"total_results":130,"total_groups":130,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":130,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":130,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["general"]},{"key":"livebench-instruction-following","display_name":"LiveBench Instruction Following","family_id":"livebench","is_slice":false,"is_overall":false,"primary_metric_key":"accuracy","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["LiveBench"],"models_count":124,"is_primary":true}],"slices":[{"key":"livebench-instruction-following-paraphrase","display_name":"LiveBench Instruction Following — Paraphrase","is_bare_stem":false,"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["LiveBench"]}],"derivedTags":["general"]},{"key":"livebench-instruction-following-story-generation","display_name":"LiveBench Instruction Following — Story Generation","is_bare_stem":false,"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["LiveBench"]}],"derivedTags":["general"]},{"key":"livebench/instruction_following","display_name":"livebench/instruction_following","is_bare_stem":false,"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["LiveBench"]}],"derivedTags":["general"]}],"summary_eval_ids":["live-bench%2Flivebench-instruction-following"],"reproducibility_summary":{"results_total":124,"has_reproducibility_gap_count":124,"populated_ratio_avg":0.10714285714285725},"provenance_summary":{"total_results":124,"total_groups":124,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":124,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":124,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["general"]},{"key":"livebench-language","display_name":"LiveBench Language","family_id":"livebench","is_slice":false,"is_overall":false,"primary_metric_key":"accuracy","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["LiveBench"],"models_count":132,"is_primary":true}],"slices":[{"key":"livebench-language-connections","display_name":"LiveBench Language — Connections","is_bare_stem":false,"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["LiveBench"]}],"derivedTags":["general"]},{"key":"livebench-language-plot-unscrambling","display_name":"LiveBench Language — Plot Unscrambling","is_bare_stem":false,"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["LiveBench"]}],"derivedTags":["general"]},{"key":"livebench-language-typos","display_name":"LiveBench Language — Typos","is_bare_stem":false,"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["LiveBench"]}],"derivedTags":["general"]},{"key":"livebench/coding","display_name":"livebench/coding","is_bare_stem":false,"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["LiveBench"]}],"derivedTags":["general"]},{"key":"livebench/instruction_following/story_generation","display_name":"livebench/instruction_following/story_generation","is_bare_stem":false,"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["LiveBench"]}],"derivedTags":["general"]},{"key":"livebench/language","display_name":"livebench/language","is_bare_stem":false,"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["LiveBench"]}],"derivedTags":["general"]}],"summary_eval_ids":["live-bench%2Flivebench-language"],"reproducibility_summary":{"results_total":132,"has_reproducibility_gap_count":132,"populated_ratio_avg":0.10714285714285728},"provenance_summary":{"total_results":132,"total_groups":132,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":132,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":132,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["general"]}],"derivedTags":["general"]},{"key":"livecodebenchpro","display_name":"Live Code Bench Pro","category":"Reasoning","tags":{"domains":["code generation","programming competitions"],"languages":["Not specified"],"tasks":["Code execution","Code generation","Self-repair","Test output prediction"]},"evals_count":1,"eval_summary_ids":["livecodebenchpro%2Flivecodebench-pro"],"reproducibility_summary":{"results_total":25,"has_reproducibility_gap_count":25,"populated_ratio_avg":0.9285714285714282},"provenance_summary":{"total_results":25,"total_groups":25,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":25,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":25,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"standalone_benchmarks":[{"key":"livecodebench-pro","display_name":"LiveCodeBench Pro","family_id":"livecodebench","is_slice":false,"is_overall":false,"primary_metric_key":"pass-at-1","has_card":true,"tags":{"domains":["code generation","programming competitions"],"languages":["Not specified"],"tasks":["Code generation","Self-repair","Code execution","Test output prediction"]},"metrics":[{"key":"pass-at-1","display_name":"Pass@1","sources":["New York University,  Princeton University, University of California San Diego, University of Washington and Canyon Crest Academy","New York University, Princeton University, University of California San Diego, University of Washington and Canyon Crest Academy"],"models_count":25,"is_primary":true}],"slices":[{"key":"easy problems","display_name":"Easy Problems","is_bare_stem":false,"metrics":[{"key":"pass-at-1","display_name":"Pass@1","sources":["New York University, Princeton University, University of California San Diego, University of Washington and Canyon Crest Academy","New York University,  Princeton University, University of California San Diego, University of Washington and Canyon Crest Academy"]}],"derivedTags":["software_engineering"]},{"key":"hard problems","display_name":"Hard Problems","is_bare_stem":false,"metrics":[{"key":"pass-at-1","display_name":"Pass@1","sources":["New York University,  Princeton University, University of California San Diego, University of Washington and Canyon Crest Academy","New York University, Princeton University, University of California San Diego, University of Washington and Canyon Crest Academy"]}],"derivedTags":["software_engineering"]},{"key":"medium problems","display_name":"Medium Problems","is_bare_stem":false,"metrics":[{"key":"pass-at-1","display_name":"Pass@1","sources":["New York University,  Princeton University, University of California San Diego, University of Washington and Canyon Crest Academy","New York University, Princeton University, University of California San Diego, University of Washington and Canyon Crest Academy"]}],"derivedTags":["software_engineering"]}],"summary_eval_ids":["livecodebenchpro%2Flivecodebench-pro"],"reproducibility_summary":{"results_total":25,"has_reproducibility_gap_count":25,"populated_ratio_avg":0.9285714285714282},"provenance_summary":{"total_results":25,"total_groups":25,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":25,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":25,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":true,"derivedTags":["software_engineering"]}],"derivedTags":["software_engineering"]},{"key":"llm-stats","display_name":"LLM Stats","category":"General","tags":{"domains":["3D Design","AI safety","API interactions","APIs","Advanced Chinese Understanding","Algebra","B2B services","Browser Automation","Combinatorics","Document Analysis and Recognition","Document-Oriented VQA","Film & TV","Financial Analysis","Fundamental Language Ability","Geometry","Handwritten Mathematical Expression Recognition (HMER)","History","Key Information Extraction (KIE)","Location Navigation","Logical Reasoning","Long Context","Mathematics","Medicine","Not specified","Number Theory","Open-ended Questions","Professional Knowledge","Repository Management","STEM","Scene Text-Centric Visual Question Answering (VQA)","Task-oriented Role Play","Technology","Telecom","Text Recognition","US history","Web Searching","Writing Ability","abstract algebra","abstraction and reasoning","accounting","agent","airlines","algorithmic problem-solving","algorithmic reasoning","anatomy","and 27 other domains","and many other professional fields","art studies","artificial general intelligence","astronomy","authentication","autonomous agents","back-end systems","binary exploitation","biology","business","business applications","business ethics","business management","chart interpretation","chemistry","civil service exams","clinical knowledge","code editing","code generation","code issue resolution","code reverse engineering","coding competitions","college entrance exams","college-level sciences","commonsense reasoning","computer programming","computer science","computer security","computer task automation","computer terminal environments","computer vision","coreference resolution","corporate law","cryptography","cybersecurity","daily applications (Chrome, VLC, Thunderbird)","data analysis","data visualization","data visualization analysis","database interfaces","databases","desktop applications","developer tools","diagram understanding","diagramming","document understanding","econometrics","economic decision-making","economics","education","egocentric video analysis","electrical engineering","elementary mathematics","email clients","embodied navigation","emotional intelligence","engineering","explanation generation","figurative language understanding","file management","file system operations","finance","financial analysis","financial question answering","financial research","forensics","formal logic","function calling","general computer use","geography","global facts","government and politics","government services","grade school mathematics","grade-school science","health","healthcare","high school subjects","history","human activity and behavior recognition","human aging","human sexuality","human-computer interaction","humanities","image classification","image editing","image-to-text","imo","information retrieval","interactive mapping","international law","investment banking","jurisprudence","language understanding","law","law school admission","lawyer qualification","literature","llm evaluation","logical fallacies","logical reasoning","long-context reasoning","long-form video understanding","long-term task execution","machine learning","machine translation","machine translation evaluation","management","management consulting","manufacturing engineering","marketing","materials science","math","math competitions","math word problems","mathematical problem-solving","mathematical reasoning","mathematics","medical genetics","medicine","model training","morality","movie","multi-app workflows","multi-application tasks","multi-modal research","multi-step reasoning","multilingual natural language processing","multimodal understanding","music","natural language generation","natural language inference","natural language processing","natural language understanding","natural sciences","network optimization","numerical reasoning","office applications (LibreOffice Calc/Writer/Impress)","office suites","open domain","operating system tasks","optimization","paragraph understanding","performance analysis","philosophy","physical commonsense reasoning","physics","planning","politics","problem solving","professional occupations","professional tools (GIMP, VS Code)","proof writing","psychology","question answering","question-answering","reading comprehension","reasoning","restaurants","retrieval","retrieval augmented generation","science","scientific computing","scientific research","signal processing","social intelligence","social sciences","software applications","software development","software engineering","speech language identification","speech recognition","speech-to-text translation","sports","standardized testing","symbolic manipulation","syntax highlighting","system administration","telecommunications","text-to-SQL","tool use","translation","travel","user interface","user interfaces","vehicle driving","video editing","vision-language integration","visual contexts","visual grounding","visual illustration interpretation","visual question answering","visual reasoning","web browsers","web browsing","web development","web domain","web exploitation","web interface design","website interaction","website understanding"],"languages":["Arabic","Bengali","Chinese","English","French","German","Hindi","Indonesian","Italian","Japanese","Korean","Not specified","Persian","Polish","Portuguese","Russian","Set1 (unspecified target languages)","Spanish","Swahili","Thai","afr","amh","ara","asm","ast","azj","bel","ben","bos","bul","cat","ceb","ces","ckb","cmn","code","cym","dan","deu","ell","eng","est","fas","fin","fra","ful","gle","glg","guj","hau","heb","hin","hrv","hun","hye","ibo","ind","isl","ita","jav","jpn","kam","kan","kat","kaz","kea","khm","kir","kor","lao","lav","lin","lit","ltz","lug","luo","mal","mar","mkd","mlt","mon","mri","msa","mya","nld","nob","npi","nso","nya","oci","orm","ory","pan","pol","por","pus","ron","rus","slk","slv","sna","snd","som","spa","srp","swe","swh","tam","tel","tgk","tgl","tha","tur","ukr","umb","urd","uzb","vie","wol","xho","yo","yor","yue","zul"],"tasks":["Action grounding","Action prediction","Advanced Chinese Understanding","Answering multiple-choice questions about social situations","Assessing model compliance with safe prompts","Assessing model refusal of unsafe prompts","Attack-Defense challenges involving system defense and attack","Automated GitHub issue resolution","Automatic speech recognition","Balancing inventories and placing orders","Binary choice fill-in-the-blank","Binary question-answering where a model must choose the correct solution to achieve a physical goal","Boolean expressions","Capture the Flag (CTF) tasks","Causal judgement","Chatting capability","Code execution","Code generation","Code generation and modification","Code generation for scientific problems","Code repository understanding","Coding problem solving","Commonsense inference","Complex computation-intensive queries","Complex financial modeling","Complex reasoning over web content","Computer terminal environment tasks","Constructing output grids corresponding to test inputs","Conversational effectiveness across multiple turns","Corpus comparison","Cybersecurity penetration testing","Date understanding","Descriptive questions examining basic chart elements","Diagram question answering","Disambiguation QA","Document-Oriented VQA","Dyck languages","Element OCR","Element grounding","Evaluating end-to-end deep-research agents by testing generated answers against ground-truth","Evaluating retriever-only systems using standard retrieval metrics like nDCG@10","Evaluation on open-ended prompts derived from real-user queries","Example tasks like training a fasttext model to meet specific requirements","Executing realistic professional work prompts requiring multi-application tool use","Executing tool-calling loops to perform various operations","Executing workflows that span multiple applications","External knowledge grounding between natural language questions and database values","Factual question answering","Few-shot evaluation","Fill-in-the-blank question answering","Fixing bugs in JavaScript software where the problem statement or unit tests contain visual elements (images)","Following language instructions to complete tasks on real-world websites","Following verifiable instructions","Formal fallacies","Four-way multiple-choice selection for event continuation","Function relevance detection","Fundamental Language Ability","Generating Python code from docstrings","Generating a final answer sequence given a math problem","Generating a proof-of-concept test to reproduce a vulnerability from a text description and codebase","Generating code solutions that pass all test cases","Generating patches to fix code from GitHub pull requests","Geometric shapes","Global integration over corpus","Handling dirty/noisy database values","Handwritten Mathematical Expression Recognition (HMER)","Heading OCR","Hyperbaton","Identifying vulnerabilities","Image classification","Image to text","Information retrieval","Instruction adherence","Interacting with MCP servers across six core domains (navigation, repository management, financial analysis, etc.)","Interacting with real web and desktop applications","Interacting with various software applications and completing workflows under different step limits (15, 50, and 100 steps)","Issue resolving","Java function calls","JavaScript function calls","Key Information Extraction (KIE)","Knowledge evaluation across subjects including mathematics, history, computer science, law, morality, and philosophy","Logical Reasoning","Logical deduction","Long in-context learning","Long structured data understanding","Long-dialogue history understanding","Long-horizon, multi-step workflows requiring interaction with multiple software applications to complete verifiable goals","Machine translation from Set1 languages to English","Managing customer relations to maximize profit over a year-long simulation","Mathematical problem solving","Mathematics","Meeting specified time and memory constraints","Movie recommendation","Multi-document QA","Multi-hop question answering","Multi-round coreference resolution","Multi-step arithmetic","Multi-step logical reasoning","Multi-turn dialogue","Multi-turn/multi-step function calling","Multimodal understanding across 32 core meta-tasks and 162 subtasks","Multiple choice","Multiple diverse tasks including abstract narrative understanding, anachronisms, analogical similarity, analytic entailment, arithmetic, and many others","Multiple function selection","Multiple-choice question answering","Multiple-choice question-answering based on long-form video content","Navigate","Not specified","Numerical methods implementation","Numerical operation execution","Object counting","Open-domain question answering","Open-ended Questions","Open-ended visual understanding based on image-text prompts","Operating a vending machine business","Operating system file I/O operations","Parallel function calls","Parallel multiple function calls","Penguins in a table","Performing operating system file I/O operations","Printing hidden flags in Jeopardy-style CTF challenges","Producing code patches that satisfy requirements and pass tests","Producing deliverables in investment banking, management consulting, and corporate law domains","Producing verifiable short answers for Olympiad problems","Professional Knowledge","Pronoun resolution","Proof writing","Question answering","Question answering requiring discrete reasoning","Question answering where the input is a chart and natural language question, and the output is an answer derived through reasoning","REST API calls","Reasoning about colored objects","Reasoning about motivation, what happens next, and emotional reactions","Reasoning questions requiring synthesis of information across complex visual elements","Resolving software issues (e.g., bug reports or feature requests)","Resolving user requests through Tool-Agent-User (TAU) interactions while adhering to domain policy","Responding to ad-hoc natural language questions about document content and structure","Responding to prompts","Retrieval","Returning the i-th instance of a specific, repeated request from within a long conversation","Ruin names","SQL efficiency optimization","SQL query generation","Salient translation error detection","Scene Text-Centric VQA","Scientific calculation","Self-repair","Setting prices and handling daily fees","Simple function calls","Single integer answer generation (0-999 range)","Single-document QA","Snarks","Solving grade school math word problems","Solving grade-school math word problems","Solving math competition problems","Solving mathematical problems with numerical solutions in the telecom domain","Solving murder mysteries","Solving object placement problems","Solving team allocation problems","Sourcing products and negotiating with suppliers","Specific scenarios involving files such as images, documents, spreadsheets, presentations, and code","Speech language identification","Speech-to-text translation","Sports understanding","Standardized exam performance evaluation","Statistical aggregation","Summarization","Syntactic parsing of diagrams","System simulation","Table question answering","Task-oriented Role Play","Tasks constructed from the representative work of industry professionals, such as designing a 3D model of a cable reel (Manufacturing Engineer) and creating a financial analysis (Financial and Investment Analyst)","Tasks scaling in difficulty across unique categories","Temporal sequences","Test output prediction","Text Recognition","Text classification","Text generation","Text generation for question answering","Text generation in response to user inputs from real conversations","Text-based computation","Text-pair classification","Text-to-SQL conversion","Tracking shuffled objects","Translation","Unit test verification","Using demonstration pairs to understand a task's nature","Visual Question Answering","Visual Question Answering on document images","Visual question answering","Web and desktop application interaction in open domains","Web browsing for information retrieval","Web captioning","Web navigation and interaction","Web of lies","Webpage question answering","Word sense disambiguation","Word sorting","Workflows spanning multiple applications following natural language instructions","Writing Ability","Yes/no question answering","Zero-shot evaluation","question answering"]},"evals_count":460,"eval_summary_ids":["llm-stats%2Faa-index","llm-stats%2Faa-lcr","llm-stats%2Facebench","llm-stats%2Factivitynet","llm-stats%2Fagieval","llm-stats%2Fai2-reasoning-challenge-arc","llm-stats%2Fai2d","llm-stats%2Faider","llm-stats%2Faider-polyglot","llm-stats%2Faider-polyglot-edit","llm-stats%2Faime","llm-stats%2Faitz-em","llm-stats%2Falignbench","llm-stats%2Falpacaeval-2-0","llm-stats%2Famc-2022-23","llm-stats%2Fandroid-control-high-em","llm-stats%2Fandroid-control-low-em","llm-stats%2Fandroidworld","llm-stats%2Fandroidworld-sr","llm-stats%2Fapex-agents","llm-stats%2Fapi-bank","llm-stats%2Farc","llm-stats%2Farc-agi","llm-stats%2Farc-c","llm-stats%2Farc-e","llm-stats%2Farena-hard","llm-stats%2Farena-hard-v2","llm-stats%2Farkitscenes","llm-stats%2Fartificial-analysis-llms","llm-stats%2Fattaq","llm-stats%2Fautologi","llm-stats%2Fbabyvision","llm-stats%2Fbbh","llm-stats%2Fbeyond-aime","llm-stats%2Fbfcl","llm-stats%2Fbfcl-v3-multiturn","llm-stats%2Fbig-bench","llm-stats%2Fbigcodebench","llm-stats%2Fbigcodebench-full","llm-stats%2Fbiolp-bench","llm-stats%2Fbird-sql-dev","llm-stats%2Fbixbench","llm-stats%2Fblink","llm-stats%2Fboolq","llm-stats%2Fbrowsecomp","llm-stats%2Fbrowsecomp-vl","llm-stats%2Fbrowsecomp-zh","llm-stats%2Fc-eval","llm-stats%2Fcbnsl","llm-stats%2Fcc-bench-v2-backend","llm-stats%2Fcc-bench-v2-frontend","llm-stats%2Fcc-ocr","llm-stats%2Fcfeval","llm-stats%2Fcharadessta","llm-stats%2Fchartqa","llm-stats%2Fcharxiv-d","llm-stats%2Fcharxiv-r","llm-stats%2Fchexpert-cxr","llm-stats%2Fclaw-eval","llm-stats%2Fcloningscenarios","llm-stats%2Fcluewsc","llm-stats%2Fcmmlu","llm-stats%2Fcnmo-2024","llm-stats%2Fcodeforces","llm-stats%2Fcollie","llm-stats%2Fcommon-voice-15","llm-stats%2Fcommonsenseqa","llm-stats%2Fcomplexfuncbench","llm-stats%2Fcorpusqa-1m","llm-stats%2Fcountbench","llm-stats%2Fcovost2","llm-stats%2Fcovost2-en-zh","llm-stats%2Fcrag","llm-stats%2Fcreative-writing-v3","llm-stats%2Fcrperelation","llm-stats%2Fcrux-o","llm-stats%2Fcruxeval-input-cot","llm-stats%2Fcruxeval-o","llm-stats%2Fcruxeval-output-cot","llm-stats%2Fcsimpleqa","llm-stats%2Fcybench","llm-stats%2Fcybergym","llm-stats%2Fcybersecurity-ctfs","llm-stats%2Fdeepsearchqa","llm-stats%2Fdermmcqa","llm-stats%2Fdesign2code","llm-stats%2Fdocvqa","llm-stats%2Fdocvqatest","llm-stats%2Fdrop","llm-stats%2Fds-arena-code","llm-stats%2Fds-fim-eval","llm-stats%2Fdynamath","llm-stats%2Feclektic","llm-stats%2Fegoschema","llm-stats%2Fembspatialbench","llm-stats%2Feq-bench","llm-stats%2Ferqa","llm-stats%2Fevalplus","llm-stats%2Ffacts-grounding","llm-stats%2Ffactscore","llm-stats%2Ffigqa","llm-stats%2Ffinance-agent","llm-stats%2Ffinqa","llm-stats%2Ffinsearchcomp-t2-t3","llm-stats%2Ffinsearchcomp-t3","llm-stats%2Fflame-vlm-code","llm-stats%2Fflenqa","llm-stats%2Ffleurs","llm-stats%2Fframes","llm-stats%2Ffrench-mmlu","llm-stats%2Ffrontiermath","llm-stats%2Ffrontierscience-research","llm-stats%2Ffullstackbench-en","llm-stats%2Ffullstackbench-zh","llm-stats%2Ffunctionalmath","llm-stats%2Fgdpval-aa","llm-stats%2Fgdpval-mm","llm-stats%2Fgenebench","llm-stats%2Fgiantsteps-tempo","llm-stats%2Fglobal-mmlu","llm-stats%2Fglobal-piqa","llm-stats%2Fgorilla-benchmark-api-bench","llm-stats%2Fgovreport","llm-stats%2Fgpqa","llm-stats%2Fgpqa-biology","llm-stats%2Fgpqa-chemistry","llm-stats%2Fgpqa-physics","llm-stats%2Fgraphwalks-bfs-128k","llm-stats%2Fgraphwalks-parents-128k","llm-stats%2Fgroundui-1k","llm-stats%2Fgsm-8k-cot","llm-stats%2Fgsm8k","llm-stats%2Fgsm8k-chat","llm-stats%2Fhallusion-bench","llm-stats%2Fhealthbench","llm-stats%2Fhellaswag","llm-stats%2Fhiddenmath","llm-stats%2Fhle","llm-stats%2Fhmmt-2025","llm-stats%2Fhmmt-feb-26","llm-stats%2Fhmmt25","llm-stats%2Fhumaneval","llm-stats%2Fhumaneval-average","llm-stats%2Fhumaneval-er","llm-stats%2Fhumaneval-mul","llm-stats%2Fhumanevalfim-average","llm-stats%2Fhypersim","llm-stats%2Fif","llm-stats%2Fifbench","llm-stats%2Fifeval","llm-stats%2Fimagemining","llm-stats%2Fimo-answerbench","llm-stats%2Finclude","llm-stats%2Finfographicsqa","llm-stats%2Finfovqa","llm-stats%2Finfovqatest","llm-stats%2Finstruct-humaneval","llm-stats%2Fintergps","llm-stats%2Finternal-api-instruction-following-hard","llm-stats%2Fipho-2025","llm-stats%2Flbpp-v2","llm-stats%2Flingoqa","llm-stats%2Flivebench","llm-stats%2Flivebench-20241125","llm-stats%2Flivecodebench","llm-stats%2Flivecodebench-01-09","llm-stats%2Fllm%20stats%20codegolf-v2","llm-stats%2Fllm%20stats%20infinitebench-en","llm-stats%2Fllm%20stats%20livecodebench-v5-24%2012-25","llm-stats%2Fllm%20stats%20mmbench-v1","llm-stats%2Fllm%20stats%20mmlu-redux-2","llm-stats%2Fllm%20stats%20omnidocbench-1","llm-stats%2Fllm-stats","llm-stats%2Flongbench-v2","llm-stats%2Flongfact-concepts","llm-stats%2Flongfact-objects","llm-stats%2Flongvideobench","llm-stats%2Flsat","llm-stats%2Flvbench","llm-stats%2Fmask","llm-stats%2Fmath","llm-stats%2Fmatharena-apex","llm-stats%2Fmathverse-mini","llm-stats%2Fmathvision","llm-stats%2Fmathvista","llm-stats%2Fmaxife","llm-stats%2Fmbpp","llm-stats%2Fmbpp-base-version","llm-stats%2Fmbpp-evalplus-base","llm-stats%2Fmbpp-pass-1","llm-stats%2Fmcp-atlas","llm-stats%2Fmcp-mark","llm-stats%2Fmcp-universe","llm-stats%2Fmedxpertqa","llm-stats%2Fmega-mlqa","llm-stats%2Fmega-tydi-qa","llm-stats%2Fmega-udpos","llm-stats%2Fmega-xcopa","llm-stats%2Fmega-xstorycloze","llm-stats%2Fmeld","llm-stats%2Fmewc","llm-stats%2Fmgsm","llm-stats%2Fmiabench","llm-stats%2Fmimic-cxr","llm-stats%2Fmle-bench-lite","llm-stats%2Fmlvu","llm-stats%2Fmlvu-m","llm-stats%2Fmm-browsercomp","llm-stats%2Fmm-clawbench","llm-stats%2Fmm-if-eval","llm-stats%2Fmm-mind2web","llm-stats%2Fmm-mt-bench","llm-stats%2Fmmau","llm-stats%2Fmmau-music","llm-stats%2Fmmau-sound","llm-stats%2Fmmau-speech","llm-stats%2Fmmbench","llm-stats%2Fmmbench-test","llm-stats%2Fmmbench-video","llm-stats%2Fmme","llm-stats%2Fmme-realworld","llm-stats%2Fmmlongbench-doc","llm-stats%2Fmmlu","llm-stats%2Fmmlu-base","llm-stats%2Fmmlu-chat","llm-stats%2Fmmlu-french","llm-stats%2Fmmlu-pro","llm-stats%2Fmmlu-prox","llm-stats%2Fmmlu-redux","llm-stats%2Fmmlu-stem","llm-stats%2Fmmmlu","llm-stats%2Fmmmu","llm-stats%2Fmmmu-val","llm-stats%2Fmmmu-validation","llm-stats%2Fmmmuval","llm-stats%2Fmmsearch","llm-stats%2Fmmsearch-plus","llm-stats%2Fmmstar","llm-stats%2Fmmt-bench","llm-stats%2Fmmvet","llm-stats%2Fmmvetgpt4turbo","llm-stats%2Fmmvu","llm-stats%2Fmobileminiwob-sr","llm-stats%2Fmotionbench","llm-stats%2Fmrcr","llm-stats%2Fmrcr-128k-2-needle","llm-stats%2Fmrcr-128k-4-needle","llm-stats%2Fmrcr-128k-8-needle","llm-stats%2Fmrcr-1m","llm-stats%2Fmrcr-1m-pointwise","llm-stats%2Fmrcr-64k-2-needle","llm-stats%2Fmrcr-64k-4-needle","llm-stats%2Fmrcr-64k-8-needle","llm-stats%2Fmrcr-v2-8-needle","llm-stats%2Fmt-bench","llm-stats%2Fmtvqa","llm-stats%2Fmuirbench","llm-stats%2Fmulti-if","llm-stats%2Fmulti-swe-bench-leaderboard","llm-stats%2Fmultilf","llm-stats%2Fmultilingual-mgsm-cot","llm-stats%2Fmultilingual-mmlu","llm-stats%2Fmultipl-e","llm-stats%2Fmusiccaps","llm-stats%2Fmusr","llm-stats%2Fmvbench","llm-stats%2Fnatural-questions","llm-stats%2Fnatural2code","llm-stats%2Fnexus","llm-stats%2Fnl2repo","llm-stats%2Fnmos","llm-stats%2Fnolima-128k","llm-stats%2Fnolima-32k","llm-stats%2Fnolima-64k","llm-stats%2Fnova-63","llm-stats%2Fnq","llm-stats%2Fnuscene","llm-stats%2Fobjectron","llm-stats%2Focrbench-v2","llm-stats%2Focrbench-v2-en","llm-stats%2Focrbench-v2-zh","llm-stats%2Foctocodingbench","llm-stats%2Fodinw","llm-stats%2Fofficeqa-pro","llm-stats%2Fojbench","llm-stats%2Folympiadbench","llm-stats%2Fomnibench","llm-stats%2Fomnibench-music","llm-stats%2Fomnigaia","llm-stats%2Fomnimath","llm-stats%2Fopen-rewrite","llm-stats%2Fopenai-mmlu","llm-stats%2Fopenai-mrcr-2-needle-128k","llm-stats%2Fopenai-mrcr-2-needle-1m","llm-stats%2Fopenai-mrcr-2-needle-256k","llm-stats%2Fopenbookqa","llm-stats%2Fopenrca","llm-stats%2Fosworld","llm-stats%2Fosworld-extended","llm-stats%2Fosworld-g","llm-stats%2Fosworld-screenshot-only","llm-stats%2Fosworld-verified","llm-stats%2Fpaperbench","llm-stats%2Fpathmcqa","llm-stats%2Fperceptiontest","llm-stats%2Fphibench","llm-stats%2Fphysicsfinals","llm-stats%2Fpinchbench","llm-stats%2Fpiqa","llm-stats%2Fpmc-vqa","llm-stats%2Fpointgrounding","llm-stats%2Fpolymath","llm-stats%2Fpolymath-en","llm-stats%2Fpope","llm-stats%2Fpopqa","llm-stats%2Fprotocolqa","llm-stats%2Fqasper","llm-stats%2Fqmsum","llm-stats%2Fqwenwebbench","llm-stats%2Frealworldqa","llm-stats%2Frefcoco-avg","llm-stats%2Frefspatialbench","llm-stats%2Frepobench","llm-stats%2Frepoqa","llm-stats%2Frobospatialhome","llm-stats%2Fruler","llm-stats%2Fruler-1000k","llm-stats%2Fruler-128k","llm-stats%2Fruler-2048k","llm-stats%2Fruler-512k","llm-stats%2Fruler-64k","llm-stats%2Fsat-math","llm-stats%2Fscicode","llm-stats%2Fscienceqa","llm-stats%2Fscienceqa-visual","llm-stats%2Fscreenspot","llm-stats%2Fseal-0","llm-stats%2Fseccodebench","llm-stats%2Fsifo","llm-stats%2Fsifo-multiturn","llm-stats%2Fsimpleqa","llm-stats%2Fsimplevqa","llm-stats%2Fskillsbench","llm-stats%2Fslakevqa","llm-stats%2Fsocial-iqa","llm-stats%2Fspider","llm-stats%2Fsquality","llm-stats%2Fstem","llm-stats%2Fsummscreenfd","llm-stats%2Fsunrgbd","llm-stats%2Fsuperglue","llm-stats%2Fsupergpqa","llm-stats%2Fswe-bench-multilingual","llm-stats%2Fswe-bench-multimodal","llm-stats%2Fswe-bench-pro","llm-stats%2Fswe-bench-verified","llm-stats%2Fswe-bench-verified-agentic-coding","llm-stats%2Fswe-bench-verified-agentless","llm-stats%2Fswe-bench-verified-multiple-attempts","llm-stats%2Fswe-lancer","llm-stats%2Fswe-lancer-ic-diamond-subset","llm-stats%2Fswe-perf","llm-stats%2Fswe-review","llm-stats%2Fswt-bench","llm-stats%2Ft2-bench","llm-stats%2Ftau-bench","llm-stats%2Ftau2-airline","llm-stats%2Ftau2-retail","llm-stats%2Ftau2-telecom","llm-stats%2Ftau3-bench","llm-stats%2Ftempcompass","llm-stats%2Fterminal-bench","llm-stats%2Fterminus","llm-stats%2Ftextvqa","llm-stats%2Ftheoremqa","llm-stats%2Ftir-bench","llm-stats%2Ftldr9-test","llm-stats%2Ftoolathlon","llm-stats%2Ftranslation-en-set1-comet22","llm-stats%2Ftranslation-en-set1-spbleu","llm-stats%2Ftranslation-set1-en-comet22","llm-stats%2Ftranslation-set1-en-spbleu","llm-stats%2Ftriviaqa","llm-stats%2Ftruthfulqa","llm-stats%2Ftydiqa","llm-stats%2Funiform-bar-exam","llm-stats%2Fusamo25","llm-stats%2Fvatex","llm-stats%2Fvcr-en-easy","llm-stats%2Fvending-bench-2","llm-stats%2Fvibe","llm-stats%2Fvibe-android","llm-stats%2Fvibe-backend","llm-stats%2Fvibe-eval","llm-stats%2Fvibe-ios","llm-stats%2Fvibe-simulation","llm-stats%2Fvibe-web","llm-stats%2Fvideo-mme","llm-stats%2Fvideo-mme-long-no-subtitles","llm-stats%2Fvideomme-w-o-sub","llm-stats%2Fvideomme-w-sub","llm-stats%2Fvideommmu","llm-stats%2Fvision2web","llm-stats%2Fvisualwebbench","llm-stats%2Fvisulogic","llm-stats%2Fvita-bench","llm-stats%2Fvlmsareblind","llm-stats%2Fvocalsound","llm-stats%2Fvoicebench-avg","llm-stats%2Fvqa-rad","llm-stats%2Fvqav2","llm-stats%2Fvqav2-test","llm-stats%2Fvqav2-val","llm-stats%2Fwe-math","llm-stats%2Fwebvoyager","llm-stats%2Fwidesearch","llm-stats%2Fwild-bench","llm-stats%2Fwinogrande","llm-stats%2Fwmdp","llm-stats%2Fwmt23","llm-stats%2Fwmt24","llm-stats%2Fworldvqa","llm-stats%2Fwritingbench","llm-stats%2Fxlsum-english","llm-stats%2Fxstest","llm-stats%2Fzclawbench","llm-stats%2Fzebralogic","llm-stats%2Fzerobench","llm-stats%2Fzerobench-sub"],"reproducibility_summary":{"results_total":2327,"has_reproducibility_gap_count":2327,"populated_ratio_avg":0.5353305911965139},"provenance_summary":{"total_results":2327,"total_groups":2327,"multi_source_groups":187,"first_party_only_groups":2140,"source_type_distribution":{"first_party":2327,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":2327,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"benchmarks":[{"key":"aa-index","display_name":"Aa Index","family_id":"aa-index","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":3,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Faa-index"],"reproducibility_summary":{"results_total":3,"has_reproducibility_gap_count":3,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":3,"total_groups":3,"multi_source_groups":0,"first_party_only_groups":3,"source_type_distribution":{"first_party":3,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":3,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["knowledge","general"]},{"key":"aa-lcr","display_name":"AA-LCR","family_id":"aa-lcr","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":13,"is_primary":true}],"slices":[{"key":"llm stats","display_name":"llm stats","is_bare_stem":false,"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"]}],"derivedTags":["general"]}],"summary_eval_ids":["llm-stats%2Faa-lcr"],"reproducibility_summary":{"results_total":13,"has_reproducibility_gap_count":13,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":13,"total_groups":13,"multi_source_groups":12,"first_party_only_groups":1,"source_type_distribution":{"first_party":13,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":13,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["general","applied_reasoning"]},{"key":"acebench","display_name":"ACEBench","family_id":"acebench","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Facebench"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["multimodal","agentic"]},{"key":"activitynet","display_name":"Activitynet","family_id":"activitynet","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Factivitynet"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["multimodal"]},{"key":"agieval","display_name":"AGIEval","family_id":"agieval","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["standardized testing","college entrance exams","law school admission","math competitions","lawyer qualification","civil service exams"],"languages":["English"],"tasks":["Multiple-choice question answering","Fill-in-the-blank question answering","Standardized exam performance evaluation"]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":10,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fagieval"],"reproducibility_summary":{"results_total":10,"has_reproducibility_gap_count":10,"populated_ratio_avg":0.9285714285714286},"provenance_summary":{"total_results":10,"total_groups":10,"multi_source_groups":0,"first_party_only_groups":10,"source_type_distribution":{"first_party":10,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":10,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["general","applied_reasoning"]},{"key":"ai2-reasoning-challenge-arc","display_name":"AI2 Reasoning Challenge Arc","family_id":"ai2-reasoning-challenge-arc","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fai2-reasoning-challenge-arc"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["logical_reasoning","knowledge","commonsense_reasoning"]},{"key":"ai2d","display_name":"Ai2d","family_id":"ai2d","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["computer vision","diagram understanding","visual illustration interpretation"],"languages":["Not specified"],"tasks":["Syntactic parsing of diagrams","Diagram question answering"]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":17,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fai2d"],"reproducibility_summary":{"results_total":17,"has_reproducibility_gap_count":17,"populated_ratio_avg":0.9285714285714287},"provenance_summary":{"total_results":17,"total_groups":17,"multi_source_groups":0,"first_party_only_groups":17,"source_type_distribution":{"first_party":17,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":17,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["multimodal","knowledge"]},{"key":"aider","display_name":"Aider","family_id":"aider","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["Not specified"],"languages":["Not specified"],"tasks":["Not specified"]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":4,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Faider"],"reproducibility_summary":{"results_total":4,"has_reproducibility_gap_count":4,"populated_ratio_avg":0.9285714285714286},"provenance_summary":{"total_results":4,"total_groups":4,"multi_source_groups":0,"first_party_only_groups":4,"source_type_distribution":{"first_party":4,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":4,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["software_engineering","agentic"]},{"key":"aider-polyglot","display_name":"Aider Polyglot","family_id":"aider-polyglot","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["Not specified"],"languages":["Not specified"],"tasks":["Not specified"]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":18,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Faider-polyglot"],"reproducibility_summary":{"results_total":18,"has_reproducibility_gap_count":18,"populated_ratio_avg":0.9285714285714286},"provenance_summary":{"total_results":18,"total_groups":18,"multi_source_groups":0,"first_party_only_groups":18,"source_type_distribution":{"first_party":18,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":18,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["software_engineering","linguistic_core"]},{"key":"aider-polyglot-edit","display_name":"Aider Polyglot Edit","family_id":"aider-polyglot-edit","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["Not specified"],"languages":["Not specified"],"tasks":["Not specified"]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":10,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Faider-polyglot-edit"],"reproducibility_summary":{"results_total":10,"has_reproducibility_gap_count":10,"populated_ratio_avg":0.9285714285714286},"provenance_summary":{"total_results":10,"total_groups":10,"multi_source_groups":0,"first_party_only_groups":10,"source_type_distribution":{"first_party":10,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":10,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["software_engineering","linguistic_core"]},{"key":"aime","display_name":"AIME","family_id":"aime","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["mathematics","mathematical problem-solving"],"languages":["Not specified"],"tasks":["Mathematical problem solving","Coding problem solving","Single integer answer generation (0-999 range)"]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[{"key":"aime-2024","display_name":"AIME 2024","is_bare_stem":false,"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"]}],"derivedTags":["mathematics"]},{"key":"aime-2025","display_name":"AIME 2025","is_bare_stem":false,"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"]}],"derivedTags":["mathematics"]},{"key":"aime-2026","display_name":"AIME 2026","is_bare_stem":false,"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"]}],"derivedTags":["mathematics"]},{"key":"llm stats","display_name":"llm stats","is_bare_stem":false,"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"]}],"derivedTags":["general"]}],"summary_eval_ids":["llm-stats%2Faime"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.9285714285714286},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":1,"first_party_only_groups":0,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["mathematics","general"]},{"key":"aitz-em","display_name":"Aitz Em","family_id":"aitz-em","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":3,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Faitz-em"],"reproducibility_summary":{"results_total":3,"has_reproducibility_gap_count":3,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":3,"total_groups":3,"multi_source_groups":0,"first_party_only_groups":3,"source_type_distribution":{"first_party":3,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":3,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["agentic"]},{"key":"alignbench","display_name":"Alignbench","family_id":"alignbench","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["Fundamental Language Ability","Advanced Chinese Understanding","Open-ended Questions","Writing Ability","Task-oriented Role Play","Professional Knowledge","Mathematics","Logical Reasoning"],"languages":["Chinese"],"tasks":["Fundamental Language Ability","Advanced Chinese Understanding","Open-ended Questions","Writing Ability","Task-oriented Role Play","Professional Knowledge","Mathematics","Logical Reasoning"]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":4,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Falignbench"],"reproducibility_summary":{"results_total":4,"has_reproducibility_gap_count":4,"populated_ratio_avg":0.9285714285714286},"provenance_summary":{"total_results":4,"total_groups":4,"multi_source_groups":0,"first_party_only_groups":4,"source_type_distribution":{"first_party":4,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":4,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["applied_reasoning","general"]},{"key":"alpacaeval-2-0","display_name":"AlpacaEval 2.0","family_id":"alpacaeval-2-0","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["Not specified"],"languages":["Not specified"],"tasks":["Not specified"]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":3,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Falpacaeval-2-0"],"reproducibility_summary":{"results_total":3,"has_reproducibility_gap_count":3,"populated_ratio_avg":0.9285714285714285},"provenance_summary":{"total_results":3,"total_groups":3,"multi_source_groups":0,"first_party_only_groups":3,"source_type_distribution":{"first_party":3,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":3,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["applied_reasoning","general"]},{"key":"amc-2022-23","display_name":"Amc 2022 23","family_id":"amc-2022-23","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["Not specified"],"languages":["Not specified"],"tasks":["Not specified"]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":4,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Famc-2022-23"],"reproducibility_summary":{"results_total":4,"has_reproducibility_gap_count":4,"populated_ratio_avg":0.9285714285714285},"provenance_summary":{"total_results":4,"total_groups":4,"multi_source_groups":0,"first_party_only_groups":4,"source_type_distribution":{"first_party":4,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":4,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["mathematics"]},{"key":"android-control-high-em","display_name":"Android Control High Em","family_id":"android-control-high-em","is_slice":false,"is_overall":false,"primary_metric_key":"exact-match","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"exact-match","display_name":"Exact Match","sources":["LLM Stats"],"models_count":3,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fandroid-control-high-em"],"reproducibility_summary":{"results_total":3,"has_reproducibility_gap_count":3,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":3,"total_groups":3,"multi_source_groups":0,"first_party_only_groups":3,"source_type_distribution":{"first_party":3,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":3,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["agentic","multimodal"]},{"key":"android-control-low-em","display_name":"Android Control Low Em","family_id":"android-control-low-em","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":3,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fandroid-control-low-em"],"reproducibility_summary":{"results_total":3,"has_reproducibility_gap_count":3,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":3,"total_groups":3,"multi_source_groups":0,"first_party_only_groups":3,"source_type_distribution":{"first_party":3,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":3,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["agentic","multimodal"]},{"key":"androidworld","display_name":"AndroidWorld","family_id":"androidworld","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":2,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fandroidworld"],"reproducibility_summary":{"results_total":2,"has_reproducibility_gap_count":2,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":2,"total_groups":2,"multi_source_groups":0,"first_party_only_groups":2,"source_type_distribution":{"first_party":2,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":2,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["agentic","multimodal"]},{"key":"androidworld-sr","display_name":"Androidworld Sr","family_id":"androidworld-sr","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":8,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fandroidworld-sr"],"reproducibility_summary":{"results_total":8,"has_reproducibility_gap_count":8,"populated_ratio_avg":0.10714285714285712},"provenance_summary":{"total_results":8,"total_groups":8,"multi_source_groups":0,"first_party_only_groups":8,"source_type_distribution":{"first_party":8,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":8,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["agentic","multimodal"]},{"key":"api-bank","display_name":"API-Bank","family_id":"api-bank","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fapi-bank"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["agentic"]},{"key":"arc","display_name":"Arc","family_id":"arc","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Farc"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["logical_reasoning","commonsense_reasoning"]},{"key":"arc-agi","display_name":"ARC-AGI","family_id":"arc-agi","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["artificial general intelligence","abstraction and reasoning","visual reasoning"],"languages":["Not specified"],"tasks":["Using demonstration pairs to understand a task's nature","Constructing output grids corresponding to test inputs"]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":7,"is_primary":true}],"slices":[{"key":"arc-agi-2","display_name":"ARC-AGI 2","is_bare_stem":false,"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"]}],"derivedTags":["logical_reasoning","commonsense_reasoning","general"]}],"summary_eval_ids":["llm-stats%2Farc-agi"],"reproducibility_summary":{"results_total":7,"has_reproducibility_gap_count":7,"populated_ratio_avg":0.9285714285714287},"provenance_summary":{"total_results":7,"total_groups":7,"multi_source_groups":0,"first_party_only_groups":7,"source_type_distribution":{"first_party":7,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":7,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["logical_reasoning","commonsense_reasoning","general"]},{"key":"arc-c","display_name":"Arc C","family_id":"arc-c","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["grade-school science"],"languages":["Not specified"],"tasks":["Multiple-choice question answering"]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":18,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Farc-c"],"reproducibility_summary":{"results_total":18,"has_reproducibility_gap_count":18,"populated_ratio_avg":0.9285714285714286},"provenance_summary":{"total_results":18,"total_groups":18,"multi_source_groups":0,"first_party_only_groups":18,"source_type_distribution":{"first_party":18,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":18,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["logical_reasoning","knowledge","commonsense_reasoning"]},{"key":"arc-e","display_name":"Arc E","family_id":"arc-e","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["question answering","science","reasoning"],"languages":["English"],"tasks":["Multiple-choice question answering"]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":6,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Farc-e"],"reproducibility_summary":{"results_total":6,"has_reproducibility_gap_count":6,"populated_ratio_avg":0.9285714285714287},"provenance_summary":{"total_results":6,"total_groups":6,"multi_source_groups":0,"first_party_only_groups":6,"source_type_distribution":{"first_party":6,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":6,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["logical_reasoning","knowledge","commonsense_reasoning"]},{"key":"arena-hard","display_name":"Arena Hard","family_id":"arena","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["Not specified"],"languages":["Not specified"],"tasks":["Evaluation on open-ended prompts derived from real-user queries"]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":17,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Farena-hard"],"reproducibility_summary":{"results_total":17,"has_reproducibility_gap_count":17,"populated_ratio_avg":0.9285714285714287},"provenance_summary":{"total_results":17,"total_groups":17,"multi_source_groups":0,"first_party_only_groups":17,"source_type_distribution":{"first_party":17,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":17,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["general","applied_reasoning"]},{"key":"arena-hard-v2","display_name":"Arena Hard v2","family_id":"arena","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":12,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Farena-hard-v2"],"reproducibility_summary":{"results_total":12,"has_reproducibility_gap_count":12,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":12,"total_groups":12,"multi_source_groups":0,"first_party_only_groups":12,"source_type_distribution":{"first_party":12,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":12,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["general","applied_reasoning"]},{"key":"arkitscenes","display_name":"Arkitscenes","family_id":"arkitscenes","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Farkitscenes"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["multimodal"]},{"key":"artificial-analysis-llms","display_name":"Artificial Analysis LLMs","family_id":"artificial-analysis-llms","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["Not specified"],"languages":["Not specified"],"tasks":["Not specified"]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[{"key":"llm stats","display_name":"llm stats","is_bare_stem":false,"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"]}],"derivedTags":["general"]}],"summary_eval_ids":["llm-stats%2Fartificial-analysis-llms"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.9285714285714286},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":1,"first_party_only_groups":0,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["general"]},{"key":"attaq","display_name":"Attaq","family_id":"attaq","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":2,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fattaq"],"reproducibility_summary":{"results_total":2,"has_reproducibility_gap_count":2,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":2,"total_groups":2,"multi_source_groups":0,"first_party_only_groups":2,"source_type_distribution":{"first_party":2,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":2,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["robustness"]},{"key":"autologi","display_name":"AutoLogi","family_id":"autologi","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fautologi"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["logical_reasoning","mathematics"]},{"key":"babyvision","display_name":"Babyvision","family_id":"babyvision","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":4,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fbabyvision"],"reproducibility_summary":{"results_total":4,"has_reproducibility_gap_count":4,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":4,"total_groups":4,"multi_source_groups":0,"first_party_only_groups":4,"source_type_distribution":{"first_party":4,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":4,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["multimodal"]},{"key":"bbh","display_name":"BBH","family_id":"bbh","is_slice":false,"is_overall":false,"primary_metric_key":"rank","has_card":true,"tags":{"domains":["natural language processing","algorithmic reasoning","mathematics","commonsense reasoning","question-answering","logical reasoning","language understanding","symbolic manipulation"],"languages":["Not specified"],"tasks":["Boolean expressions","Causal judgement","Date understanding","Disambiguation QA","Dyck languages","Formal fallacies","Geometric shapes","Hyperbaton","Logical deduction","Movie recommendation","Multi-step arithmetic","Navigate","Object counting","Penguins in a table","Reasoning about colored objects","Ruin names","Salient translation error detection","Snarks","Sports understanding","Temporal sequences","Tracking shuffled objects","Web of lies","Word sorting"]},"metrics":[{"key":"rank","display_name":"Rank","sources":["LLM Stats"],"models_count":11,"is_primary":true}],"slices":[{"key":"llm stats","display_name":"llm stats","is_bare_stem":false,"metrics":[{"key":"rank","display_name":"Rank","sources":["LLM Stats"]}],"derivedTags":["general"]}],"summary_eval_ids":["llm-stats%2Fbbh"],"reproducibility_summary":{"results_total":11,"has_reproducibility_gap_count":11,"populated_ratio_avg":0.9285714285714287},"provenance_summary":{"total_results":11,"total_groups":11,"multi_source_groups":4,"first_party_only_groups":7,"source_type_distribution":{"first_party":11,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":11,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["logical_reasoning","applied_reasoning","general"]},{"key":"beyond-aime","display_name":"Beyond Aime","family_id":"beyond-aime","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":2,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fbeyond-aime"],"reproducibility_summary":{"results_total":2,"has_reproducibility_gap_count":2,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":2,"total_groups":2,"multi_source_groups":0,"first_party_only_groups":2,"source_type_distribution":{"first_party":2,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":2,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["mathematics"]},{"key":"bfcl","display_name":"BFCL","family_id":"bfcl","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["function calling","tool use","API interactions","multi-step reasoning"],"languages":["English"],"tasks":["Simple function calls","Multiple function selection","Parallel function calls","Parallel multiple function calls","Chatting capability","Function relevance detection","REST API calls","SQL query generation","Java function calls","JavaScript function calls","Multi-turn/multi-step function calling"]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":8,"is_primary":true}],"slices":[{"key":"bfcl","display_name":"BFCL","is_bare_stem":true,"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"]}],"derivedTags":["agentic","software_engineering"]},{"key":"bfcl-v2","display_name":"Bfcl v2","is_bare_stem":false,"metrics":[{"key":"ast-accuracy","display_name":"AST Accuracy","sources":["LLM Stats"]}],"derivedTags":["agentic","software_engineering"]},{"key":"bfcl-v3","display_name":"Bfcl v3","is_bare_stem":false,"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"]}],"derivedTags":["agentic","software_engineering"]},{"key":"bfcl-v4","display_name":"Bfcl v4","is_bare_stem":false,"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"]}],"derivedTags":["agentic","software_engineering"]}],"summary_eval_ids":["llm-stats%2Fbfcl"],"reproducibility_summary":{"results_total":8,"has_reproducibility_gap_count":8,"populated_ratio_avg":0.9285714285714287},"provenance_summary":{"total_results":8,"total_groups":8,"multi_source_groups":0,"first_party_only_groups":8,"source_type_distribution":{"first_party":8,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":8,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["agentic","software_engineering"]},{"key":"bfcl-v3-multiturn","display_name":"Bfcl v3 Multiturn","family_id":"bfcl-v3-multiturn","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":2,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fbfcl-v3-multiturn"],"reproducibility_summary":{"results_total":2,"has_reproducibility_gap_count":2,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":2,"total_groups":2,"multi_source_groups":0,"first_party_only_groups":2,"source_type_distribution":{"first_party":2,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":2,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["agentic","software_engineering"]},{"key":"big-bench","display_name":"Big Bench","family_id":"big-bench","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":[],"languages":["Not specified"],"tasks":["Zero-shot evaluation","Few-shot evaluation","Multiple diverse tasks including abstract narrative understanding, anachronisms, analogical similarity, analytic entailment, arithmetic, and many others"]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":3,"is_primary":true}],"slices":[{"key":"big-bench","display_name":"Big Bench","is_bare_stem":true,"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"]}],"derivedTags":["general","applied_reasoning"]},{"key":"big-bench-extra-hard","display_name":"Big Bench Extra Hard","is_bare_stem":false,"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"]}],"derivedTags":["general","applied_reasoning"]},{"key":"big-bench-hard","display_name":"BIG-Bench Hard","is_bare_stem":false,"metrics":[{"key":"rank","display_name":"Rank","sources":["LLM Stats"]}],"derivedTags":["general","applied_reasoning"]}],"summary_eval_ids":["llm-stats%2Fbig-bench"],"reproducibility_summary":{"results_total":3,"has_reproducibility_gap_count":3,"populated_ratio_avg":0.9285714285714285},"provenance_summary":{"total_results":3,"total_groups":3,"multi_source_groups":0,"first_party_only_groups":3,"source_type_distribution":{"first_party":3,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":3,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["general","applied_reasoning"]},{"key":"bigcodebench","display_name":"BigCodeBench","family_id":"bigcodebench","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":2,"is_primary":true}],"slices":[{"key":"bigcodebench","display_name":"BigCodeBench","is_bare_stem":true,"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"]}],"derivedTags":["software_engineering"]},{"key":"bigcodebench-hard","display_name":"Bigcodebench Hard","is_bare_stem":false,"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"]}],"derivedTags":["software_engineering"]}],"summary_eval_ids":["llm-stats%2Fbigcodebench"],"reproducibility_summary":{"results_total":2,"has_reproducibility_gap_count":2,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":2,"total_groups":2,"multi_source_groups":0,"first_party_only_groups":2,"source_type_distribution":{"first_party":2,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":2,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["software_engineering"]},{"key":"bigcodebench-full","display_name":"Bigcodebench Full","family_id":"bigcodebench-full","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fbigcodebench-full"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["software_engineering"]},{"key":"biolp-bench","display_name":"Biolp Bench","family_id":"biolp-bench","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fbiolp-bench"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["natural_sciences"]},{"key":"bird-sql-dev","display_name":"Bird Sql Dev","family_id":"bird-sql-dev","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["text-to-SQL","database interfaces","natural language processing"],"languages":["English"],"tasks":["Text-to-SQL conversion","Handling dirty/noisy database values","External knowledge grounding between natural language questions and database values","SQL efficiency optimization"]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":7,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fbird-sql-dev"],"reproducibility_summary":{"results_total":7,"has_reproducibility_gap_count":7,"populated_ratio_avg":0.9285714285714287},"provenance_summary":{"total_results":7,"total_groups":7,"multi_source_groups":0,"first_party_only_groups":7,"source_type_distribution":{"first_party":7,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":7,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["software_engineering"]},{"key":"bixbench","display_name":"Bixbench","family_id":"bixbench","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fbixbench"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["applied_reasoning"]},{"key":"blink","display_name":"Blink","family_id":"blink","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":6,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fblink"],"reproducibility_summary":{"results_total":6,"has_reproducibility_gap_count":6,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":6,"total_groups":6,"multi_source_groups":0,"first_party_only_groups":6,"source_type_distribution":{"first_party":6,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":6,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["knowledge","linguistic_core"]},{"key":"boolq","display_name":"BoolQ","family_id":"boolq","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["natural language understanding","reading comprehension","natural language inference"],"languages":["English"],"tasks":["Yes/no question answering","Text-pair classification"]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":8,"is_primary":true}],"slices":[{"key":"llm stats","display_name":"llm stats","is_bare_stem":false,"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"]}],"derivedTags":["general"]}],"summary_eval_ids":["llm-stats%2Fboolq"],"reproducibility_summary":{"results_total":8,"has_reproducibility_gap_count":8,"populated_ratio_avg":0.9285714285714287},"provenance_summary":{"total_results":8,"total_groups":8,"multi_source_groups":0,"first_party_only_groups":8,"source_type_distribution":{"first_party":8,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":8,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["commonsense_reasoning","general"]},{"key":"browsecomp","display_name":"Browsecomp","family_id":"browsecomp","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["retrieval augmented generation","information retrieval","llm evaluation"],"languages":["Not specified"],"tasks":["Evaluating end-to-end deep-research agents by testing generated answers against ground-truth","Evaluating retriever-only systems using standard retrieval metrics like nDCG@10","Question answering"]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":19,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fbrowsecomp"],"reproducibility_summary":{"results_total":19,"has_reproducibility_gap_count":19,"populated_ratio_avg":0.9285714285714285},"provenance_summary":{"total_results":19,"total_groups":19,"multi_source_groups":0,"first_party_only_groups":19,"source_type_distribution":{"first_party":19,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":19,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["agentic"]},{"key":"browsecomp-vl","display_name":"BrowseComp-VL","family_id":"browsecomp-vl","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fbrowsecomp-vl"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["agentic","multimodal"]},{"key":"browsecomp-zh","display_name":"BrowseComp-ZH","family_id":"browsecomp-zh","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["Film & TV","Technology","Medicine","History"],"languages":["Chinese"],"tasks":["Multi-hop question answering","Web browsing for information retrieval","Complex reasoning over web content"]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":12,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fbrowsecomp-zh"],"reproducibility_summary":{"results_total":12,"has_reproducibility_gap_count":12,"populated_ratio_avg":0.9285714285714287},"provenance_summary":{"total_results":12,"total_groups":12,"multi_source_groups":0,"first_party_only_groups":12,"source_type_distribution":{"first_party":12,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":12,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["agentic"]},{"key":"c-eval","display_name":"C-Eval","family_id":"c-eval","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["accounting","mathematics","art studies","medicine","business","literature","law","engineering","education","computer science","physics","chemistry","biology","politics","history","and many other professional fields"],"languages":["Chinese"],"tasks":["Multiple-choice question answering","Text classification","Question answering"]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":17,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fc-eval"],"reproducibility_summary":{"results_total":17,"has_reproducibility_gap_count":17,"populated_ratio_avg":0.9285714285714287},"provenance_summary":{"total_results":17,"total_groups":17,"multi_source_groups":0,"first_party_only_groups":17,"source_type_distribution":{"first_party":17,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":17,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["knowledge","mathematics","natural_sciences","humanities_and_social_sciences"]},{"key":"cbnsl","display_name":"Cbnsl","family_id":"cbnsl","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fcbnsl"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["linguistic_core"]},{"key":"cc-bench-v2-backend","display_name":"Cc Bench v2 Backend","family_id":"cc-bench-v2-backend","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fcc-bench-v2-backend"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["software_engineering"]},{"key":"cc-bench-v2-frontend","display_name":"Cc Bench v2 Frontend","family_id":"cc-bench-v2-frontend","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fcc-bench-v2-frontend"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["software_engineering"]},{"key":"cc-ocr","display_name":"Cc Ocr","family_id":"cc-ocr","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":14,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fcc-ocr"],"reproducibility_summary":{"results_total":14,"has_reproducibility_gap_count":14,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":14,"total_groups":14,"multi_source_groups":0,"first_party_only_groups":14,"source_type_distribution":{"first_party":14,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":14,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["multimodal"]},{"key":"cfeval","display_name":"Cfeval","family_id":"cfeval","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":2,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fcfeval"],"reproducibility_summary":{"results_total":2,"has_reproducibility_gap_count":2,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":2,"total_groups":2,"multi_source_groups":0,"first_party_only_groups":2,"source_type_distribution":{"first_party":2,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":2,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["software_engineering"]},{"key":"charadessta","display_name":"Charadessta","family_id":"charadessta","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":7,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fcharadessta"],"reproducibility_summary":{"results_total":7,"has_reproducibility_gap_count":7,"populated_ratio_avg":0.10714285714285712},"provenance_summary":{"total_results":7,"total_groups":7,"multi_source_groups":0,"first_party_only_groups":7,"source_type_distribution":{"first_party":7,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":7,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["multimodal"]},{"key":"chartqa","display_name":"ChartQA","family_id":"chartqa","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["data visualization analysis","visual question answering","chart interpretation"],"languages":["English"],"tasks":["Question answering where the input is a chart and natural language question, and the output is an answer derived through reasoning"]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":19,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fchartqa"],"reproducibility_summary":{"results_total":19,"has_reproducibility_gap_count":19,"populated_ratio_avg":0.9285714285714285},"provenance_summary":{"total_results":19,"total_groups":19,"multi_source_groups":0,"first_party_only_groups":19,"source_type_distribution":{"first_party":19,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":19,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["multimodal"]},{"key":"charxiv-d","display_name":"Charxiv D","family_id":"charxiv-d","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":9,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fcharxiv-d"],"reproducibility_summary":{"results_total":9,"has_reproducibility_gap_count":9,"populated_ratio_avg":0.10714285714285712},"provenance_summary":{"total_results":9,"total_groups":9,"multi_source_groups":0,"first_party_only_groups":9,"source_type_distribution":{"first_party":9,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":9,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["multimodal"]},{"key":"charxiv-r","display_name":"Charxiv R","family_id":"charxiv-r","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["scientific research"],"languages":["English"],"tasks":["Descriptive questions examining basic chart elements","Reasoning questions requiring synthesis of information across complex visual elements"]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":20,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fcharxiv-r"],"reproducibility_summary":{"results_total":20,"has_reproducibility_gap_count":20,"populated_ratio_avg":0.9285714285714285},"provenance_summary":{"total_results":20,"total_groups":20,"multi_source_groups":0,"first_party_only_groups":20,"source_type_distribution":{"first_party":20,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":20,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["multimodal"]},{"key":"chexpert-cxr","display_name":"Chexpert Cxr","family_id":"chexpert-cxr","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fchexpert-cxr"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["multimodal","natural_sciences"]},{"key":"claw-eval","display_name":"Claw Eval","family_id":"claw-eval","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":6,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fclaw-eval"],"reproducibility_summary":{"results_total":6,"has_reproducibility_gap_count":6,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":6,"total_groups":6,"multi_source_groups":0,"first_party_only_groups":6,"source_type_distribution":{"first_party":6,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":6,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["law"]},{"key":"cloningscenarios","display_name":"Cloningscenarios","family_id":"cloningscenarios","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fcloningscenarios"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["robustness"]},{"key":"cluewsc","display_name":"Cluewsc","family_id":"cluewsc","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["Not specified"],"languages":["Chinese"],"tasks":["Word sense disambiguation"]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":3,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fcluewsc"],"reproducibility_summary":{"results_total":3,"has_reproducibility_gap_count":3,"populated_ratio_avg":0.9285714285714285},"provenance_summary":{"total_results":3,"total_groups":3,"multi_source_groups":0,"first_party_only_groups":3,"source_type_distribution":{"first_party":3,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":3,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["linguistic_core"]},{"key":"cmmlu","display_name":"Cmmlu","family_id":"cmmlu","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["natural sciences","social sciences","engineering","humanities"],"languages":["Chinese"],"tasks":["Multiple-choice question answering"]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":4,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fcmmlu"],"reproducibility_summary":{"results_total":4,"has_reproducibility_gap_count":4,"populated_ratio_avg":0.9285714285714286},"provenance_summary":{"total_results":4,"total_groups":4,"multi_source_groups":0,"first_party_only_groups":4,"source_type_distribution":{"first_party":4,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":4,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["knowledge","mathematics","natural_sciences","humanities_and_social_sciences"]},{"key":"cnmo-2024","display_name":"Cnmo 2024","family_id":"cnmo-2024","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["Not specified"],"languages":["Not specified"],"tasks":["Not specified"]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":2,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fcnmo-2024"],"reproducibility_summary":{"results_total":2,"has_reproducibility_gap_count":2,"populated_ratio_avg":0.9285714285714286},"provenance_summary":{"total_results":2,"total_groups":2,"multi_source_groups":0,"first_party_only_groups":2,"source_type_distribution":{"first_party":2,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":2,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["mathematics"]},{"key":"codeforces","display_name":"Codeforces","family_id":"codeforces","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["algorithmic problem-solving","code generation","optimization"],"languages":[],"tasks":["Generating code solutions that pass all test cases","Meeting specified time and memory constraints"]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":11,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fcodeforces"],"reproducibility_summary":{"results_total":11,"has_reproducibility_gap_count":11,"populated_ratio_avg":0.9285714285714287},"provenance_summary":{"total_results":11,"total_groups":11,"multi_source_groups":0,"first_party_only_groups":11,"source_type_distribution":{"first_party":11,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":11,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["mathematics","software_engineering"]},{"key":"collie","display_name":"Collie","family_id":"collie","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":9,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fcollie"],"reproducibility_summary":{"results_total":9,"has_reproducibility_gap_count":9,"populated_ratio_avg":0.10714285714285712},"provenance_summary":{"total_results":9,"total_groups":9,"multi_source_groups":0,"first_party_only_groups":9,"source_type_distribution":{"first_party":9,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":9,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["applied_reasoning"]},{"key":"common-voice-15","display_name":"Common Voice 15","family_id":"common-voice-15","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fcommon-voice-15"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["linguistic_core"]},{"key":"commonsenseqa","display_name":"CommonsenseQA","family_id":"commonsenseqa","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fcommonsenseqa"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["commonsense_reasoning"]},{"key":"complexfuncbench","display_name":"ComplexFuncBench","family_id":"complexfuncbench","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":6,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fcomplexfuncbench"],"reproducibility_summary":{"results_total":6,"has_reproducibility_gap_count":6,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":6,"total_groups":6,"multi_source_groups":0,"first_party_only_groups":6,"source_type_distribution":{"first_party":6,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":6,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["software_engineering"]},{"key":"corpusqa-1m","display_name":"Corpusqa 1M","family_id":"corpusqa-1m","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["finance","law","scientific research"],"languages":["Not specified"],"tasks":["Complex computation-intensive queries","Global integration over corpus","Corpus comparison","Statistical aggregation"]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fcorpusqa-1m"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.9285714285714286},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["knowledge"]},{"key":"countbench","display_name":"Countbench","family_id":"countbench","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":6,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fcountbench"],"reproducibility_summary":{"results_total":6,"has_reproducibility_gap_count":6,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":6,"total_groups":6,"multi_source_groups":0,"first_party_only_groups":6,"source_type_distribution":{"first_party":6,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":6,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["mathematics"]},{"key":"covost2","display_name":"Covost2","family_id":"covost2","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["speech-to-text translation","multilingual natural language processing"],"languages":["Not specified"],"tasks":["Speech-to-text translation"]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":2,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fcovost2"],"reproducibility_summary":{"results_total":2,"has_reproducibility_gap_count":2,"populated_ratio_avg":0.9285714285714286},"provenance_summary":{"total_results":2,"total_groups":2,"multi_source_groups":0,"first_party_only_groups":2,"source_type_distribution":{"first_party":2,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":2,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["linguistic_core"]},{"key":"covost2-en-zh","display_name":"Covost2 En Zh","family_id":"covost2-en-zh","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fcovost2-en-zh"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["linguistic_core"]},{"key":"crag","display_name":"CRAG","family_id":"crag","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["finance","music","sports","movie","open domain"],"languages":["English"],"tasks":["Factual question answering","Question answering","Summarization"]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":3,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fcrag"],"reproducibility_summary":{"results_total":3,"has_reproducibility_gap_count":3,"populated_ratio_avg":0.9285714285714285},"provenance_summary":{"total_results":3,"total_groups":3,"multi_source_groups":0,"first_party_only_groups":3,"source_type_distribution":{"first_party":3,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":3,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["applied_reasoning"]},{"key":"creative-writing-v3","display_name":"Creative Writing v3","family_id":"creative-writing-v3","is_slice":false,"is_overall":false,"primary_metric_key":"elo","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"elo","display_name":"Elo Rating","sources":["LLM Stats"],"models_count":9,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fcreative-writing-v3"],"reproducibility_summary":{"results_total":9,"has_reproducibility_gap_count":9,"populated_ratio_avg":0.10714285714285712},"provenance_summary":{"total_results":9,"total_groups":9,"multi_source_groups":0,"first_party_only_groups":9,"source_type_distribution":{"first_party":9,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":9,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["linguistic_core"]},{"key":"crperelation","display_name":"Crperelation","family_id":"crperelation","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fcrperelation"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["linguistic_core"]},{"key":"crux-o","display_name":"Crux O","family_id":"crux-o","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fcrux-o"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["software_engineering"]},{"key":"cruxeval-input-cot","display_name":"Cruxeval Input Cot","family_id":"cruxeval-input-cot","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fcruxeval-input-cot"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["software_engineering"]},{"key":"cruxeval-o","display_name":"Cruxeval O","family_id":"cruxeval-o","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fcruxeval-o"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["software_engineering"]},{"key":"cruxeval-output-cot","display_name":"Cruxeval Output Cot","family_id":"cruxeval-output-cot","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fcruxeval-output-cot"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["software_engineering"]},{"key":"csimpleqa","display_name":"Csimpleqa","family_id":"csimpleqa","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["Not specified"],"languages":["Not specified"],"tasks":["Not specified"]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":5,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fcsimpleqa"],"reproducibility_summary":{"results_total":5,"has_reproducibility_gap_count":5,"populated_ratio_avg":0.9285714285714286},"provenance_summary":{"total_results":5,"total_groups":5,"multi_source_groups":0,"first_party_only_groups":5,"source_type_distribution":{"first_party":5,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":5,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["knowledge"]},{"key":"cybench","display_name":"Cybench","family_id":"cybench","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["cybersecurity"],"languages":["Not specified"],"tasks":["Capture the Flag (CTF) tasks","Cybersecurity penetration testing"]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":2,"is_primary":true}],"slices":[{"key":"llm stats","display_name":"llm stats","is_bare_stem":false,"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"]}],"derivedTags":["general"]}],"summary_eval_ids":["llm-stats%2Fcybench"],"reproducibility_summary":{"results_total":2,"has_reproducibility_gap_count":2,"populated_ratio_avg":0.9285714285714286},"provenance_summary":{"total_results":2,"total_groups":2,"multi_source_groups":0,"first_party_only_groups":2,"source_type_distribution":{"first_party":2,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":2,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["software_engineering","safety","general"]},{"key":"cybergym","display_name":"Cybergym","family_id":"cybergym","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["cybersecurity"],"languages":[],"tasks":["Generating a proof-of-concept test to reproduce a vulnerability from a text description and codebase"]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":6,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fcybergym"],"reproducibility_summary":{"results_total":6,"has_reproducibility_gap_count":6,"populated_ratio_avg":0.9285714285714287},"provenance_summary":{"total_results":6,"total_groups":6,"multi_source_groups":0,"first_party_only_groups":6,"source_type_distribution":{"first_party":6,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":6,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["agentic","safety"]},{"key":"cybersecurity-ctfs","display_name":"Cybersecurity Ctfs","family_id":"cybersecurity-ctfs","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["cryptography","forensics","binary exploitation","code reverse engineering","web exploitation"],"languages":["Not specified"],"tasks":["Identifying vulnerabilities","Printing hidden flags in Jeopardy-style CTF challenges","Attack-Defense challenges involving system defense and attack"]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":3,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fcybersecurity-ctfs"],"reproducibility_summary":{"results_total":3,"has_reproducibility_gap_count":3,"populated_ratio_avg":0.9285714285714285},"provenance_summary":{"total_results":3,"total_groups":3,"multi_source_groups":0,"first_party_only_groups":3,"source_type_distribution":{"first_party":3,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":3,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["agentic","safety"]},{"key":"deepsearchqa","display_name":"Deepsearchqa","family_id":"deepsearchqa","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":5,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fdeepsearchqa"],"reproducibility_summary":{"results_total":5,"has_reproducibility_gap_count":5,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":5,"total_groups":5,"multi_source_groups":0,"first_party_only_groups":5,"source_type_distribution":{"first_party":5,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":5,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["applied_reasoning"]},{"key":"dermmcqa","display_name":"Dermmcqa","family_id":"dermmcqa","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fdermmcqa"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["natural_sciences"]},{"key":"design2code","display_name":"Design2Code","family_id":"design2code","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":2,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fdesign2code"],"reproducibility_summary":{"results_total":2,"has_reproducibility_gap_count":2,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":2,"total_groups":2,"multi_source_groups":0,"first_party_only_groups":2,"source_type_distribution":{"first_party":2,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":2,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["software_engineering"]},{"key":"docvqa","display_name":"Docvqa","family_id":"docvqa","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["Document Analysis and Recognition","visual question answering","document understanding"],"languages":["Not specified"],"tasks":["Visual Question Answering on document images","Responding to ad-hoc natural language questions about document content and structure"]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":18,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fdocvqa"],"reproducibility_summary":{"results_total":18,"has_reproducibility_gap_count":18,"populated_ratio_avg":0.9285714285714286},"provenance_summary":{"total_results":18,"total_groups":18,"multi_source_groups":0,"first_party_only_groups":18,"source_type_distribution":{"first_party":18,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":18,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["multimodal"]},{"key":"docvqatest","display_name":"Docvqatest","family_id":"docvqatest","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":6,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fdocvqatest"],"reproducibility_summary":{"results_total":6,"has_reproducibility_gap_count":6,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":6,"total_groups":6,"multi_source_groups":0,"first_party_only_groups":6,"source_type_distribution":{"first_party":6,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":6,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["multimodal"]},{"key":"drop","display_name":"DROP","family_id":"drop","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["reading comprehension","paragraph understanding"],"languages":["English"],"tasks":["Question answering requiring discrete reasoning","Numerical operation execution","Text-based computation"]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":18,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fdrop"],"reproducibility_summary":{"results_total":18,"has_reproducibility_gap_count":18,"populated_ratio_avg":0.9285714285714286},"provenance_summary":{"total_results":18,"total_groups":18,"multi_source_groups":0,"first_party_only_groups":18,"source_type_distribution":{"first_party":18,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":18,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["linguistic_core","applied_reasoning"]},{"key":"ds-arena-code","display_name":"Ds Arena Code","family_id":"ds-arena-code","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["Not specified"],"languages":["Not specified"],"tasks":["Not specified"]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fds-arena-code"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.9285714285714286},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["software_engineering"]},{"key":"ds-fim-eval","display_name":"Ds Fim Eval","family_id":"ds-fim-eval","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["Not specified"],"languages":["Not specified"],"tasks":["Not specified"]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fds-fim-eval"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.9285714285714286},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["software_engineering"]},{"key":"dynamath","display_name":"DynaMath","family_id":"dynamath","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":5,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fdynamath"],"reproducibility_summary":{"results_total":5,"has_reproducibility_gap_count":5,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":5,"total_groups":5,"multi_source_groups":0,"first_party_only_groups":5,"source_type_distribution":{"first_party":5,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":5,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["mathematics"]},{"key":"eclektic","display_name":"ECLeKTic","family_id":"eclektic","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":8,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Feclektic"],"reproducibility_summary":{"results_total":8,"has_reproducibility_gap_count":8,"populated_ratio_avg":0.10714285714285712},"provenance_summary":{"total_results":8,"total_groups":8,"multi_source_groups":0,"first_party_only_groups":8,"source_type_distribution":{"first_party":8,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":8,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["knowledge"]},{"key":"egoschema","display_name":"Egoschema","family_id":"egoschema","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["long-form video understanding","egocentric video analysis","human activity and behavior recognition"],"languages":["English"],"tasks":["Multiple-choice question-answering based on long-form video content"]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":9,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fegoschema"],"reproducibility_summary":{"results_total":9,"has_reproducibility_gap_count":9,"populated_ratio_avg":0.9285714285714286},"provenance_summary":{"total_results":9,"total_groups":9,"multi_source_groups":0,"first_party_only_groups":9,"source_type_distribution":{"first_party":9,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":9,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["multimodal"]},{"key":"embspatialbench","display_name":"Embspatialbench","family_id":"embspatialbench","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":6,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fembspatialbench"],"reproducibility_summary":{"results_total":6,"has_reproducibility_gap_count":6,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":6,"total_groups":6,"multi_source_groups":0,"first_party_only_groups":6,"source_type_distribution":{"first_party":6,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":6,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["commonsense_reasoning"]},{"key":"eq-bench","display_name":"Eq Bench","family_id":"eq-bench","is_slice":false,"is_overall":false,"primary_metric_key":"elo","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"elo","display_name":"Elo Rating","sources":["LLM Stats"],"models_count":2,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Feq-bench"],"reproducibility_summary":{"results_total":2,"has_reproducibility_gap_count":2,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":2,"total_groups":2,"multi_source_groups":0,"first_party_only_groups":2,"source_type_distribution":{"first_party":2,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":2,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["commonsense_reasoning"]},{"key":"erqa","display_name":"Erqa","family_id":"erqa","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":14,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Ferqa"],"reproducibility_summary":{"results_total":14,"has_reproducibility_gap_count":14,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":14,"total_groups":14,"multi_source_groups":0,"first_party_only_groups":14,"source_type_distribution":{"first_party":14,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":14,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["knowledge"]},{"key":"evalplus","display_name":"EvalPlus","family_id":"evalplus","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":4,"is_primary":true}],"slices":[{"key":"humaneval-plus","display_name":"Humaneval-PLUS","is_bare_stem":false,"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"]}],"derivedTags":["software_engineering"]},{"key":"mbpp-plus","display_name":"MBPP-PLUS","is_bare_stem":false,"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"]}],"derivedTags":["software_engineering"]}],"summary_eval_ids":["llm-stats%2Fevalplus"],"reproducibility_summary":{"results_total":4,"has_reproducibility_gap_count":4,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":4,"total_groups":4,"multi_source_groups":0,"first_party_only_groups":4,"source_type_distribution":{"first_party":4,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":4,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["software_engineering"]},{"key":"factscore","display_name":"Factscore","family_id":"factscore","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":2,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Ffactscore"],"reproducibility_summary":{"results_total":2,"has_reproducibility_gap_count":2,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":2,"total_groups":2,"multi_source_groups":0,"first_party_only_groups":2,"source_type_distribution":{"first_party":2,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":2,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["hallucination"]},{"key":"figqa","display_name":"Figqa","family_id":"figqa","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["natural language processing","figurative language understanding","commonsense reasoning"],"languages":["English"],"tasks":["Multiple-choice question answering"]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":3,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Ffigqa"],"reproducibility_summary":{"results_total":3,"has_reproducibility_gap_count":3,"populated_ratio_avg":0.9285714285714285},"provenance_summary":{"total_results":3,"total_groups":3,"multi_source_groups":0,"first_party_only_groups":3,"source_type_distribution":{"first_party":3,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":3,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["multimodal"]},{"key":"finance-agent","display_name":"Finance Agent","family_id":"finance-agent","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["finance","financial analysis","financial research"],"languages":[],"tasks":["Information retrieval","Complex financial modeling"]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":5,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Ffinance-agent"],"reproducibility_summary":{"results_total":5,"has_reproducibility_gap_count":5,"populated_ratio_avg":0.9285714285714286},"provenance_summary":{"total_results":5,"total_groups":5,"multi_source_groups":0,"first_party_only_groups":5,"source_type_distribution":{"first_party":5,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":5,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["agentic"]},{"key":"finqa","display_name":"Finqa","family_id":"finqa","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["financial question answering","numerical reasoning"],"languages":["English"],"tasks":["Not specified"]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":3,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Ffinqa"],"reproducibility_summary":{"results_total":3,"has_reproducibility_gap_count":3,"populated_ratio_avg":0.9285714285714285},"provenance_summary":{"total_results":3,"total_groups":3,"multi_source_groups":0,"first_party_only_groups":3,"source_type_distribution":{"first_party":3,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":3,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["finance"]},{"key":"finsearchcomp-t2-t3","display_name":"Finsearchcomp T2 T3","family_id":"finsearchcomp-t2-t3","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Ffinsearchcomp-t2-t3"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["finance"]},{"key":"finsearchcomp-t3","display_name":"Finsearchcomp T3","family_id":"finsearchcomp-t3","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Ffinsearchcomp-t3"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["finance"]},{"key":"flame-vlm-code","display_name":"Flame Vlm Code","family_id":"flame-vlm-code","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fflame-vlm-code"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["software_engineering"]},{"key":"flenqa","display_name":"Flenqa","family_id":"flenqa","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fflenqa"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["knowledge"]},{"key":"fleurs","display_name":"Fleurs","family_id":"fleurs","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["speech recognition","speech language identification","translation","retrieval"],"languages":["afr","amh","ara","asm","ast","azj","bel","ben","bos","bul","cat","ceb","ces","ckb","cmn","cym","dan","deu","ell","eng","est","fas","fin","fra","ful","gle","glg","guj","hau","heb","hin","hrv","hun","hye","ibo","ind","isl","ita","jav","jpn","kam","kan","kat","kaz","kea","khm","kir","kor","lao","lav","lin","lit","ltz","lug","luo","mal","mar","mkd","mlt","mon","mri","msa","mya","nld","nob","npi","nso","nya","oci","orm","ory","pan","pol","por","pus","ron","rus","slk","slv","sna","snd","som","spa","srp","swe","swh","tam","tel","tgk","tgl","tha","tur","ukr","umb","urd","uzb","vie","wol","xho","yor","yue","zul"],"tasks":["Automatic speech recognition","Speech language identification","Translation","Retrieval"]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":5,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Ffleurs"],"reproducibility_summary":{"results_total":5,"has_reproducibility_gap_count":5,"populated_ratio_avg":0.9285714285714286},"provenance_summary":{"total_results":5,"total_groups":5,"multi_source_groups":0,"first_party_only_groups":5,"source_type_distribution":{"first_party":5,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":5,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["linguistic_core"]},{"key":"frames","display_name":"Frames","family_id":"frames","is_slice":false,"is_overall":false,"primary_metric_key":"accuracy","has_card":true,"tags":{"domains":["Not specified"],"languages":["Not specified"],"tasks":["Not specified"]},"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["LLM Stats"],"models_count":2,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fframes"],"reproducibility_summary":{"results_total":2,"has_reproducibility_gap_count":2,"populated_ratio_avg":0.9285714285714286},"provenance_summary":{"total_results":2,"total_groups":2,"multi_source_groups":0,"first_party_only_groups":2,"source_type_distribution":{"first_party":2,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":2,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["linguistic_core"]},{"key":"french-mmlu","display_name":"French Mmlu","family_id":"french-mmlu","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Ffrench-mmlu"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["general"]},{"key":"frontiermath","display_name":"FrontierMath","family_id":"frontiermath","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":13,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Ffrontiermath"],"reproducibility_summary":{"results_total":13,"has_reproducibility_gap_count":13,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":13,"total_groups":13,"multi_source_groups":0,"first_party_only_groups":13,"source_type_distribution":{"first_party":13,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":13,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["mathematics"]},{"key":"frontierscience-research","display_name":"Frontierscience Research","family_id":"frontierscience-research","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Ffrontierscience-research"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["natural_sciences"]},{"key":"fullstackbench-en","display_name":"Fullstackbench En","family_id":"fullstackbench-en","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":3,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Ffullstackbench-en"],"reproducibility_summary":{"results_total":3,"has_reproducibility_gap_count":3,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":3,"total_groups":3,"multi_source_groups":0,"first_party_only_groups":3,"source_type_distribution":{"first_party":3,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":3,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["software_engineering"]},{"key":"fullstackbench-zh","display_name":"Fullstackbench Zh","family_id":"fullstackbench-zh","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":3,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Ffullstackbench-zh"],"reproducibility_summary":{"results_total":3,"has_reproducibility_gap_count":3,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":3,"total_groups":3,"multi_source_groups":0,"first_party_only_groups":3,"source_type_distribution":{"first_party":3,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":3,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["software_engineering"]},{"key":"functionalmath","display_name":"Functionalmath","family_id":"functionalmath","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["Not specified"],"languages":["Not specified"],"tasks":["Not specified"]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":2,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Ffunctionalmath"],"reproducibility_summary":{"results_total":2,"has_reproducibility_gap_count":2,"populated_ratio_avg":0.9285714285714286},"provenance_summary":{"total_results":2,"total_groups":2,"multi_source_groups":0,"first_party_only_groups":2,"source_type_distribution":{"first_party":2,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":2,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["mathematics"]},{"key":"gdpval-aa","display_name":"Gdpval Aa","family_id":"gdpval-aa","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["manufacturing engineering","financial analysis","healthcare","professional occupations"],"languages":["English"],"tasks":["Tasks constructed from the representative work of industry professionals, such as designing a 3D model of a cable reel (Manufacturing Engineer) and creating a financial analysis (Financial and Investment Analyst)"]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":7,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fgdpval-aa"],"reproducibility_summary":{"results_total":7,"has_reproducibility_gap_count":7,"populated_ratio_avg":0.9285714285714287},"provenance_summary":{"total_results":7,"total_groups":7,"multi_source_groups":0,"first_party_only_groups":7,"source_type_distribution":{"first_party":7,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":7,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["general"]},{"key":"gdpval-mm","display_name":"Gdpval Mm","family_id":"gdpval-mm","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":3,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fgdpval-mm"],"reproducibility_summary":{"results_total":3,"has_reproducibility_gap_count":3,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":3,"total_groups":3,"multi_source_groups":0,"first_party_only_groups":3,"source_type_distribution":{"first_party":3,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":3,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["multimodal"]},{"key":"genebench","display_name":"Genebench","family_id":"genebench","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":2,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fgenebench"],"reproducibility_summary":{"results_total":2,"has_reproducibility_gap_count":2,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":2,"total_groups":2,"multi_source_groups":0,"first_party_only_groups":2,"source_type_distribution":{"first_party":2,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":2,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["natural_sciences"]},{"key":"giantsteps-tempo","display_name":"Giantsteps Tempo","family_id":"giantsteps-tempo","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fgiantsteps-tempo"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["multimodal"]},{"key":"global-mmlu","display_name":"Global Mmlu","family_id":"global-mmlu","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":4,"is_primary":true}],"slices":[{"key":"global-mmlu","display_name":"Global Mmlu","is_bare_stem":true,"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"]}],"derivedTags":["general"]},{"key":"global-mmlu-lite","display_name":"Global MMLU Lite","is_bare_stem":false,"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"]}],"derivedTags":["general"]}],"summary_eval_ids":["llm-stats%2Fglobal-mmlu"],"reproducibility_summary":{"results_total":4,"has_reproducibility_gap_count":4,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":4,"total_groups":4,"multi_source_groups":0,"first_party_only_groups":4,"source_type_distribution":{"first_party":4,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":4,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["general"]},{"key":"global-piqa","display_name":"Global PIQA","family_id":"global-piqa","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":11,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fglobal-piqa"],"reproducibility_summary":{"results_total":11,"has_reproducibility_gap_count":11,"populated_ratio_avg":0.10714285714285712},"provenance_summary":{"total_results":11,"total_groups":11,"multi_source_groups":0,"first_party_only_groups":11,"source_type_distribution":{"first_party":11,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":11,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["commonsense_reasoning"]},{"key":"gorilla-benchmark-api-bench","display_name":"Gorilla Benchmark Api Bench","family_id":"gorilla-benchmark-api-bench","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fgorilla-benchmark-api-bench"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["software_engineering","agentic"]},{"key":"govreport","display_name":"Govreport","family_id":"govreport","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":2,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fgovreport"],"reproducibility_summary":{"results_total":2,"has_reproducibility_gap_count":2,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":2,"total_groups":2,"multi_source_groups":0,"first_party_only_groups":2,"source_type_distribution":{"first_party":2,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":2,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["linguistic_core"]},{"key":"gpqa","display_name":"GPQA","family_id":"gpqa","is_slice":false,"is_overall":false,"primary_metric_key":"accuracy","has_card":true,"tags":{"domains":["biology","physics","chemistry"],"languages":["English"],"tasks":["Multiple-choice question answering"]},"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["LLM Stats"],"models_count":19,"is_primary":true}],"slices":[{"key":"llm stats","display_name":"llm stats","is_bare_stem":false,"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["LLM Stats"]}],"derivedTags":["general"]}],"summary_eval_ids":["llm-stats%2Fgpqa"],"reproducibility_summary":{"results_total":19,"has_reproducibility_gap_count":19,"populated_ratio_avg":0.9285714285714287},"provenance_summary":{"total_results":19,"total_groups":19,"multi_source_groups":12,"first_party_only_groups":7,"source_type_distribution":{"first_party":19,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":19,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["natural_sciences","general"]},{"key":"gpqa-biology","display_name":"Gpqa Biology","family_id":"gpqa-biology","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fgpqa-biology"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["natural_sciences"]},{"key":"gpqa-chemistry","display_name":"Gpqa Chemistry","family_id":"gpqa-chemistry","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fgpqa-chemistry"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["natural_sciences"]},{"key":"gpqa-physics","display_name":"Gpqa Physics","family_id":"gpqa-physics","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fgpqa-physics"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["natural_sciences"]},{"key":"graphwalks-bfs-128k","display_name":"Graphwalks Bfs 128k","family_id":"graphwalks-bfs-128k","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["Not specified"],"languages":["Not specified"],"tasks":["Not specified"]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":14,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fgraphwalks-bfs-128k"],"reproducibility_summary":{"results_total":14,"has_reproducibility_gap_count":14,"populated_ratio_avg":0.9285714285714287},"provenance_summary":{"total_results":14,"total_groups":14,"multi_source_groups":0,"first_party_only_groups":14,"source_type_distribution":{"first_party":14,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":14,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["logical_reasoning"]},{"key":"graphwalks-parents-128k","display_name":"Graphwalks Parents 128k","family_id":"graphwalks-parents-128k","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["Not specified"],"languages":["Not specified"],"tasks":["Not specified"]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":13,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fgraphwalks-parents-128k"],"reproducibility_summary":{"results_total":13,"has_reproducibility_gap_count":13,"populated_ratio_avg":0.9285714285714287},"provenance_summary":{"total_results":13,"total_groups":13,"multi_source_groups":0,"first_party_only_groups":13,"source_type_distribution":{"first_party":13,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":13,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["logical_reasoning"]},{"key":"groundui-1k","display_name":"Groundui 1K","family_id":"groundui-1k","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["visual grounding","user interface"],"languages":["Not specified"],"tasks":["Not specified"]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":2,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fgroundui-1k"],"reproducibility_summary":{"results_total":2,"has_reproducibility_gap_count":2,"populated_ratio_avg":0.9285714285714286},"provenance_summary":{"total_results":2,"total_groups":2,"multi_source_groups":0,"first_party_only_groups":2,"source_type_distribution":{"first_party":2,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":2,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["multimodal","agentic"]},{"key":"gsm-8k-cot","display_name":"Gsm 8K Cot","family_id":"gsm-8k-cot","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fgsm-8k-cot"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["mathematics"]},{"key":"gsm8k","display_name":"GSM8K","family_id":"gsm8k","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["grade school mathematics","math word problems"],"languages":["English"],"tasks":["Solving grade school math word problems","Text generation for question answering"]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":18,"is_primary":true}],"slices":[{"key":"llm stats","display_name":"llm stats","is_bare_stem":false,"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"]}],"derivedTags":["general"]}],"summary_eval_ids":["llm-stats%2Fgsm8k"],"reproducibility_summary":{"results_total":18,"has_reproducibility_gap_count":18,"populated_ratio_avg":0.9285714285714286},"provenance_summary":{"total_results":18,"total_groups":18,"multi_source_groups":9,"first_party_only_groups":9,"source_type_distribution":{"first_party":18,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":18,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["mathematics","general"]},{"key":"gsm8k-chat","display_name":"Gsm8k Chat","family_id":"gsm8k-chat","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fgsm8k-chat"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["mathematics"]},{"key":"hallusion-bench","display_name":"Hallusion Bench","family_id":"hallusion-bench","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":11,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fhallusion-bench"],"reproducibility_summary":{"results_total":11,"has_reproducibility_gap_count":11,"populated_ratio_avg":0.10714285714285712},"provenance_summary":{"total_results":11,"total_groups":11,"multi_source_groups":0,"first_party_only_groups":11,"source_type_distribution":{"first_party":11,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":11,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["hallucination"]},{"key":"healthbench","display_name":"Healthbench","family_id":"healthbench","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":4,"is_primary":true}],"slices":[{"key":"healthbench","display_name":"Healthbench","is_bare_stem":true,"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"]}],"derivedTags":["natural_sciences","knowledge"]},{"key":"healthbench-hard","display_name":"Healthbench Hard","is_bare_stem":false,"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"]}],"derivedTags":["natural_sciences","knowledge"]}],"summary_eval_ids":["llm-stats%2Fhealthbench"],"reproducibility_summary":{"results_total":4,"has_reproducibility_gap_count":4,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":4,"total_groups":4,"multi_source_groups":0,"first_party_only_groups":4,"source_type_distribution":{"first_party":4,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":4,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["natural_sciences","knowledge"]},{"key":"hellaswag","display_name":"HellaSwag","family_id":"hellaswag","is_slice":false,"is_overall":false,"primary_metric_key":"accuracy","has_card":true,"tags":{"domains":["commonsense reasoning","natural language inference"],"languages":["English"],"tasks":["Four-way multiple-choice selection for event continuation","Commonsense inference"]},"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["LLM Stats"],"models_count":19,"is_primary":true}],"slices":[{"key":"llm stats","display_name":"llm stats","is_bare_stem":false,"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["LLM Stats"]}],"derivedTags":["general"]}],"summary_eval_ids":["llm-stats%2Fhellaswag"],"reproducibility_summary":{"results_total":19,"has_reproducibility_gap_count":19,"populated_ratio_avg":0.9285714285714285},"provenance_summary":{"total_results":19,"total_groups":19,"multi_source_groups":0,"first_party_only_groups":19,"source_type_distribution":{"first_party":19,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":19,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["commonsense_reasoning","general"]},{"key":"hiddenmath","display_name":"Hiddenmath","family_id":"hiddenmath","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["Not specified"],"languages":["Not specified"],"tasks":["Not specified"]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":13,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fhiddenmath"],"reproducibility_summary":{"results_total":13,"has_reproducibility_gap_count":13,"populated_ratio_avg":0.9285714285714287},"provenance_summary":{"total_results":13,"total_groups":13,"multi_source_groups":0,"first_party_only_groups":13,"source_type_distribution":{"first_party":13,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":13,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["mathematics"]},{"key":"hle","display_name":"HLE","family_id":"hle","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":19,"is_primary":true}],"slices":[{"key":"llm stats","display_name":"llm stats","is_bare_stem":false,"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"]}],"derivedTags":["general"]}],"summary_eval_ids":["llm-stats%2Fhle"],"reproducibility_summary":{"results_total":19,"has_reproducibility_gap_count":19,"populated_ratio_avg":0.10714285714285716},"provenance_summary":{"total_results":19,"total_groups":19,"multi_source_groups":14,"first_party_only_groups":5,"source_type_distribution":{"first_party":19,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":19,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["general"]},{"key":"hmmt-2025","display_name":"Hmmt 2025","family_id":"hmmt-2025","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["Not specified"],"languages":["Not specified"],"tasks":["Not specified"]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":19,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fhmmt-2025"],"reproducibility_summary":{"results_total":19,"has_reproducibility_gap_count":19,"populated_ratio_avg":0.9285714285714285},"provenance_summary":{"total_results":19,"total_groups":19,"multi_source_groups":0,"first_party_only_groups":19,"source_type_distribution":{"first_party":19,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":19,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["mathematics"]},{"key":"hmmt-feb-26","display_name":"Hmmt Feb 26","family_id":"hmmt-feb-26","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["Not specified"],"languages":["Not specified"],"tasks":["Not specified"]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":6,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fhmmt-feb-26"],"reproducibility_summary":{"results_total":6,"has_reproducibility_gap_count":6,"populated_ratio_avg":0.9285714285714287},"provenance_summary":{"total_results":6,"total_groups":6,"multi_source_groups":0,"first_party_only_groups":6,"source_type_distribution":{"first_party":6,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":6,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["mathematics"]},{"key":"hmmt25","display_name":"Hmmt25","family_id":"hmmt25","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":17,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fhmmt25"],"reproducibility_summary":{"results_total":17,"has_reproducibility_gap_count":17,"populated_ratio_avg":0.10714285714285715},"provenance_summary":{"total_results":17,"total_groups":17,"multi_source_groups":0,"first_party_only_groups":17,"source_type_distribution":{"first_party":17,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":17,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["mathematics"]},{"key":"humaneval","display_name":"HumanEval","family_id":"humaneval","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["code generation"],"languages":["English"],"tasks":["Generating Python code from docstrings"]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":22,"is_primary":true}],"slices":[{"key":"llm stats","display_name":"llm stats","is_bare_stem":false,"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"]}],"derivedTags":["general"]}],"summary_eval_ids":["llm-stats%2Fhumaneval"],"reproducibility_summary":{"results_total":22,"has_reproducibility_gap_count":22,"populated_ratio_avg":0.9285714285714284},"provenance_summary":{"total_results":22,"total_groups":22,"multi_source_groups":3,"first_party_only_groups":19,"source_type_distribution":{"first_party":22,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":22,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["software_engineering","general"]},{"key":"humaneval-average","display_name":"Humaneval Average","family_id":"humaneval-average","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fhumaneval-average"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["software_engineering"]},{"key":"humaneval-er","display_name":"Humaneval Er","family_id":"humaneval-er","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fhumaneval-er"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["software_engineering"]},{"key":"humaneval-mul","display_name":"Humaneval Mul","family_id":"humaneval-mul","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["Not specified"],"languages":["Not specified"],"tasks":["Not specified"]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":2,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fhumaneval-mul"],"reproducibility_summary":{"results_total":2,"has_reproducibility_gap_count":2,"populated_ratio_avg":0.9285714285714286},"provenance_summary":{"total_results":2,"total_groups":2,"multi_source_groups":0,"first_party_only_groups":2,"source_type_distribution":{"first_party":2,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":2,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["software_engineering"]},{"key":"humanevalfim-average","display_name":"Humanevalfim Average","family_id":"humanevalfim-average","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fhumanevalfim-average"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["software_engineering"]},{"key":"hypersim","display_name":"Hypersim","family_id":"hypersim","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":4,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fhypersim"],"reproducibility_summary":{"results_total":4,"has_reproducibility_gap_count":4,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":4,"total_groups":4,"multi_source_groups":0,"first_party_only_groups":4,"source_type_distribution":{"first_party":4,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":4,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["multimodal"]},{"key":"if","display_name":"If","family_id":"if","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":2,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fif"],"reproducibility_summary":{"results_total":2,"has_reproducibility_gap_count":2,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":2,"total_groups":2,"multi_source_groups":0,"first_party_only_groups":2,"source_type_distribution":{"first_party":2,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":2,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["linguistic_core"]},{"key":"ifbench","display_name":"IFBench","family_id":"ifbench","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":16,"is_primary":true}],"slices":[{"key":"llm stats","display_name":"llm stats","is_bare_stem":false,"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"]}],"derivedTags":["general"]}],"summary_eval_ids":["llm-stats%2Fifbench"],"reproducibility_summary":{"results_total":16,"has_reproducibility_gap_count":16,"populated_ratio_avg":0.10714285714285715},"provenance_summary":{"total_results":16,"total_groups":16,"multi_source_groups":13,"first_party_only_groups":3,"source_type_distribution":{"first_party":16,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":16,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["linguistic_core","general"]},{"key":"imagemining","display_name":"Imagemining","family_id":"imagemining","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fimagemining"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["multimodal"]},{"key":"imo-answerbench","display_name":"Imo Answerbench","family_id":"imo-answerbench","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["mathematical reasoning","imo","problem solving","Algebra","Combinatorics","Geometry","Number Theory"],"languages":["English"],"tasks":["Mathematical problem solving","Producing verifiable short answers for Olympiad problems"]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":13,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fimo-answerbench"],"reproducibility_summary":{"results_total":13,"has_reproducibility_gap_count":13,"populated_ratio_avg":0.9285714285714287},"provenance_summary":{"total_results":13,"total_groups":13,"multi_source_groups":0,"first_party_only_groups":13,"source_type_distribution":{"first_party":13,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":13,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["mathematics"]},{"key":"include","display_name":"Include","family_id":"include","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":13,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Finclude"],"reproducibility_summary":{"results_total":13,"has_reproducibility_gap_count":13,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":13,"total_groups":13,"multi_source_groups":0,"first_party_only_groups":13,"source_type_distribution":{"first_party":13,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":13,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["knowledge","linguistic_core"]},{"key":"infographicsqa","display_name":"Infographicsqa","family_id":"infographicsqa","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Finfographicsqa"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["multimodal","knowledge"]},{"key":"infovqa","display_name":"Infovqa","family_id":"infovqa","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["Not specified"],"languages":["Not specified"],"tasks":["Visual Question Answering"]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":9,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Finfovqa"],"reproducibility_summary":{"results_total":9,"has_reproducibility_gap_count":9,"populated_ratio_avg":0.9285714285714286},"provenance_summary":{"total_results":9,"total_groups":9,"multi_source_groups":0,"first_party_only_groups":9,"source_type_distribution":{"first_party":9,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":9,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["multimodal","knowledge"]},{"key":"infovqatest","display_name":"Infovqatest","family_id":"infovqatest","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":7,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Finfovqatest"],"reproducibility_summary":{"results_total":7,"has_reproducibility_gap_count":7,"populated_ratio_avg":0.10714285714285712},"provenance_summary":{"total_results":7,"total_groups":7,"multi_source_groups":0,"first_party_only_groups":7,"source_type_distribution":{"first_party":7,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":7,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["multimodal","knowledge"]},{"key":"instruct-humaneval","display_name":"Instruct Humaneval","family_id":"instruct-humaneval","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Finstruct-humaneval"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["software_engineering"]},{"key":"intergps","display_name":"Intergps","family_id":"intergps","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":2,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fintergps"],"reproducibility_summary":{"results_total":2,"has_reproducibility_gap_count":2,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":2,"total_groups":2,"multi_source_groups":0,"first_party_only_groups":2,"source_type_distribution":{"first_party":2,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":2,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["multimodal"]},{"key":"internal-api-instruction-following-hard","display_name":"Internal Api Instruction Following Hard","family_id":"internal-api-instruction-following-hard","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":7,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Finternal-api-instruction-following-hard"],"reproducibility_summary":{"results_total":7,"has_reproducibility_gap_count":7,"populated_ratio_avg":0.10714285714285712},"provenance_summary":{"total_results":7,"total_groups":7,"multi_source_groups":0,"first_party_only_groups":7,"source_type_distribution":{"first_party":7,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":7,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["linguistic_core"]},{"key":"ipho-2025","display_name":"Ipho 2025","family_id":"ipho-2025","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fipho-2025"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["natural_sciences","mathematics"]},{"key":"lbpp-v2","display_name":"Lbpp v2","family_id":"lbpp-v2","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Flbpp-v2"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["software_engineering"]},{"key":"lingoqa","display_name":"Lingoqa","family_id":"lingoqa","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":3,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Flingoqa"],"reproducibility_summary":{"results_total":3,"has_reproducibility_gap_count":3,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":3,"total_groups":3,"multi_source_groups":0,"first_party_only_groups":3,"source_type_distribution":{"first_party":3,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":3,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["linguistic_core","knowledge"]},{"key":"livebench-20241125","display_name":"Livebench 20241125","family_id":"livebench-20241125","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":7,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Flivebench-20241125"],"reproducibility_summary":{"results_total":7,"has_reproducibility_gap_count":7,"populated_ratio_avg":0.10714285714285712},"provenance_summary":{"total_results":7,"total_groups":7,"multi_source_groups":0,"first_party_only_groups":7,"source_type_distribution":{"first_party":7,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":7,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["general"]},{"key":"livecodebench","display_name":"LiveCodeBench","family_id":"livecodebench","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["computer programming","software development","coding competitions"],"languages":["Not specified"],"tasks":["Code generation","Self-repair","Code execution","Test output prediction"]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":16,"is_primary":true}],"slices":[{"key":"livecodebench","display_name":"LiveCodeBench","is_bare_stem":true,"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"]}],"derivedTags":["software_engineering"]},{"key":"livecodebench-pro","display_name":"LiveCodeBench Pro","is_bare_stem":false,"metrics":[{"key":"elo","display_name":"Elo Rating","sources":["LLM Stats"]}],"derivedTags":["software_engineering"]},{"key":"livecodebench-v5","display_name":"Livecodebench v5","is_bare_stem":false,"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"]}],"derivedTags":["software_engineering"]},{"key":"livecodebench-v6","display_name":"Livecodebench v6","is_bare_stem":false,"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"]}],"derivedTags":["software_engineering"]},{"key":"llm stats","display_name":"llm stats","is_bare_stem":false,"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"]}],"derivedTags":["general"]}],"summary_eval_ids":["llm-stats%2Flivecodebench"],"reproducibility_summary":{"results_total":16,"has_reproducibility_gap_count":16,"populated_ratio_avg":0.9285714285714287},"provenance_summary":{"total_results":16,"total_groups":16,"multi_source_groups":12,"first_party_only_groups":4,"source_type_distribution":{"first_party":16,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":16,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["software_engineering","general"]},{"key":"livecodebench-01-09","display_name":"Livecodebench 01 09","family_id":"livecodebench-01-09","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["computer programming","software development","code generation"],"languages":["Not specified"],"tasks":["Code generation","Self-repair","Code execution","Test output prediction"]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Flivecodebench-01-09"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.9285714285714286},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["software_engineering"]},{"key":"llm stats codegolf-v2","display_name":"Llm Stats Codegolf-v2","family_id":"llm stats codegolf-v2","is_slice":false,"is_overall":false,"primary_metric_key":"Codegolf v2.2 benchmark","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"Codegolf v2.2 benchmark","display_name":"Codegolf v2.2 benchmark","sources":["LLM Stats"],"models_count":4,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fllm%20stats%20codegolf-v2"],"reproducibility_summary":{"results_total":4,"has_reproducibility_gap_count":4,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":4,"total_groups":4,"multi_source_groups":0,"first_party_only_groups":4,"source_type_distribution":{"first_party":4,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":4,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["general"]},{"key":"llm stats infinitebench-en","display_name":"Llm Stats Infinitebench-En","family_id":"llm stats infinitebench-en","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fllm%20stats%20infinitebench-en"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["general"]},{"key":"llm stats livecodebench-v5-24 12-25","display_name":"Llm Stats Livecodebench-v5-24 12-25","family_id":"llm stats livecodebench-v5-24 12-25","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fllm%20stats%20livecodebench-v5-24%2012-25"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["general"]},{"key":"llm stats mmbench-v1","display_name":"Llm Stats Mmbench-v1","family_id":"llm stats mmbench-v1","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":14,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fllm%20stats%20mmbench-v1"],"reproducibility_summary":{"results_total":14,"has_reproducibility_gap_count":14,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":14,"total_groups":14,"multi_source_groups":0,"first_party_only_groups":14,"source_type_distribution":{"first_party":14,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":14,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["general"]},{"key":"llm stats mmlu-redux-2","display_name":"Llm Stats MMLU-Redux-2","family_id":"llm stats mmlu-redux-2","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fllm%20stats%20mmlu-redux-2"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["general"]},{"key":"llm stats omnidocbench-1","display_name":"Llm Stats Omnidocbench-1","family_id":"llm stats omnidocbench-1","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":11,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fllm%20stats%20omnidocbench-1"],"reproducibility_summary":{"results_total":11,"has_reproducibility_gap_count":11,"populated_ratio_avg":0.10714285714285712},"provenance_summary":{"total_results":11,"total_groups":11,"multi_source_groups":0,"first_party_only_groups":11,"source_type_distribution":{"first_party":11,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":11,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["general"]},{"key":"llm-stats","display_name":"LLM-Stats","family_id":"llm-stats","is_slice":false,"is_overall":true,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"DeepPlanning evaluates LLMs","display_name":"DeepPlanning evaluates LLMs","sources":["LLM Stats"],"models_count":8,"is_primary":false},{"key":"rank","display_name":"Rank","sources":["LLM Stats"],"models_count":2,"is_primary":false},{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":29,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fllm-stats"],"reproducibility_summary":{"results_total":29,"has_reproducibility_gap_count":29,"populated_ratio_avg":0.10714285714285718},"provenance_summary":{"total_results":29,"total_groups":29,"multi_source_groups":0,"first_party_only_groups":29,"source_type_distribution":{"first_party":29,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":29,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":true,"derivedTags":["general"]},{"key":"longbench-v2","display_name":"LongBench v2","family_id":"longbench-v2","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["Long Context","reasoning"],"languages":["English"],"tasks":["Single-document QA","Multi-document QA","Long in-context learning","Long-dialogue history understanding","Code repository understanding","Long structured data understanding","Multiple choice","Question answering","Text classification","Table question answering"]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":13,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Flongbench-v2"],"reproducibility_summary":{"results_total":13,"has_reproducibility_gap_count":13,"populated_ratio_avg":0.9285714285714287},"provenance_summary":{"total_results":13,"total_groups":13,"multi_source_groups":0,"first_party_only_groups":13,"source_type_distribution":{"first_party":13,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":13,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["linguistic_core"]},{"key":"longfact-concepts","display_name":"Longfact Concepts","family_id":"longfact-concepts","is_slice":false,"is_overall":false,"primary_metric_key":"accuracy","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Flongfact-concepts"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["hallucination"]},{"key":"longfact-objects","display_name":"Longfact Objects","family_id":"longfact-objects","is_slice":false,"is_overall":false,"primary_metric_key":"accuracy","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Flongfact-objects"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["hallucination"]},{"key":"longvideobench","display_name":"Longvideobench","family_id":"longvideobench","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":2,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Flongvideobench"],"reproducibility_summary":{"results_total":2,"has_reproducibility_gap_count":2,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":2,"total_groups":2,"multi_source_groups":0,"first_party_only_groups":2,"source_type_distribution":{"first_party":2,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":2,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["multimodal"]},{"key":"lsat","display_name":"Lsat","family_id":"lsat","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Flsat"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["law","logical_reasoning"]},{"key":"lvbench","display_name":"Lvbench","family_id":"lvbench","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["Not specified"],"languages":["Not specified"],"tasks":["Not specified"]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":15,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Flvbench"],"reproducibility_summary":{"results_total":15,"has_reproducibility_gap_count":15,"populated_ratio_avg":0.9285714285714287},"provenance_summary":{"total_results":15,"total_groups":15,"multi_source_groups":0,"first_party_only_groups":15,"source_type_distribution":{"first_party":15,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":15,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["multimodal"]},{"key":"mask","display_name":"Mask","family_id":"mask","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fmask"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["linguistic_core"]},{"key":"math","display_name":"MATH","family_id":"math","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["mathematical reasoning","explanation generation"],"languages":["English"],"tasks":["Generating a final answer sequence given a math problem"]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":20,"is_primary":true}],"slices":[{"key":"llm stats","display_name":"llm stats","is_bare_stem":false,"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"]}],"derivedTags":["general"]},{"key":"math","display_name":"MATH","is_bare_stem":true,"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"]}],"derivedTags":["mathematics"]},{"key":"math-500","display_name":"MATH-500","is_bare_stem":false,"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"]}],"derivedTags":["mathematics"]},{"key":"math-cot","display_name":"Math Cot","is_bare_stem":false,"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"]}],"derivedTags":["mathematics"]}],"summary_eval_ids":["llm-stats%2Fmath"],"reproducibility_summary":{"results_total":20,"has_reproducibility_gap_count":20,"populated_ratio_avg":0.9285714285714285},"provenance_summary":{"total_results":20,"total_groups":20,"multi_source_groups":7,"first_party_only_groups":13,"source_type_distribution":{"first_party":20,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":20,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["mathematics","general"]},{"key":"matharena-apex","display_name":"Matharena Apex","family_id":"matharena-apex","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["mathematical reasoning","proof writing","math competitions"],"languages":["English"],"tasks":["Solving math competition problems","Multi-step logical reasoning","Proof writing"]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":2,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fmatharena-apex"],"reproducibility_summary":{"results_total":2,"has_reproducibility_gap_count":2,"populated_ratio_avg":0.9285714285714286},"provenance_summary":{"total_results":2,"total_groups":2,"multi_source_groups":0,"first_party_only_groups":2,"source_type_distribution":{"first_party":2,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":2,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["mathematics"]},{"key":"mathverse-mini","display_name":"Mathverse Mini","family_id":"mathverse","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fmathverse-mini"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["mathematics","multimodal"]},{"key":"mathvision","display_name":"Mathvision","family_id":"mathvision","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":16,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fmathvision"],"reproducibility_summary":{"results_total":16,"has_reproducibility_gap_count":16,"populated_ratio_avg":0.10714285714285715},"provenance_summary":{"total_results":16,"total_groups":16,"multi_source_groups":0,"first_party_only_groups":16,"source_type_distribution":{"first_party":16,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":16,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["mathematics","multimodal"]},{"key":"mathvista","display_name":"MathVista","family_id":"mathvista","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["mathematical reasoning","visual contexts"],"languages":["Chinese","English","Persian"],"tasks":["Multiple choice","Question answering","Visual question answering","Text classification"]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":18,"is_primary":true}],"slices":[{"key":"llm stats","display_name":"llm stats","is_bare_stem":false,"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"]}],"derivedTags":["general"]},{"key":"mathvista","display_name":"MathVista","is_bare_stem":true,"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"]}],"derivedTags":["mathematics","multimodal"]},{"key":"mathvista-mini","display_name":"Mathvista Mini","is_bare_stem":false,"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"]}],"derivedTags":["mathematics","multimodal"]}],"summary_eval_ids":["llm-stats%2Fmathvista"],"reproducibility_summary":{"results_total":18,"has_reproducibility_gap_count":18,"populated_ratio_avg":0.9285714285714286},"provenance_summary":{"total_results":18,"total_groups":18,"multi_source_groups":1,"first_party_only_groups":17,"source_type_distribution":{"first_party":18,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":18,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["mathematics","multimodal","general"]},{"key":"maxife","display_name":"Maxife","family_id":"maxife","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":9,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fmaxife"],"reproducibility_summary":{"results_total":9,"has_reproducibility_gap_count":9,"populated_ratio_avg":0.10714285714285712},"provenance_summary":{"total_results":9,"total_groups":9,"multi_source_groups":0,"first_party_only_groups":9,"source_type_distribution":{"first_party":9,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":9,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["multimodal"]},{"key":"mbpp","display_name":"MBPP","family_id":"mbpp","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["Not specified"],"languages":["Not specified"],"tasks":["Not specified"]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":21,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fmbpp"],"reproducibility_summary":{"results_total":21,"has_reproducibility_gap_count":21,"populated_ratio_avg":0.9285714285714284},"provenance_summary":{"total_results":21,"total_groups":21,"multi_source_groups":0,"first_party_only_groups":21,"source_type_distribution":{"first_party":21,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":21,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["software_engineering"]},{"key":"mbpp-base-version","display_name":"Mbpp Base Version","family_id":"mbpp-base-version","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fmbpp-base-version"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["software_engineering"]},{"key":"mbpp-evalplus-base","display_name":"Mbpp Evalplus Base","family_id":"mbpp-evalplus-base","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fmbpp-evalplus-base"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["software_engineering"]},{"key":"mbpp-pass-1","display_name":"Mbpp Pass 1","family_id":"mbpp-pass-1","is_slice":false,"is_overall":false,"primary_metric_key":"pass-at-1","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"pass-at-1","display_name":"Pass@1","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fmbpp-pass-1"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["software_engineering"]},{"key":"mcp-atlas","display_name":"Mcp Atlas","family_id":"mcp-atlas","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":16,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fmcp-atlas"],"reproducibility_summary":{"results_total":16,"has_reproducibility_gap_count":16,"populated_ratio_avg":0.10714285714285715},"provenance_summary":{"total_results":16,"total_groups":16,"multi_source_groups":0,"first_party_only_groups":16,"source_type_distribution":{"first_party":16,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":16,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["agentic"]},{"key":"mcp-mark","display_name":"Mcp Mark","family_id":"mcp-mark","is_slice":false,"is_overall":false,"primary_metric_key":"MCP-Mark evaluates LLMs","has_card":true,"tags":{"domains":["Not specified"],"languages":["Not specified"],"tasks":["Executing tool-calling loops to perform various operations"]},"metrics":[{"key":"MCP-Mark evaluates LLMs","display_name":"MCP-Mark evaluates LLMs","sources":["LLM Stats"],"models_count":5,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fmcp-mark"],"reproducibility_summary":{"results_total":5,"has_reproducibility_gap_count":5,"populated_ratio_avg":0.9285714285714286},"provenance_summary":{"total_results":5,"total_groups":5,"multi_source_groups":0,"first_party_only_groups":5,"source_type_distribution":{"first_party":5,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":5,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["agentic"]},{"key":"mcp-universe","display_name":"MCP-Universe","family_id":"mcp-universe","is_slice":false,"is_overall":false,"primary_metric_key":"MCP-Universe evaluates LLMs","has_card":true,"tags":{"domains":["Location Navigation","Repository Management","Financial Analysis","3D Design","Browser Automation","Web Searching"],"languages":[],"tasks":["Interacting with MCP servers across six core domains (navigation, repository management, financial analysis, etc.)"]},"metrics":[{"key":"MCP-Universe evaluates LLMs","display_name":"MCP-Universe evaluates LLMs","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fmcp-universe"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.9285714285714286},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["agentic"]},{"key":"medxpertqa","display_name":"MedXpertQA","family_id":"medxpertqa","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":9,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fmedxpertqa"],"reproducibility_summary":{"results_total":9,"has_reproducibility_gap_count":9,"populated_ratio_avg":0.10714285714285712},"provenance_summary":{"total_results":9,"total_groups":9,"multi_source_groups":0,"first_party_only_groups":9,"source_type_distribution":{"first_party":9,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":9,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["knowledge","natural_sciences"]},{"key":"mega-mlqa","display_name":"Mega Mlqa","family_id":"mega-mlqa","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":2,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fmega-mlqa"],"reproducibility_summary":{"results_total":2,"has_reproducibility_gap_count":2,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":2,"total_groups":2,"multi_source_groups":0,"first_party_only_groups":2,"source_type_distribution":{"first_party":2,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":2,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["knowledge"]},{"key":"mega-tydi-qa","display_name":"Mega Tydi Qa","family_id":"mega-tydi-qa","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":2,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fmega-tydi-qa"],"reproducibility_summary":{"results_total":2,"has_reproducibility_gap_count":2,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":2,"total_groups":2,"multi_source_groups":0,"first_party_only_groups":2,"source_type_distribution":{"first_party":2,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":2,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["knowledge"]},{"key":"mega-udpos","display_name":"Mega Udpos","family_id":"mega-udpos","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":2,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fmega-udpos"],"reproducibility_summary":{"results_total":2,"has_reproducibility_gap_count":2,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":2,"total_groups":2,"multi_source_groups":0,"first_party_only_groups":2,"source_type_distribution":{"first_party":2,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":2,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["linguistic_core"]},{"key":"mega-xcopa","display_name":"Mega Xcopa","family_id":"mega-xcopa","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":2,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fmega-xcopa"],"reproducibility_summary":{"results_total":2,"has_reproducibility_gap_count":2,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":2,"total_groups":2,"multi_source_groups":0,"first_party_only_groups":2,"source_type_distribution":{"first_party":2,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":2,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["commonsense_reasoning"]},{"key":"mega-xstorycloze","display_name":"Mega Xstorycloze","family_id":"mega-xstorycloze","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":2,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fmega-xstorycloze"],"reproducibility_summary":{"results_total":2,"has_reproducibility_gap_count":2,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":2,"total_groups":2,"multi_source_groups":0,"first_party_only_groups":2,"source_type_distribution":{"first_party":2,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":2,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["commonsense_reasoning"]},{"key":"meld","display_name":"Meld","family_id":"meld","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fmeld"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["linguistic_core","commonsense_reasoning"]},{"key":"mewc","display_name":"Mewc","family_id":"mewc","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fmewc"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["knowledge"]},{"key":"mgsm","display_name":"MGSM","family_id":"mgsm","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["mathematical reasoning","question answering","math word problems"],"languages":["Bengali","Chinese","English","French","German","Japanese","Russian","Spanish","Swahili","Thai"],"tasks":["Solving grade-school math word problems","Text generation"]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":18,"is_primary":true}],"slices":[{"key":"llm stats","display_name":"llm stats","is_bare_stem":false,"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"]}],"derivedTags":["general"]}],"summary_eval_ids":["llm-stats%2Fmgsm"],"reproducibility_summary":{"results_total":18,"has_reproducibility_gap_count":18,"populated_ratio_avg":0.9285714285714286},"provenance_summary":{"total_results":18,"total_groups":18,"multi_source_groups":10,"first_party_only_groups":8,"source_type_distribution":{"first_party":18,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":18,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["mathematics","general"]},{"key":"miabench","display_name":"Miabench","family_id":"miabench","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fmiabench"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["applied_reasoning"]},{"key":"mimic-cxr","display_name":"Mimic Cxr","family_id":"mimic-cxr","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fmimic-cxr"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["multimodal","natural_sciences"]},{"key":"mle-bench-lite","display_name":"Mle Bench Lite","family_id":"mle-bench-lite","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fmle-bench-lite"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["software_engineering"]},{"key":"mlvu","display_name":"Mlvu","family_id":"mlvu","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":8,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fmlvu"],"reproducibility_summary":{"results_total":8,"has_reproducibility_gap_count":8,"populated_ratio_avg":0.10714285714285712},"provenance_summary":{"total_results":8,"total_groups":8,"multi_source_groups":0,"first_party_only_groups":8,"source_type_distribution":{"first_party":8,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":8,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["multimodal"]},{"key":"mlvu-m","display_name":"Mlvu M","family_id":"mlvu-m","is_slice":false,"is_overall":false,"primary_metric_key":"MLVU-M benchmark","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"MLVU-M benchmark","display_name":"MLVU-M benchmark","sources":["LLM Stats"],"models_count":5,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fmlvu-m"],"reproducibility_summary":{"results_total":5,"has_reproducibility_gap_count":5,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":5,"total_groups":5,"multi_source_groups":0,"first_party_only_groups":5,"source_type_distribution":{"first_party":5,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":5,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["multimodal"]},{"key":"mm-browsercomp","display_name":"Mm Browsercomp","family_id":"mm-browsercomp","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fmm-browsercomp"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["agentic","multimodal"]},{"key":"mm-clawbench","display_name":"Mm Clawbench","family_id":"mm-clawbench","is_slice":false,"is_overall":false,"primary_metric_key":"MM-ClawBench evaluates models","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"MM-ClawBench evaluates models","display_name":"MM-ClawBench evaluates models","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fmm-clawbench"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["agentic","multimodal"]},{"key":"mm-if-eval","display_name":"Mm If Eval","family_id":"mm-if-eval","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fmm-if-eval"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["multimodal"]},{"key":"mm-mind2web","display_name":"Mm Mind2web","family_id":"mm-mind2web","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["travel","airlines","restaurants","government services","and 27 other domains"],"languages":["Not specified"],"tasks":["Following language instructions to complete tasks on real-world websites","Web navigation and interaction"]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":3,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fmm-mind2web"],"reproducibility_summary":{"results_total":3,"has_reproducibility_gap_count":3,"populated_ratio_avg":0.9285714285714285},"provenance_summary":{"total_results":3,"total_groups":3,"multi_source_groups":0,"first_party_only_groups":3,"source_type_distribution":{"first_party":3,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":3,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["agentic","multimodal"]},{"key":"mm-mt-bench","display_name":"Mm Mt Bench","family_id":"mm-mt-bench","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":12,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fmm-mt-bench"],"reproducibility_summary":{"results_total":12,"has_reproducibility_gap_count":12,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":12,"total_groups":12,"multi_source_groups":0,"first_party_only_groups":12,"source_type_distribution":{"first_party":12,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":12,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["agentic","multimodal"]},{"key":"mmau","display_name":"Mmau","family_id":"mmau","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fmmau"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["multimodal"]},{"key":"mmau-music","display_name":"Mmau Music","family_id":"mmau-music","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fmmau-music"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["multimodal"]},{"key":"mmau-sound","display_name":"Mmau Sound","family_id":"mmau-sound","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fmmau-sound"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["multimodal"]},{"key":"mmau-speech","display_name":"Mmau Speech","family_id":"mmau-speech","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fmmau-speech"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["multimodal"]},{"key":"mmbench","display_name":"MMBench","family_id":"mmbench","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":8,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fmmbench"],"reproducibility_summary":{"results_total":8,"has_reproducibility_gap_count":8,"populated_ratio_avg":0.10714285714285712},"provenance_summary":{"total_results":8,"total_groups":8,"multi_source_groups":0,"first_party_only_groups":8,"source_type_distribution":{"first_party":8,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":8,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["multimodal"]},{"key":"mmbench-test","display_name":"Mmbench Test","family_id":"mmbench-test","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fmmbench-test"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["multimodal"]},{"key":"mmbench-video","display_name":"MMBench-Video","family_id":"mmbench-video","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":3,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fmmbench-video"],"reproducibility_summary":{"results_total":3,"has_reproducibility_gap_count":3,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":3,"total_groups":3,"multi_source_groups":0,"first_party_only_groups":3,"source_type_distribution":{"first_party":3,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":3,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["multimodal"]},{"key":"mme","display_name":"Mme","family_id":"mme","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["Not specified"],"languages":["Not specified"],"tasks":["Not specified"]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":3,"is_primary":true}],"slices":[{"key":"mme","display_name":"Mme","is_bare_stem":true,"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"]}],"derivedTags":["multimodal"]}],"summary_eval_ids":["llm-stats%2Fmme"],"reproducibility_summary":{"results_total":3,"has_reproducibility_gap_count":3,"populated_ratio_avg":0.9285714285714285},"provenance_summary":{"total_results":3,"total_groups":3,"multi_source_groups":0,"first_party_only_groups":3,"source_type_distribution":{"first_party":3,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":3,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["multimodal"]},{"key":"mme-realworld","display_name":"Mme Realworld","family_id":"mme-realworld","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fmme-realworld"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["multimodal"]},{"key":"mmlongbench-doc","display_name":"Mmlongbench Doc","family_id":"mmlongbench-doc","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":5,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fmmlongbench-doc"],"reproducibility_summary":{"results_total":5,"has_reproducibility_gap_count":5,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":5,"total_groups":5,"multi_source_groups":0,"first_party_only_groups":5,"source_type_distribution":{"first_party":5,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":5,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["multimodal","linguistic_core"]},{"key":"mmlu-base","display_name":"Mmlu Base","family_id":"mmlu-base","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fmmlu-base"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["general","knowledge"]},{"key":"mmlu-chat","display_name":"Mmlu Chat","family_id":"mmlu-chat","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fmmlu-chat"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["general","knowledge"]},{"key":"mmlu-french","display_name":"Mmlu French","family_id":"mmlu-french","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fmmlu-french"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["general","knowledge"]},{"key":"mmlu-prox","display_name":"Mmlu Prox","family_id":"mmlu-prox","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":14,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fmmlu-prox"],"reproducibility_summary":{"results_total":14,"has_reproducibility_gap_count":14,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":14,"total_groups":14,"multi_source_groups":0,"first_party_only_groups":14,"source_type_distribution":{"first_party":14,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":14,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["general","knowledge"]},{"key":"mmlu-redux","display_name":"Mmlu Redux","family_id":"mmlu-redux","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["abstract algebra","anatomy","astronomy","business ethics","clinical knowledge","college-level sciences","computer security","econometrics","electrical engineering","formal logic","global facts","high school subjects","human aging","human sexuality","international law","jurisprudence","logical fallacies","machine learning","management","marketing","medical genetics"],"languages":["English"],"tasks":["Question answering"]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":17,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fmmlu-redux"],"reproducibility_summary":{"results_total":17,"has_reproducibility_gap_count":17,"populated_ratio_avg":0.9285714285714287},"provenance_summary":{"total_results":17,"total_groups":17,"multi_source_groups":0,"first_party_only_groups":17,"source_type_distribution":{"first_party":17,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":17,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["general","knowledge"]},{"key":"mmlu-stem","display_name":"Mmlu Stem","family_id":"mmlu-stem","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":2,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fmmlu-stem"],"reproducibility_summary":{"results_total":2,"has_reproducibility_gap_count":2,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":2,"total_groups":2,"multi_source_groups":0,"first_party_only_groups":2,"source_type_distribution":{"first_party":2,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":2,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["general","knowledge","natural_sciences","mathematics"]},{"key":"mmmlu","display_name":"Mmmlu","family_id":"mmmlu","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["mathematics","history","computer science","law","morality","physics"],"languages":["Arabic","Bengali","Chinese","French","German","Hindi","Indonesian","Italian","Japanese","Korean","Portuguese","Spanish","Swahili","yo"],"tasks":["Question answering"]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":19,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fmmmlu"],"reproducibility_summary":{"results_total":19,"has_reproducibility_gap_count":19,"populated_ratio_avg":0.9285714285714285},"provenance_summary":{"total_results":19,"total_groups":19,"multi_source_groups":0,"first_party_only_groups":19,"source_type_distribution":{"first_party":19,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":19,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["multimodal","knowledge"]},{"key":"mmmu","display_name":"MMMU","family_id":"mmmu","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":19,"is_primary":true}],"slices":[{"key":"llm stats","display_name":"llm stats","is_bare_stem":false,"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"]}],"derivedTags":["general"]},{"key":"mmmu","display_name":"MMMU","is_bare_stem":true,"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"]}],"derivedTags":["multimodal","knowledge"]},{"key":"mmmu-pro","display_name":"MMMU-Pro","is_bare_stem":false,"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"]}],"derivedTags":["multimodal","knowledge"]}],"summary_eval_ids":["llm-stats%2Fmmmu"],"reproducibility_summary":{"results_total":19,"has_reproducibility_gap_count":19,"populated_ratio_avg":0.10714285714285716},"provenance_summary":{"total_results":19,"total_groups":19,"multi_source_groups":8,"first_party_only_groups":11,"source_type_distribution":{"first_party":19,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":19,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["multimodal","knowledge","general"]},{"key":"mmmu-val","display_name":"Mmmu Val","family_id":"mmmu-val","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":7,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fmmmu-val"],"reproducibility_summary":{"results_total":7,"has_reproducibility_gap_count":7,"populated_ratio_avg":0.10714285714285712},"provenance_summary":{"total_results":7,"total_groups":7,"multi_source_groups":0,"first_party_only_groups":7,"source_type_distribution":{"first_party":7,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":7,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["multimodal","knowledge"]},{"key":"mmmu-validation","display_name":"Mmmu Validation","family_id":"mmmu-validation","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":4,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fmmmu-validation"],"reproducibility_summary":{"results_total":4,"has_reproducibility_gap_count":4,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":4,"total_groups":4,"multi_source_groups":0,"first_party_only_groups":4,"source_type_distribution":{"first_party":4,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":4,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["multimodal","knowledge"]},{"key":"mmmuval","display_name":"Mmmuval","family_id":"mmmuval","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":3,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fmmmuval"],"reproducibility_summary":{"results_total":3,"has_reproducibility_gap_count":3,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":3,"total_groups":3,"multi_source_groups":0,"first_party_only_groups":3,"source_type_distribution":{"first_party":3,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":3,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["multimodal","knowledge"]},{"key":"mmsearch","display_name":"Mmsearch","family_id":"mmsearch","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fmmsearch"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["multimodal","applied_reasoning"]},{"key":"mmsearch-plus","display_name":"MMSearch-Plus","family_id":"mmsearch-plus","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fmmsearch-plus"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["multimodal","applied_reasoning"]},{"key":"mmstar","display_name":"MMStar","family_id":"mmstar","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["vision-language integration","multi-modal research"],"languages":["English"],"tasks":["Visual question answering","Multiple-choice question answering"]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":15,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fmmstar"],"reproducibility_summary":{"results_total":15,"has_reproducibility_gap_count":15,"populated_ratio_avg":0.9285714285714287},"provenance_summary":{"total_results":15,"total_groups":15,"multi_source_groups":0,"first_party_only_groups":15,"source_type_distribution":{"first_party":15,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":15,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["multimodal","applied_reasoning"]},{"key":"mmt-bench","display_name":"MMT-Bench","family_id":"mmt-bench","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["multimodal understanding","vehicle driving","embodied navigation"],"languages":["Not specified"],"tasks":["Multimodal understanding across 32 core meta-tasks and 162 subtasks"]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":4,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fmmt-bench"],"reproducibility_summary":{"results_total":4,"has_reproducibility_gap_count":4,"populated_ratio_avg":0.9285714285714286},"provenance_summary":{"total_results":4,"total_groups":4,"multi_source_groups":0,"first_party_only_groups":4,"source_type_distribution":{"first_party":4,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":4,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["multimodal","applied_reasoning"]},{"key":"mmvet","display_name":"Mmvet","family_id":"mmvet","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":2,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fmmvet"],"reproducibility_summary":{"results_total":2,"has_reproducibility_gap_count":2,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":2,"total_groups":2,"multi_source_groups":0,"first_party_only_groups":2,"source_type_distribution":{"first_party":2,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":2,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["multimodal","applied_reasoning"]},{"key":"mmvetgpt4turbo","display_name":"Mmvetgpt4turbo","family_id":"mmvetgpt4turbo","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fmmvetgpt4turbo"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["multimodal","applied_reasoning"]},{"key":"mmvu","display_name":"MMVU","family_id":"mmvu","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":4,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fmmvu"],"reproducibility_summary":{"results_total":4,"has_reproducibility_gap_count":4,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":4,"total_groups":4,"multi_source_groups":0,"first_party_only_groups":4,"source_type_distribution":{"first_party":4,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":4,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["multimodal","knowledge"]},{"key":"mobileminiwob-sr","display_name":"Mobileminiwob Sr","family_id":"mobileminiwob-sr","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":2,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fmobileminiwob-sr"],"reproducibility_summary":{"results_total":2,"has_reproducibility_gap_count":2,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":2,"total_groups":2,"multi_source_groups":0,"first_party_only_groups":2,"source_type_distribution":{"first_party":2,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":2,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["agentic"]},{"key":"motionbench","display_name":"MotionBench","family_id":"motionbench","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fmotionbench"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["multimodal","applied_reasoning"]},{"key":"mrcr","display_name":"Mrcr","family_id":"mrcr","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["natural language processing","coreference resolution","long-context reasoning"],"languages":["English"],"tasks":["Multi-round coreference resolution","Returning the i-th instance of a specific, repeated request from within a long conversation"]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":7,"is_primary":true}],"slices":[{"key":"mrcr","display_name":"Mrcr","is_bare_stem":true,"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"]}],"derivedTags":["linguistic_core","applied_reasoning"]},{"key":"mrcr-v2","display_name":"Mrcr v2","is_bare_stem":false,"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"]}],"derivedTags":["linguistic_core","applied_reasoning"]}],"summary_eval_ids":["llm-stats%2Fmrcr"],"reproducibility_summary":{"results_total":7,"has_reproducibility_gap_count":7,"populated_ratio_avg":0.9285714285714287},"provenance_summary":{"total_results":7,"total_groups":7,"multi_source_groups":0,"first_party_only_groups":7,"source_type_distribution":{"first_party":7,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":7,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["linguistic_core","applied_reasoning"]},{"key":"mrcr-128k-2-needle","display_name":"Mrcr 128k 2 Needle","family_id":"mrcr-128k-2-needle","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fmrcr-128k-2-needle"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["linguistic_core","applied_reasoning"]},{"key":"mrcr-128k-4-needle","display_name":"Mrcr 128k 4 Needle","family_id":"mrcr-128k-4-needle","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fmrcr-128k-4-needle"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["linguistic_core","applied_reasoning"]},{"key":"mrcr-128k-8-needle","display_name":"Mrcr 128k 8 Needle","family_id":"mrcr-128k-8-needle","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fmrcr-128k-8-needle"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["linguistic_core","applied_reasoning"]},{"key":"mrcr-1m","display_name":"Mrcr 1M","family_id":"mrcr-1m","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["Not specified"],"languages":["Not specified"],"tasks":["Not specified"]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":2,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fmrcr-1m"],"reproducibility_summary":{"results_total":2,"has_reproducibility_gap_count":2,"populated_ratio_avg":0.9285714285714286},"provenance_summary":{"total_results":2,"total_groups":2,"multi_source_groups":0,"first_party_only_groups":2,"source_type_distribution":{"first_party":2,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":2,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["linguistic_core","applied_reasoning"]},{"key":"mrcr-1m-pointwise","display_name":"Mrcr 1M Pointwise","family_id":"mrcr-1m-pointwise","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fmrcr-1m-pointwise"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["linguistic_core","applied_reasoning"]},{"key":"mrcr-64k-2-needle","display_name":"Mrcr 64K 2 Needle","family_id":"mrcr-64k-2-needle","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fmrcr-64k-2-needle"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["linguistic_core","applied_reasoning"]},{"key":"mrcr-64k-4-needle","display_name":"Mrcr 64K 4 Needle","family_id":"mrcr-64k-4-needle","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fmrcr-64k-4-needle"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["linguistic_core","applied_reasoning"]},{"key":"mrcr-64k-8-needle","display_name":"Mrcr 64K 8 Needle","family_id":"mrcr-64k-8-needle","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fmrcr-64k-8-needle"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["linguistic_core","applied_reasoning"]},{"key":"mrcr-v2-8-needle","display_name":"Mrcr v2 8 Needle","family_id":"mrcr-v2-8-needle","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["Not specified"],"languages":["Not specified"],"tasks":["Not specified"]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":9,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fmrcr-v2-8-needle"],"reproducibility_summary":{"results_total":9,"has_reproducibility_gap_count":9,"populated_ratio_avg":0.9285714285714286},"provenance_summary":{"total_results":9,"total_groups":9,"multi_source_groups":0,"first_party_only_groups":9,"source_type_distribution":{"first_party":9,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":9,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["linguistic_core","applied_reasoning"]},{"key":"mt-bench","display_name":"Mt Bench","family_id":"mt-bench","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["Not specified"],"languages":["Not specified"],"tasks":["Multi-turn dialogue","Instruction adherence","Conversational effectiveness across multiple turns"]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":12,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fmt-bench"],"reproducibility_summary":{"results_total":12,"has_reproducibility_gap_count":12,"populated_ratio_avg":0.9285714285714287},"provenance_summary":{"total_results":12,"total_groups":12,"multi_source_groups":0,"first_party_only_groups":12,"source_type_distribution":{"first_party":12,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":12,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["linguistic_core","applied_reasoning"]},{"key":"mtvqa","display_name":"Mtvqa","family_id":"mtvqa","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fmtvqa"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["multimodal","applied_reasoning"]},{"key":"muirbench","display_name":"MuirBench","family_id":"muirbench","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":6,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fmuirbench"],"reproducibility_summary":{"results_total":6,"has_reproducibility_gap_count":6,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":6,"total_groups":6,"multi_source_groups":0,"first_party_only_groups":6,"source_type_distribution":{"first_party":6,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":6,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["multimodal","applied_reasoning"]},{"key":"multi-if","display_name":"Multi If","family_id":"multi-if","is_slice":false,"is_overall":false,"primary_metric_key":"Multi-IF benchmarks LLMs","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"Multi-IF benchmarks LLMs","display_name":"Multi-IF benchmarks LLMs","sources":["LLM Stats"],"models_count":14,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fmulti-if"],"reproducibility_summary":{"results_total":14,"has_reproducibility_gap_count":14,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":14,"total_groups":14,"multi_source_groups":0,"first_party_only_groups":14,"source_type_distribution":{"first_party":14,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":14,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["applied_reasoning"]},{"key":"multi-swe-bench-leaderboard","display_name":"Multi-SWE-Bench-Leaderboard","family_id":"multi-swe-bench-leaderboard","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["software engineering","code issue resolution"],"languages":["Not specified"],"tasks":["Issue resolving","Generating patches to fix code from GitHub pull requests"]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":6,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fmulti-swe-bench-leaderboard"],"reproducibility_summary":{"results_total":6,"has_reproducibility_gap_count":6,"populated_ratio_avg":0.9285714285714287},"provenance_summary":{"total_results":6,"total_groups":6,"multi_source_groups":0,"first_party_only_groups":6,"source_type_distribution":{"first_party":6,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":6,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["software_engineering"]},{"key":"multilf","display_name":"Multilf","family_id":"multilf","is_slice":false,"is_overall":false,"primary_metric_key":"MultiLF benchmark","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"MultiLF benchmark","display_name":"MultiLF benchmark","sources":["LLM Stats"],"models_count":2,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fmultilf"],"reproducibility_summary":{"results_total":2,"has_reproducibility_gap_count":2,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":2,"total_groups":2,"multi_source_groups":0,"first_party_only_groups":2,"source_type_distribution":{"first_party":2,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":2,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["logical_reasoning","applied_reasoning"]},{"key":"multilingual-mgsm-cot","display_name":"Multilingual Mgsm Cot","family_id":"multilingual-mgsm-cot","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fmultilingual-mgsm-cot"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["mathematics","linguistic_core"]},{"key":"multilingual-mmlu","display_name":"Multilingual Mmlu","family_id":"multilingual-mmlu","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":5,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fmultilingual-mmlu"],"reproducibility_summary":{"results_total":5,"has_reproducibility_gap_count":5,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":5,"total_groups":5,"multi_source_groups":0,"first_party_only_groups":5,"source_type_distribution":{"first_party":5,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":5,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["knowledge","linguistic_core"]},{"key":"multipl-e","display_name":"Multipl E","family_id":"multipl-e","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":11,"is_primary":true}],"slices":[{"key":"multipl-e-humaneval","display_name":"MultiPL-E HumanEval","is_bare_stem":false,"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"]}],"derivedTags":["software_engineering"]},{"key":"multipl-e-mbpp","display_name":"MultiPL-E MBPP","is_bare_stem":false,"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"]}],"derivedTags":["software_engineering"]}],"summary_eval_ids":["llm-stats%2Fmultipl-e"],"reproducibility_summary":{"results_total":11,"has_reproducibility_gap_count":11,"populated_ratio_avg":0.10714285714285712},"provenance_summary":{"total_results":11,"total_groups":11,"multi_source_groups":0,"first_party_only_groups":11,"source_type_distribution":{"first_party":11,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":11,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["software_engineering"]},{"key":"musiccaps","display_name":"Musiccaps","family_id":"musiccaps","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fmusiccaps"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["multimodal"]},{"key":"musr","display_name":"MuSR","family_id":"musr","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["reasoning","commonsense reasoning","planning"],"languages":["English"],"tasks":["Question answering","Solving murder mysteries","Solving object placement problems","Solving team allocation problems"]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":2,"is_primary":true}],"slices":[{"key":"llm stats","display_name":"llm stats","is_bare_stem":false,"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"]}],"derivedTags":["general"]}],"summary_eval_ids":["llm-stats%2Fmusr"],"reproducibility_summary":{"results_total":2,"has_reproducibility_gap_count":2,"populated_ratio_avg":0.9285714285714286},"provenance_summary":{"total_results":2,"total_groups":2,"multi_source_groups":1,"first_party_only_groups":1,"source_type_distribution":{"first_party":2,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":2,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["commonsense_reasoning","applied_reasoning","general"]},{"key":"mvbench","display_name":"MVBench","family_id":"mvbench","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":13,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fmvbench"],"reproducibility_summary":{"results_total":13,"has_reproducibility_gap_count":13,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":13,"total_groups":13,"multi_source_groups":0,"first_party_only_groups":13,"source_type_distribution":{"first_party":13,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":13,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["multimodal","applied_reasoning"]},{"key":"natural-questions","display_name":"Natural Questions","family_id":"natural-questions","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":5,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fnatural-questions"],"reproducibility_summary":{"results_total":5,"has_reproducibility_gap_count":5,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":5,"total_groups":5,"multi_source_groups":0,"first_party_only_groups":5,"source_type_distribution":{"first_party":5,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":5,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["knowledge"]},{"key":"natural2code","display_name":"Natural2code","family_id":"natural2code","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["Not specified"],"languages":["Not specified"],"tasks":["Not specified"]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":8,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fnatural2code"],"reproducibility_summary":{"results_total":8,"has_reproducibility_gap_count":8,"populated_ratio_avg":0.9285714285714287},"provenance_summary":{"total_results":8,"total_groups":8,"multi_source_groups":0,"first_party_only_groups":8,"source_type_distribution":{"first_party":8,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":8,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["software_engineering"]},{"key":"nexus","display_name":"Nexus","family_id":"nexus","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":2,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fnexus"],"reproducibility_summary":{"results_total":2,"has_reproducibility_gap_count":2,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":2,"total_groups":2,"multi_source_groups":0,"first_party_only_groups":2,"source_type_distribution":{"first_party":2,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":2,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["agentic"]},{"key":"nl2repo","display_name":"Nl2repo","family_id":"nl2repo","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":5,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fnl2repo"],"reproducibility_summary":{"results_total":5,"has_reproducibility_gap_count":5,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":5,"total_groups":5,"multi_source_groups":0,"first_party_only_groups":5,"source_type_distribution":{"first_party":5,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":5,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["software_engineering"]},{"key":"nmos","display_name":"Nmos","family_id":"nmos","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fnmos"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["multimodal"]},{"key":"nolima-128k","display_name":"Nolima 128k","family_id":"nolima-128k","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fnolima-128k"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["robustness"]},{"key":"nolima-32k","display_name":"Nolima 32K","family_id":"nolima-32k","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fnolima-32k"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["robustness"]},{"key":"nolima-64k","display_name":"Nolima 64K","family_id":"nolima-64k","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fnolima-64k"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["robustness"]},{"key":"nova-63","display_name":"Nova 63","family_id":"nova-63","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":9,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fnova-63"],"reproducibility_summary":{"results_total":9,"has_reproducibility_gap_count":9,"populated_ratio_avg":0.10714285714285712},"provenance_summary":{"total_results":9,"total_groups":9,"multi_source_groups":0,"first_party_only_groups":9,"source_type_distribution":{"first_party":9,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":9,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["knowledge"]},{"key":"nq","display_name":"Nq","family_id":"nq","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fnq"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["knowledge"]},{"key":"nuscene","display_name":"Nuscene","family_id":"nuscene","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":3,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fnuscene"],"reproducibility_summary":{"results_total":3,"has_reproducibility_gap_count":3,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":3,"total_groups":3,"multi_source_groups":0,"first_party_only_groups":3,"source_type_distribution":{"first_party":3,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":3,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["multimodal"]},{"key":"objectron","display_name":"Objectron","family_id":"objectron","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fobjectron"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["multimodal"]},{"key":"ocrbench-v2","display_name":"OCRBench v2","family_id":"ocrbench-v2","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["Text Recognition","Scene Text-Centric Visual Question Answering (VQA)","Document-Oriented VQA","Key Information Extraction (KIE)","Handwritten Mathematical Expression Recognition (HMER)"],"languages":["Not specified"],"tasks":["Text Recognition","Scene Text-Centric VQA","Document-Oriented VQA","Key Information Extraction (KIE)","Handwritten Mathematical Expression Recognition (HMER)"]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":17,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Focrbench-v2"],"reproducibility_summary":{"results_total":17,"has_reproducibility_gap_count":17,"populated_ratio_avg":0.9285714285714287},"provenance_summary":{"total_results":17,"total_groups":17,"multi_source_groups":0,"first_party_only_groups":17,"source_type_distribution":{"first_party":17,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":17,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["multimodal"]},{"key":"ocrbench-v2-en","display_name":"Ocrbench v2 En","family_id":"ocrbench-v2-en","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":7,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Focrbench-v2-en"],"reproducibility_summary":{"results_total":7,"has_reproducibility_gap_count":7,"populated_ratio_avg":0.10714285714285712},"provenance_summary":{"total_results":7,"total_groups":7,"multi_source_groups":0,"first_party_only_groups":7,"source_type_distribution":{"first_party":7,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":7,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["multimodal"]},{"key":"ocrbench-v2-zh","display_name":"Ocrbench v2 Zh","family_id":"ocrbench-v2-zh","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":6,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Focrbench-v2-zh"],"reproducibility_summary":{"results_total":6,"has_reproducibility_gap_count":6,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":6,"total_groups":6,"multi_source_groups":0,"first_party_only_groups":6,"source_type_distribution":{"first_party":6,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":6,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["multimodal"]},{"key":"octocodingbench","display_name":"Octocodingbench","family_id":"octocodingbench","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Foctocodingbench"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["software_engineering"]},{"key":"odinw","display_name":"Odinw","family_id":"odinw","is_slice":false,"is_overall":false,"primary_metric_key":"accuracy","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["LLM Stats"],"models_count":11,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fodinw"],"reproducibility_summary":{"results_total":11,"has_reproducibility_gap_count":11,"populated_ratio_avg":0.10714285714285712},"provenance_summary":{"total_results":11,"total_groups":11,"multi_source_groups":0,"first_party_only_groups":11,"source_type_distribution":{"first_party":11,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":11,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["multimodal"]},{"key":"officeqa-pro","display_name":"Officeqa Pro","family_id":"officeqa-pro","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fofficeqa-pro"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["agentic"]},{"key":"ojbench","display_name":"Ojbench","family_id":"ojbench","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":8,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fojbench"],"reproducibility_summary":{"results_total":8,"has_reproducibility_gap_count":8,"populated_ratio_avg":0.10714285714285712},"provenance_summary":{"total_results":8,"total_groups":8,"multi_source_groups":0,"first_party_only_groups":8,"source_type_distribution":{"first_party":8,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":8,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["software_engineering"]},{"key":"olympiadbench","display_name":"OlympiadBench","family_id":"olympiadbench","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Folympiadbench"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["mathematics"]},{"key":"omnibench","display_name":"OmniBench","family_id":"omnibench","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fomnibench"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["general"]},{"key":"omnibench-music","display_name":"Omnibench Music","family_id":"omnibench-music","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fomnibench-music"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["multimodal"]},{"key":"omnigaia","display_name":"Omnigaia","family_id":"omnigaia","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fomnigaia"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["applied_reasoning"]},{"key":"omnimath","display_name":"Omnimath","family_id":"omnimath","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fomnimath"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["mathematics"]},{"key":"open-rewrite","display_name":"Open Rewrite","family_id":"open-rewrite","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fopen-rewrite"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["software_engineering"]},{"key":"openai-mmlu","display_name":"Openai Mmlu","family_id":"openai-mmlu","is_slice":false,"is_overall":false,"primary_metric_key":"accuracy","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["LLM Stats"],"models_count":2,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fopenai-mmlu"],"reproducibility_summary":{"results_total":2,"has_reproducibility_gap_count":2,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":2,"total_groups":2,"multi_source_groups":0,"first_party_only_groups":2,"source_type_distribution":{"first_party":2,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":2,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["knowledge"]},{"key":"openai-mrcr-2-needle-128k","display_name":"Openai Mrcr 2 Needle 128k","family_id":"openai-mrcr-2-needle-128k","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":8,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fopenai-mrcr-2-needle-128k"],"reproducibility_summary":{"results_total":8,"has_reproducibility_gap_count":8,"populated_ratio_avg":0.10714285714285712},"provenance_summary":{"total_results":8,"total_groups":8,"multi_source_groups":0,"first_party_only_groups":8,"source_type_distribution":{"first_party":8,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":8,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["hallucination"]},{"key":"openai-mrcr-2-needle-1m","display_name":"Openai Mrcr 2 Needle 1M","family_id":"openai-mrcr-2-needle-1m","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":4,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fopenai-mrcr-2-needle-1m"],"reproducibility_summary":{"results_total":4,"has_reproducibility_gap_count":4,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":4,"total_groups":4,"multi_source_groups":0,"first_party_only_groups":4,"source_type_distribution":{"first_party":4,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":4,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["hallucination"]},{"key":"openai-mrcr-2-needle-256k","display_name":"Openai Mrcr 2 Needle 256k","family_id":"openai-mrcr-2-needle-256k","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fopenai-mrcr-2-needle-256k"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["hallucination"]},{"key":"openbookqa","display_name":"OpenbookQA","family_id":"openbookqa","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["Not specified"],"languages":["English"],"tasks":["Open-domain question answering"]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":5,"is_primary":true}],"slices":[{"key":"llm stats","display_name":"llm stats","is_bare_stem":false,"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"]}],"derivedTags":["general"]}],"summary_eval_ids":["llm-stats%2Fopenbookqa"],"reproducibility_summary":{"results_total":5,"has_reproducibility_gap_count":5,"populated_ratio_avg":0.9285714285714286},"provenance_summary":{"total_results":5,"total_groups":5,"multi_source_groups":1,"first_party_only_groups":4,"source_type_distribution":{"first_party":5,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":5,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["commonsense_reasoning","general"]},{"key":"openrca","display_name":"Openrca","family_id":"openrca","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["Not specified"],"languages":["Not specified"],"tasks":["Not specified"]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fopenrca"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.9285714285714286},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["applied_reasoning"]},{"key":"osworld","display_name":"OSWorld","family_id":"osworld","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["computer task automation","human-computer interaction","web browsing","video editing","file management","data analysis","software development","web browsers","email clients","office suites","image editing","code editing","multi-application tasks"],"languages":["Not specified"],"tasks":["Web and desktop application interaction in open domains","Operating system file I/O operations","Workflows spanning multiple applications following natural language instructions","Specific scenarios involving files such as images, documents, spreadsheets, presentations, and code"]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":13,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fosworld"],"reproducibility_summary":{"results_total":13,"has_reproducibility_gap_count":13,"populated_ratio_avg":0.9285714285714287},"provenance_summary":{"total_results":13,"total_groups":13,"multi_source_groups":0,"first_party_only_groups":13,"source_type_distribution":{"first_party":13,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":13,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["agentic"]},{"key":"osworld-extended","display_name":"Osworld Extended","family_id":"osworld-extended","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["general computer use","web browsing","desktop applications","file system operations"],"languages":["Not specified"],"tasks":["Interacting with real web and desktop applications","Performing operating system file I/O operations","Executing workflows that span multiple applications"]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fosworld-extended"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.9285714285714286},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["agentic"]},{"key":"osworld-g","display_name":"OSWORLD-G","family_id":"osworld-g","is_slice":false,"is_overall":false,"primary_metric_key":"accuracy","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fosworld-g"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["agentic"]},{"key":"osworld-screenshot-only","display_name":"Osworld Screenshot Only","family_id":"osworld-screenshot-only","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fosworld-screenshot-only"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["agentic"]},{"key":"osworld-verified","display_name":"Osworld Verified","family_id":"osworld-verified","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["office applications (LibreOffice Calc/Writer/Impress)","daily applications (Chrome, VLC, Thunderbird)","professional tools (GIMP, VS Code)","multi-app workflows","operating system tasks"],"languages":["Not specified"],"tasks":["Interacting with various software applications and completing workflows under different step limits (15, 50, and 100 steps)"]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":12,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fosworld-verified"],"reproducibility_summary":{"results_total":12,"has_reproducibility_gap_count":12,"populated_ratio_avg":0.9285714285714287},"provenance_summary":{"total_results":12,"total_groups":12,"multi_source_groups":0,"first_party_only_groups":12,"source_type_distribution":{"first_party":12,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":12,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["agentic"]},{"key":"paperbench","display_name":"Paperbench","family_id":"paperbench","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fpaperbench"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["knowledge","applied_reasoning","agentic"]},{"key":"pathmcqa","display_name":"Pathmcqa","family_id":"pathmcqa","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fpathmcqa"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["natural_sciences","knowledge"]},{"key":"perceptiontest","display_name":"Perceptiontest","family_id":"perceptiontest","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":2,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fperceptiontest"],"reproducibility_summary":{"results_total":2,"has_reproducibility_gap_count":2,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":2,"total_groups":2,"multi_source_groups":0,"first_party_only_groups":2,"source_type_distribution":{"first_party":2,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":2,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["multimodal","applied_reasoning"]},{"key":"phibench","display_name":"Phibench","family_id":"phibench","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fphibench"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["knowledge","applied_reasoning"]},{"key":"physicsfinals","display_name":"Physicsfinals","family_id":"physicsfinals","is_slice":false,"is_overall":false,"primary_metric_key":"accuracy","has_card":true,"tags":{"domains":["Not specified"],"languages":["Not specified"],"tasks":["Not specified"]},"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["LLM Stats"],"models_count":2,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fphysicsfinals"],"reproducibility_summary":{"results_total":2,"has_reproducibility_gap_count":2,"populated_ratio_avg":0.9285714285714286},"provenance_summary":{"total_results":2,"total_groups":2,"multi_source_groups":0,"first_party_only_groups":2,"source_type_distribution":{"first_party":2,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":2,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["natural_sciences","mathematics"]},{"key":"pinchbench","display_name":"Pinchbench","family_id":"pinchbench","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":2,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fpinchbench"],"reproducibility_summary":{"results_total":2,"has_reproducibility_gap_count":2,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":2,"total_groups":2,"multi_source_groups":0,"first_party_only_groups":2,"source_type_distribution":{"first_party":2,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":2,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["multimodal","applied_reasoning"]},{"key":"piqa","display_name":"Piqa","family_id":"piqa","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["physical commonsense reasoning"],"languages":["Not specified"],"tasks":["Binary question-answering where a model must choose the correct solution to achieve a physical goal"]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":9,"is_primary":true}],"slices":[{"key":"llm stats","display_name":"llm stats","is_bare_stem":false,"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"]}],"derivedTags":["general"]}],"summary_eval_ids":["llm-stats%2Fpiqa"],"reproducibility_summary":{"results_total":9,"has_reproducibility_gap_count":9,"populated_ratio_avg":0.9285714285714286},"provenance_summary":{"total_results":9,"total_groups":9,"multi_source_groups":0,"first_party_only_groups":9,"source_type_distribution":{"first_party":9,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":9,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["commonsense_reasoning","general"]},{"key":"pmc-vqa","display_name":"Pmc Vqa","family_id":"pmc-vqa","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":3,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fpmc-vqa"],"reproducibility_summary":{"results_total":3,"has_reproducibility_gap_count":3,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":3,"total_groups":3,"multi_source_groups":0,"first_party_only_groups":3,"source_type_distribution":{"first_party":3,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":3,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["multimodal"]},{"key":"pointgrounding","display_name":"Pointgrounding","family_id":"pointgrounding","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fpointgrounding"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["multimodal"]},{"key":"polymath","display_name":"POLYMATH","family_id":"polymath","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":14,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fpolymath"],"reproducibility_summary":{"results_total":14,"has_reproducibility_gap_count":14,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":14,"total_groups":14,"multi_source_groups":0,"first_party_only_groups":14,"source_type_distribution":{"first_party":14,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":14,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["mathematics","applied_reasoning"]},{"key":"polymath-en","display_name":"Polymath En","family_id":"polymath-en","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fpolymath-en"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["mathematics","applied_reasoning"]},{"key":"pope","display_name":"Pope","family_id":"pope","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":2,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fpope"],"reproducibility_summary":{"results_total":2,"has_reproducibility_gap_count":2,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":2,"total_groups":2,"multi_source_groups":0,"first_party_only_groups":2,"source_type_distribution":{"first_party":2,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":2,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["hallucination"]},{"key":"popqa","display_name":"POPQA","family_id":"popqa","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":2,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fpopqa"],"reproducibility_summary":{"results_total":2,"has_reproducibility_gap_count":2,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":2,"total_groups":2,"multi_source_groups":0,"first_party_only_groups":2,"source_type_distribution":{"first_party":2,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":2,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["knowledge"]},{"key":"protocolqa","display_name":"Protocolqa","family_id":"protocolqa","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fprotocolqa"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["knowledge","applied_reasoning"]},{"key":"qasper","display_name":"Qasper","family_id":"qasper","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":2,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fqasper"],"reproducibility_summary":{"results_total":2,"has_reproducibility_gap_count":2,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":2,"total_groups":2,"multi_source_groups":0,"first_party_only_groups":2,"source_type_distribution":{"first_party":2,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":2,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["linguistic_core"]},{"key":"qmsum","display_name":"Qmsum","family_id":"qmsum","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":2,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fqmsum"],"reproducibility_summary":{"results_total":2,"has_reproducibility_gap_count":2,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":2,"total_groups":2,"multi_source_groups":0,"first_party_only_groups":2,"source_type_distribution":{"first_party":2,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":2,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["linguistic_core"]},{"key":"qwenwebbench","display_name":"Qwenwebbench","family_id":"qwenwebbench","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fqwenwebbench"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["knowledge","applied_reasoning"]},{"key":"realworldqa","display_name":"Realworldqa","family_id":"realworldqa","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["Not specified"],"languages":[],"tasks":["Not specified"]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":15,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Frealworldqa"],"reproducibility_summary":{"results_total":15,"has_reproducibility_gap_count":15,"populated_ratio_avg":0.9285714285714287},"provenance_summary":{"total_results":15,"total_groups":15,"multi_source_groups":0,"first_party_only_groups":15,"source_type_distribution":{"first_party":15,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":15,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["knowledge","applied_reasoning"]},{"key":"refcoco-avg","display_name":"Refcoco Avg","family_id":"refcoco-avg","is_slice":false,"is_overall":false,"primary_metric_key":"accuracy","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["LLM Stats"],"models_count":7,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Frefcoco-avg"],"reproducibility_summary":{"results_total":7,"has_reproducibility_gap_count":7,"populated_ratio_avg":0.10714285714285712},"provenance_summary":{"total_results":7,"total_groups":7,"multi_source_groups":0,"first_party_only_groups":7,"source_type_distribution":{"first_party":7,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":7,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["multimodal"]},{"key":"refspatialbench","display_name":"Refspatialbench","family_id":"refspatialbench","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":6,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Frefspatialbench"],"reproducibility_summary":{"results_total":6,"has_reproducibility_gap_count":6,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":6,"total_groups":6,"multi_source_groups":0,"first_party_only_groups":6,"source_type_distribution":{"first_party":6,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":6,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["multimodal","commonsense_reasoning"]},{"key":"repobench","display_name":"RepoBench","family_id":"repobench","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Frepobench"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["software_engineering"]},{"key":"repoqa","display_name":"Repoqa","family_id":"repoqa","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":2,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Frepoqa"],"reproducibility_summary":{"results_total":2,"has_reproducibility_gap_count":2,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":2,"total_groups":2,"multi_source_groups":0,"first_party_only_groups":2,"source_type_distribution":{"first_party":2,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":2,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["software_engineering"]},{"key":"robospatialhome","display_name":"Robospatialhome","family_id":"robospatialhome","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Frobospatialhome"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["agentic","multimodal"]},{"key":"ruler","display_name":"Ruler","family_id":"ruler","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":3,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fruler"],"reproducibility_summary":{"results_total":3,"has_reproducibility_gap_count":3,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":3,"total_groups":3,"multi_source_groups":0,"first_party_only_groups":3,"source_type_distribution":{"first_party":3,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":3,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["linguistic_core","robustness"]},{"key":"ruler-1000k","display_name":"Ruler 1000k","family_id":"ruler-1000k","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fruler-1000k"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["linguistic_core","robustness"]},{"key":"ruler-128k","display_name":"Ruler 128k","family_id":"ruler-128k","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fruler-128k"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["linguistic_core","robustness"]},{"key":"ruler-2048k","display_name":"Ruler 2048k","family_id":"ruler-2048k","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fruler-2048k"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["linguistic_core","robustness"]},{"key":"ruler-512k","display_name":"Ruler 512k","family_id":"ruler-512k","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fruler-512k"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["linguistic_core","robustness"]},{"key":"ruler-64k","display_name":"Ruler 64K","family_id":"ruler-64k","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fruler-64k"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["linguistic_core","robustness"]},{"key":"sat-math","display_name":"Sat Math","family_id":"sat-math","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fsat-math"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["mathematics"]},{"key":"scicode","display_name":"SciCode","family_id":"scicode","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["natural sciences","code generation","scientific computing","mathematics","physics","chemistry","biology","materials science"],"languages":["code"],"tasks":["Code generation for scientific problems","Numerical methods implementation","System simulation","Scientific calculation"]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":11,"is_primary":true}],"slices":[{"key":"llm stats","display_name":"llm stats","is_bare_stem":false,"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"]}],"derivedTags":["general"]}],"summary_eval_ids":["llm-stats%2Fscicode"],"reproducibility_summary":{"results_total":11,"has_reproducibility_gap_count":11,"populated_ratio_avg":0.9285714285714287},"provenance_summary":{"total_results":11,"total_groups":11,"multi_source_groups":8,"first_party_only_groups":3,"source_type_distribution":{"first_party":11,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":11,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["natural_sciences","software_engineering","general"]},{"key":"scienceqa","display_name":"ScienceQA","family_id":"scienceqa","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fscienceqa"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["natural_sciences","knowledge"]},{"key":"scienceqa-visual","display_name":"Scienceqa Visual","family_id":"scienceqa-visual","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fscienceqa-visual"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["natural_sciences","multimodal"]},{"key":"screenspot","display_name":"ScreenSpot","family_id":"screenspot","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":8,"is_primary":true}],"slices":[{"key":"screenspot","display_name":"ScreenSpot","is_bare_stem":true,"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"]}],"derivedTags":["agentic","multimodal"]},{"key":"screenspot-pro","display_name":"ScreenSpot-Pro","is_bare_stem":false,"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"]}],"derivedTags":["agentic","multimodal"]}],"summary_eval_ids":["llm-stats%2Fscreenspot"],"reproducibility_summary":{"results_total":8,"has_reproducibility_gap_count":8,"populated_ratio_avg":0.10714285714285712},"provenance_summary":{"total_results":8,"total_groups":8,"multi_source_groups":0,"first_party_only_groups":8,"source_type_distribution":{"first_party":8,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":8,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["agentic","multimodal"]},{"key":"seal-0","display_name":"Seal 0","family_id":"seal-0","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":6,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fseal-0"],"reproducibility_summary":{"results_total":6,"has_reproducibility_gap_count":6,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":6,"total_groups":6,"multi_source_groups":0,"first_party_only_groups":6,"source_type_distribution":{"first_party":6,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":6,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["safety"]},{"key":"seccodebench","display_name":"Seccodebench","family_id":"seccodebench","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fseccodebench"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["software_engineering","safety"]},{"key":"sifo","display_name":"Sifo","family_id":"sifo","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fsifo"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["knowledge","applied_reasoning"]},{"key":"sifo-multiturn","display_name":"Sifo Multiturn","family_id":"sifo-multiturn","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fsifo-multiturn"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["knowledge","applied_reasoning"]},{"key":"simpleqa","display_name":"SimpleQA","family_id":"simpleqa","is_slice":false,"is_overall":false,"primary_metric_key":"accuracy","has_card":true,"tags":{"domains":["social intelligence","emotional intelligence","commonsense reasoning"],"languages":["Not specified"],"tasks":["Answering multiple-choice questions about social situations","Reasoning about motivation, what happens next, and emotional reactions"]},"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["LLM Stats"],"models_count":18,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fsimpleqa"],"reproducibility_summary":{"results_total":18,"has_reproducibility_gap_count":18,"populated_ratio_avg":0.9285714285714286},"provenance_summary":{"total_results":18,"total_groups":18,"multi_source_groups":0,"first_party_only_groups":18,"source_type_distribution":{"first_party":18,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":18,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["knowledge","hallucination"]},{"key":"simplevqa","display_name":"Simplevqa","family_id":"simplevqa","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":10,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fsimplevqa"],"reproducibility_summary":{"results_total":10,"has_reproducibility_gap_count":10,"populated_ratio_avg":0.10714285714285712},"provenance_summary":{"total_results":10,"total_groups":10,"multi_source_groups":0,"first_party_only_groups":10,"source_type_distribution":{"first_party":10,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":10,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["multimodal"]},{"key":"skillsbench","display_name":"Skillsbench","family_id":"skillsbench","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":3,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fskillsbench"],"reproducibility_summary":{"results_total":3,"has_reproducibility_gap_count":3,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":3,"total_groups":3,"multi_source_groups":0,"first_party_only_groups":3,"source_type_distribution":{"first_party":3,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":3,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["applied_reasoning"]},{"key":"slakevqa","display_name":"Slakevqa","family_id":"slakevqa","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":4,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fslakevqa"],"reproducibility_summary":{"results_total":4,"has_reproducibility_gap_count":4,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":4,"total_groups":4,"multi_source_groups":0,"first_party_only_groups":4,"source_type_distribution":{"first_party":4,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":4,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["multimodal"]},{"key":"social-iqa","display_name":"Social Iqa","family_id":"social-iqa","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":7,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fsocial-iqa"],"reproducibility_summary":{"results_total":7,"has_reproducibility_gap_count":7,"populated_ratio_avg":0.10714285714285712},"provenance_summary":{"total_results":7,"total_groups":7,"multi_source_groups":0,"first_party_only_groups":7,"source_type_distribution":{"first_party":7,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":7,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["commonsense_reasoning"]},{"key":"spider","display_name":"Spider","family_id":"spider","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":2,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fspider"],"reproducibility_summary":{"results_total":2,"has_reproducibility_gap_count":2,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":2,"total_groups":2,"multi_source_groups":0,"first_party_only_groups":2,"source_type_distribution":{"first_party":2,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":2,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["software_engineering","logical_reasoning"]},{"key":"squality","display_name":"Squality","family_id":"squality","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["Not specified"],"languages":["Not specified"],"tasks":["Not specified"]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":5,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fsquality"],"reproducibility_summary":{"results_total":5,"has_reproducibility_gap_count":5,"populated_ratio_avg":0.9285714285714286},"provenance_summary":{"total_results":5,"total_groups":5,"multi_source_groups":0,"first_party_only_groups":5,"source_type_distribution":{"first_party":5,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":5,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["linguistic_core"]},{"key":"stem","display_name":"Stem","family_id":"stem","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fstem"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["natural_sciences","mathematics"]},{"key":"summscreenfd","display_name":"Summscreenfd","family_id":"summscreenfd","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":2,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fsummscreenfd"],"reproducibility_summary":{"results_total":2,"has_reproducibility_gap_count":2,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":2,"total_groups":2,"multi_source_groups":0,"first_party_only_groups":2,"source_type_distribution":{"first_party":2,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":2,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["linguistic_core"]},{"key":"sunrgbd","display_name":"Sunrgbd","family_id":"sunrgbd","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":4,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fsunrgbd"],"reproducibility_summary":{"results_total":4,"has_reproducibility_gap_count":4,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":4,"total_groups":4,"multi_source_groups":0,"first_party_only_groups":4,"source_type_distribution":{"first_party":4,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":4,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["multimodal"]},{"key":"superglue","display_name":"Superglue","family_id":"superglue","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fsuperglue"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["linguistic_core","general"]},{"key":"supergpqa","display_name":"SuperGPQA","family_id":"supergpqa","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":15,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fsupergpqa"],"reproducibility_summary":{"results_total":15,"has_reproducibility_gap_count":15,"populated_ratio_avg":0.10714285714285715},"provenance_summary":{"total_results":15,"total_groups":15,"multi_source_groups":0,"first_party_only_groups":15,"source_type_distribution":{"first_party":15,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":15,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["knowledge","applied_reasoning"]},{"key":"swe-bench-multilingual","display_name":"Swe Bench Multilingual","family_id":"swe-bench-multilingual","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["Not specified"],"languages":["English"],"tasks":["Not specified"]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":17,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fswe-bench-multilingual"],"reproducibility_summary":{"results_total":17,"has_reproducibility_gap_count":17,"populated_ratio_avg":0.9285714285714287},"provenance_summary":{"total_results":17,"total_groups":17,"multi_source_groups":0,"first_party_only_groups":17,"source_type_distribution":{"first_party":17,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":17,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["software_engineering"]},{"key":"swe-bench-multimodal","display_name":"SWE-bench Multimodal","family_id":"swe-bench-multimodal","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["software engineering","web interface design","diagramming","data visualization","syntax highlighting","interactive mapping"],"languages":["Not specified"],"tasks":["Fixing bugs in JavaScript software where the problem statement or unit tests contain visual elements (images)"]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fswe-bench-multimodal"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.9285714285714286},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["software_engineering","multimodal"]},{"key":"swe-bench-verified-agentic-coding","display_name":"Swe Bench Verified Agentic Coding","family_id":"swe-bench-verified-agentic-coding","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["Not specified"],"languages":["Not specified"],"tasks":["Not specified"]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":2,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fswe-bench-verified-agentic-coding"],"reproducibility_summary":{"results_total":2,"has_reproducibility_gap_count":2,"populated_ratio_avg":0.9285714285714286},"provenance_summary":{"total_results":2,"total_groups":2,"multi_source_groups":0,"first_party_only_groups":2,"source_type_distribution":{"first_party":2,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":2,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["software_engineering","agentic"]},{"key":"swe-bench-verified-agentless","display_name":"Swe Bench Verified Agentless","family_id":"swe-bench-verified-agentless","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fswe-bench-verified-agentless"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["software_engineering"]},{"key":"swe-bench-verified-multiple-attempts","display_name":"Swe Bench Verified Multiple Attempts","family_id":"swe-bench-verified-multiple-attempts","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fswe-bench-verified-multiple-attempts"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["software_engineering"]},{"key":"swe-lancer","display_name":"SWE-Lancer","family_id":"swe-lancer","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":4,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fswe-lancer"],"reproducibility_summary":{"results_total":4,"has_reproducibility_gap_count":4,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":4,"total_groups":4,"multi_source_groups":0,"first_party_only_groups":4,"source_type_distribution":{"first_party":4,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":4,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["software_engineering"]},{"key":"swe-lancer-ic-diamond-subset","display_name":"Swe Lancer Ic Diamond Subset","family_id":"swe-lancer-ic-diamond-subset","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":6,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fswe-lancer-ic-diamond-subset"],"reproducibility_summary":{"results_total":6,"has_reproducibility_gap_count":6,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":6,"total_groups":6,"multi_source_groups":0,"first_party_only_groups":6,"source_type_distribution":{"first_party":6,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":6,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["software_engineering"]},{"key":"swe-perf","display_name":"Swe Perf","family_id":"swe-perf","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fswe-perf"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["software_engineering"]},{"key":"swe-review","display_name":"Swe Review","family_id":"swe-review","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fswe-review"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["software_engineering"]},{"key":"swt-bench","display_name":"SWT-Bench","family_id":"swt-bench","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fswt-bench"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["software_engineering"]},{"key":"t2-bench","display_name":"T2 Bench","family_id":"t2-bench","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["Telecom"],"languages":["Not specified"],"tasks":["Resolving user requests through Tool-Agent-User (TAU) interactions while adhering to domain policy"]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":19,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Ft2-bench"],"reproducibility_summary":{"results_total":19,"has_reproducibility_gap_count":19,"populated_ratio_avg":0.9285714285714285},"provenance_summary":{"total_results":19,"total_groups":19,"multi_source_groups":0,"first_party_only_groups":19,"source_type_distribution":{"first_party":19,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":19,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["general"]},{"key":"tau-bench","display_name":"Tau Bench","family_id":"tau-bench","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":5,"is_primary":true}],"slices":[{"key":"tau-bench-airline","display_name":"Tau Bench Airline","is_bare_stem":false,"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"]}],"derivedTags":["agentic"]},{"key":"tau-bench-retail","display_name":"Tau Bench Retail","is_bare_stem":false,"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"]}],"derivedTags":["agentic"]}],"summary_eval_ids":["llm-stats%2Ftau-bench"],"reproducibility_summary":{"results_total":5,"has_reproducibility_gap_count":5,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":5,"total_groups":5,"multi_source_groups":0,"first_party_only_groups":5,"source_type_distribution":{"first_party":5,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":5,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["agentic"]},{"key":"tau2-airline","display_name":"Tau2-Airline","family_id":"tau2-bench","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["Not specified"],"languages":["Not specified"],"tasks":["Not specified"]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":14,"is_primary":true}],"slices":[{"key":"llm stats","display_name":"llm stats","is_bare_stem":false,"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"]}],"derivedTags":["general"]}],"summary_eval_ids":["llm-stats%2Ftau2-airline"],"reproducibility_summary":{"results_total":14,"has_reproducibility_gap_count":14,"populated_ratio_avg":0.9285714285714287},"provenance_summary":{"total_results":14,"total_groups":14,"multi_source_groups":0,"first_party_only_groups":14,"source_type_distribution":{"first_party":14,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":14,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["general"]},{"key":"tau2-retail","display_name":"Tau2-Retail","family_id":"tau2-bench","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["Not specified"],"languages":["Not specified"],"tasks":["Not specified"]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":15,"is_primary":true}],"slices":[{"key":"llm stats","display_name":"llm stats","is_bare_stem":false,"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"]}],"derivedTags":["general"]}],"summary_eval_ids":["llm-stats%2Ftau2-retail"],"reproducibility_summary":{"results_total":15,"has_reproducibility_gap_count":15,"populated_ratio_avg":0.9285714285714287},"provenance_summary":{"total_results":15,"total_groups":15,"multi_source_groups":2,"first_party_only_groups":13,"source_type_distribution":{"first_party":15,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":15,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["general"]},{"key":"tau2-telecom","display_name":"Tau2-Telecom","family_id":"tau2-bench","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["telecommunications","signal processing","network optimization","performance analysis"],"languages":["Not specified"],"tasks":["Solving mathematical problems with numerical solutions in the telecom domain"]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":18,"is_primary":true}],"slices":[{"key":"llm stats","display_name":"llm stats","is_bare_stem":false,"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"]}],"derivedTags":["general"]}],"summary_eval_ids":["llm-stats%2Ftau2-telecom"],"reproducibility_summary":{"results_total":18,"has_reproducibility_gap_count":18,"populated_ratio_avg":0.9285714285714286},"provenance_summary":{"total_results":18,"total_groups":18,"multi_source_groups":2,"first_party_only_groups":16,"source_type_distribution":{"first_party":18,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":18,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["general"]},{"key":"tau3-bench","display_name":"Tau3 Bench","family_id":"tau3-bench","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":3,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Ftau3-bench"],"reproducibility_summary":{"results_total":3,"has_reproducibility_gap_count":3,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":3,"total_groups":3,"multi_source_groups":0,"first_party_only_groups":3,"source_type_distribution":{"first_party":3,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":3,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["agentic"]},{"key":"tempcompass","display_name":"Tempcompass","family_id":"tempcompass","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":2,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Ftempcompass"],"reproducibility_summary":{"results_total":2,"has_reproducibility_gap_count":2,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":2,"total_groups":2,"multi_source_groups":0,"first_party_only_groups":2,"source_type_distribution":{"first_party":2,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":2,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["knowledge","robustness"]},{"key":"terminus","display_name":"Terminus","family_id":"terminus","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fterminus"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["agentic"]},{"key":"textvqa","display_name":"Textvqa","family_id":"textvqa","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":[],"languages":[],"tasks":["Not specified"]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":15,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Ftextvqa"],"reproducibility_summary":{"results_total":15,"has_reproducibility_gap_count":15,"populated_ratio_avg":0.9285714285714287},"provenance_summary":{"total_results":15,"total_groups":15,"multi_source_groups":0,"first_party_only_groups":15,"source_type_distribution":{"first_party":15,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":15,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["multimodal"]},{"key":"theoremqa","display_name":"Theoremqa","family_id":"theoremqa","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":6,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Ftheoremqa"],"reproducibility_summary":{"results_total":6,"has_reproducibility_gap_count":6,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":6,"total_groups":6,"multi_source_groups":0,"first_party_only_groups":6,"source_type_distribution":{"first_party":6,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":6,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["mathematics","logical_reasoning"]},{"key":"tir-bench","display_name":"TIR-Bench","family_id":"tir-bench","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":4,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Ftir-bench"],"reproducibility_summary":{"results_total":4,"has_reproducibility_gap_count":4,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":4,"total_groups":4,"multi_source_groups":0,"first_party_only_groups":4,"source_type_distribution":{"first_party":4,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":4,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["agentic"]},{"key":"tldr9-test","display_name":"Tldr9 Test","family_id":"tldr9-test","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Ftldr9-test"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["linguistic_core"]},{"key":"toolathlon","display_name":"Toolathlon","family_id":"toolathlon","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["agent","tool use","software applications"],"languages":["English"],"tasks":["Long-horizon, multi-step workflows requiring interaction with multiple software applications to complete verifiable goals"]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":16,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Ftoolathlon"],"reproducibility_summary":{"results_total":16,"has_reproducibility_gap_count":16,"populated_ratio_avg":0.9285714285714287},"provenance_summary":{"total_results":16,"total_groups":16,"multi_source_groups":0,"first_party_only_groups":16,"source_type_distribution":{"first_party":16,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":16,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["agentic"]},{"key":"translation-en-set1-comet22","display_name":"Translation En Set1 Comet22","family_id":"translation-en-set1-comet22","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["machine translation evaluation"],"languages":["English","Set1 (unspecified target languages)"],"tasks":["Not specified"]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":3,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Ftranslation-en-set1-comet22"],"reproducibility_summary":{"results_total":3,"has_reproducibility_gap_count":3,"populated_ratio_avg":0.9285714285714285},"provenance_summary":{"total_results":3,"total_groups":3,"multi_source_groups":0,"first_party_only_groups":3,"source_type_distribution":{"first_party":3,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":3,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["linguistic_core"]},{"key":"translation-en-set1-spbleu","display_name":"Translation En Set1 Spbleu","family_id":"translation-en-set1-spbleu","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["Not specified"],"languages":["Not specified"],"tasks":["Not specified"]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":3,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Ftranslation-en-set1-spbleu"],"reproducibility_summary":{"results_total":3,"has_reproducibility_gap_count":3,"populated_ratio_avg":0.9285714285714285},"provenance_summary":{"total_results":3,"total_groups":3,"multi_source_groups":0,"first_party_only_groups":3,"source_type_distribution":{"first_party":3,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":3,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["linguistic_core"]},{"key":"translation-set1-en-comet22","display_name":"Translation Set1 En Comet22","family_id":"translation-set1-en-comet22","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":3,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Ftranslation-set1-en-comet22"],"reproducibility_summary":{"results_total":3,"has_reproducibility_gap_count":3,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":3,"total_groups":3,"multi_source_groups":0,"first_party_only_groups":3,"source_type_distribution":{"first_party":3,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":3,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["linguistic_core"]},{"key":"translation-set1-en-spbleu","display_name":"Translation Set1 En Spbleu","family_id":"translation-set1-en-spbleu","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["natural language generation","machine translation"],"languages":["Not specified"],"tasks":["Machine translation from Set1 languages to English"]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":3,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Ftranslation-set1-en-spbleu"],"reproducibility_summary":{"results_total":3,"has_reproducibility_gap_count":3,"populated_ratio_avg":0.9285714285714285},"provenance_summary":{"total_results":3,"total_groups":3,"multi_source_groups":0,"first_party_only_groups":3,"source_type_distribution":{"first_party":3,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":3,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["linguistic_core"]},{"key":"triviaqa","display_name":"Triviaqa","family_id":"triviaqa","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":14,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Ftriviaqa"],"reproducibility_summary":{"results_total":14,"has_reproducibility_gap_count":14,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":14,"total_groups":14,"multi_source_groups":0,"first_party_only_groups":14,"source_type_distribution":{"first_party":14,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":14,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["knowledge"]},{"key":"truthfulqa","display_name":"TruthfulQA","family_id":"truthfulqa","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["health","law","finance","politics"],"languages":["English"],"tasks":["Text generation","Multiple-choice question answering"]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":16,"is_primary":true}],"slices":[{"key":"llm stats","display_name":"llm stats","is_bare_stem":false,"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"]}],"derivedTags":["general"]}],"summary_eval_ids":["llm-stats%2Ftruthfulqa"],"reproducibility_summary":{"results_total":16,"has_reproducibility_gap_count":16,"populated_ratio_avg":0.9285714285714287},"provenance_summary":{"total_results":16,"total_groups":16,"multi_source_groups":0,"first_party_only_groups":16,"source_type_distribution":{"first_party":16,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":16,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["hallucination","general"]},{"key":"tydiqa","display_name":"Tydiqa","family_id":"tydiqa","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Ftydiqa"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["linguistic_core"]},{"key":"uniform-bar-exam","display_name":"Uniform Bar Exam","family_id":"uniform-bar-exam","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Funiform-bar-exam"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["law"]},{"key":"usamo25","display_name":"Usamo25","family_id":"usamo25","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["Not specified"],"languages":["Not specified"],"tasks":["Not specified"]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":2,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fusamo25"],"reproducibility_summary":{"results_total":2,"has_reproducibility_gap_count":2,"populated_ratio_avg":0.9285714285714286},"provenance_summary":{"total_results":2,"total_groups":2,"multi_source_groups":0,"first_party_only_groups":2,"source_type_distribution":{"first_party":2,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":2,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["mathematics"]},{"key":"vatex","display_name":"Vatex","family_id":"vatex","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["Not specified"],"languages":["Not specified"],"tasks":["Not specified"]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":2,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fvatex"],"reproducibility_summary":{"results_total":2,"has_reproducibility_gap_count":2,"populated_ratio_avg":0.9285714285714286},"provenance_summary":{"total_results":2,"total_groups":2,"multi_source_groups":0,"first_party_only_groups":2,"source_type_distribution":{"first_party":2,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":2,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["multimodal"]},{"key":"vcr-en-easy","display_name":"Vcr En Easy","family_id":"vcr-en-easy","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fvcr-en-easy"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["multimodal"]},{"key":"vending-bench-2","display_name":"Vending Bench 2","family_id":"vending-bench-2","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["autonomous agents","long-term task execution","business management","economic decision-making","AI safety"],"languages":["Not specified"],"tasks":["Operating a vending machine business","Balancing inventories and placing orders","Setting prices and handling daily fees","Sourcing products and negotiating with suppliers","Managing customer relations to maximize profit over a year-long simulation"]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":4,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fvending-bench-2"],"reproducibility_summary":{"results_total":4,"has_reproducibility_gap_count":4,"populated_ratio_avg":0.9285714285714286},"provenance_summary":{"total_results":4,"total_groups":4,"multi_source_groups":0,"first_party_only_groups":4,"source_type_distribution":{"first_party":4,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":4,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["agentic"]},{"key":"vibe","display_name":"Vibe","family_id":"vibe","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[{"key":"vibe","display_name":"Vibe","is_bare_stem":true,"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"]}],"derivedTags":["agentic"]},{"key":"vibe-pro","display_name":"Vibe Pro","is_bare_stem":false,"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"]}],"derivedTags":["agentic"]}],"summary_eval_ids":["llm-stats%2Fvibe"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["agentic"]},{"key":"vibe-android","display_name":"Vibe Android","family_id":"vibe-android","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fvibe-android"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["agentic"]},{"key":"vibe-backend","display_name":"Vibe Backend","family_id":"vibe-backend","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fvibe-backend"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["agentic"]},{"key":"vibe-eval","display_name":"Vibe Eval","family_id":"vibe-eval","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["image-to-text","image classification"],"languages":["Chinese","English","Japanese","Polish"],"tasks":["Open-ended visual understanding based on image-text prompts","Image to text","Image classification"]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":7,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fvibe-eval"],"reproducibility_summary":{"results_total":7,"has_reproducibility_gap_count":7,"populated_ratio_avg":0.9285714285714287},"provenance_summary":{"total_results":7,"total_groups":7,"multi_source_groups":0,"first_party_only_groups":7,"source_type_distribution":{"first_party":7,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":7,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["applied_reasoning","multimodal"]},{"key":"vibe-ios","display_name":"Vibe Ios","family_id":"vibe-ios","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fvibe-ios"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["agentic"]},{"key":"vibe-simulation","display_name":"Vibe Simulation","family_id":"vibe-simulation","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fvibe-simulation"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["agentic"]},{"key":"vibe-web","display_name":"Vibe Web","family_id":"vibe-web","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fvibe-web"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["agentic"]},{"key":"video-mme","display_name":"Video Mme","family_id":"video-mme","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["Not specified"],"languages":["Not specified"],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":9,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fvideo-mme"],"reproducibility_summary":{"results_total":9,"has_reproducibility_gap_count":9,"populated_ratio_avg":0.9285714285714286},"provenance_summary":{"total_results":9,"total_groups":9,"multi_source_groups":0,"first_party_only_groups":9,"source_type_distribution":{"first_party":9,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":9,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["multimodal"]},{"key":"video-mme-long-no-subtitles","display_name":"Video Mme Long No Subtitles","family_id":"video-mme-long-no-subtitles","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fvideo-mme-long-no-subtitles"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["multimodal"]},{"key":"videomme-w-o-sub","display_name":"Videomme W O Sub","family_id":"videomme","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":9,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fvideomme-w-o-sub"],"reproducibility_summary":{"results_total":9,"has_reproducibility_gap_count":9,"populated_ratio_avg":0.10714285714285712},"provenance_summary":{"total_results":9,"total_groups":9,"multi_source_groups":0,"first_party_only_groups":9,"source_type_distribution":{"first_party":9,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":9,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["multimodal"]},{"key":"videomme-w-sub","display_name":"Videomme W Sub","family_id":"videomme","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":9,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fvideomme-w-sub"],"reproducibility_summary":{"results_total":9,"has_reproducibility_gap_count":9,"populated_ratio_avg":0.10714285714285712},"provenance_summary":{"total_results":9,"total_groups":9,"multi_source_groups":0,"first_party_only_groups":9,"source_type_distribution":{"first_party":9,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":9,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["multimodal"]},{"key":"videommmu","display_name":"Videommmu","family_id":"videommmu","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":19,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fvideommmu"],"reproducibility_summary":{"results_total":19,"has_reproducibility_gap_count":19,"populated_ratio_avg":0.10714285714285716},"provenance_summary":{"total_results":19,"total_groups":19,"multi_source_groups":0,"first_party_only_groups":19,"source_type_distribution":{"first_party":19,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":19,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["multimodal"]},{"key":"vision2web","display_name":"Vision2web","family_id":"vision2web","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fvision2web"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["agentic"]},{"key":"visualwebbench","display_name":"Visualwebbench","family_id":"visualwebbench","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["web domain","website understanding","website interaction"],"languages":["English"],"tasks":["Web captioning","Webpage question answering","Heading OCR","Element OCR","Element grounding","Action prediction","Action grounding","Image to text","Visual question answering"]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":2,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fvisualwebbench"],"reproducibility_summary":{"results_total":2,"has_reproducibility_gap_count":2,"populated_ratio_avg":0.9285714285714286},"provenance_summary":{"total_results":2,"total_groups":2,"multi_source_groups":0,"first_party_only_groups":2,"source_type_distribution":{"first_party":2,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":2,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["agentic"]},{"key":"visulogic","display_name":"VisuLogic","family_id":"visulogic","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fvisulogic"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["multimodal"]},{"key":"vita-bench","display_name":"Vita Bench","family_id":"vita-bench","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":8,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fvita-bench"],"reproducibility_summary":{"results_total":8,"has_reproducibility_gap_count":8,"populated_ratio_avg":0.10714285714285712},"provenance_summary":{"total_results":8,"total_groups":8,"multi_source_groups":0,"first_party_only_groups":8,"source_type_distribution":{"first_party":8,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":8,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["multimodal"]},{"key":"vlmsareblind","display_name":"Vlmsareblind","family_id":"vlmsareblind","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":4,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fvlmsareblind"],"reproducibility_summary":{"results_total":4,"has_reproducibility_gap_count":4,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":4,"total_groups":4,"multi_source_groups":0,"first_party_only_groups":4,"source_type_distribution":{"first_party":4,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":4,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["multimodal"]},{"key":"vocalsound","display_name":"Vocalsound","family_id":"vocalsound","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fvocalsound"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["multimodal"]},{"key":"voicebench-avg","display_name":"Voicebench Avg","family_id":"voicebench-avg","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fvoicebench-avg"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["linguistic_core"]},{"key":"vqa-rad","display_name":"Vqa Rad","family_id":"vqa-rad","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fvqa-rad"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["multimodal"]},{"key":"vqav2","display_name":"Vqav2","family_id":"vqav2","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":3,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fvqav2"],"reproducibility_summary":{"results_total":3,"has_reproducibility_gap_count":3,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":3,"total_groups":3,"multi_source_groups":0,"first_party_only_groups":3,"source_type_distribution":{"first_party":3,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":3,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["multimodal"]},{"key":"vqav2-test","display_name":"Vqav2 Test","family_id":"vqav2-test","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fvqav2-test"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["multimodal"]},{"key":"vqav2-val","display_name":"Vqav2 Val","family_id":"vqav2-val","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":3,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fvqav2-val"],"reproducibility_summary":{"results_total":3,"has_reproducibility_gap_count":3,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":3,"total_groups":3,"multi_source_groups":0,"first_party_only_groups":3,"source_type_distribution":{"first_party":3,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":3,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["multimodal"]},{"key":"we-math","display_name":"We-Math","family_id":"we-math","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fwe-math"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["mathematics"]},{"key":"webvoyager","display_name":"WebVoyager","family_id":"webvoyager","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fwebvoyager"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["agentic"]},{"key":"widesearch","display_name":"WideSearch","family_id":"widesearch","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":8,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fwidesearch"],"reproducibility_summary":{"results_total":8,"has_reproducibility_gap_count":8,"populated_ratio_avg":0.10714285714285712},"provenance_summary":{"total_results":8,"total_groups":8,"multi_source_groups":0,"first_party_only_groups":8,"source_type_distribution":{"first_party":8,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":8,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["agentic"]},{"key":"wild-bench","display_name":"Wild Bench","family_id":"wild-bench","is_slice":false,"is_overall":false,"primary_metric_key":"WB Score","has_card":true,"tags":{"domains":["Not specified"],"languages":["English"],"tasks":["Text generation in response to user inputs from real conversations"]},"metrics":[{"key":"WB Score","display_name":"WB Score","sources":["LLM Stats"],"models_count":8,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fwild-bench"],"reproducibility_summary":{"results_total":8,"has_reproducibility_gap_count":8,"populated_ratio_avg":0.9285714285714287},"provenance_summary":{"total_results":8,"total_groups":8,"multi_source_groups":0,"first_party_only_groups":8,"source_type_distribution":{"first_party":8,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":8,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["general"]},{"key":"winogrande","display_name":"WinoGrande","family_id":"winogrande","is_slice":false,"is_overall":false,"primary_metric_key":"accuracy","has_card":true,"tags":{"domains":["commonsense reasoning"],"languages":["English"],"tasks":["Pronoun resolution","Binary choice fill-in-the-blank"]},"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["LLM Stats"],"models_count":18,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fwinogrande"],"reproducibility_summary":{"results_total":18,"has_reproducibility_gap_count":18,"populated_ratio_avg":0.9285714285714286},"provenance_summary":{"total_results":18,"total_groups":18,"multi_source_groups":0,"first_party_only_groups":18,"source_type_distribution":{"first_party":18,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":18,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["commonsense_reasoning"]},{"key":"wmdp","display_name":"Wmdp","family_id":"wmdp","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fwmdp"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["safety"]},{"key":"wmt23","display_name":"Wmt23","family_id":"wmt23","is_slice":false,"is_overall":false,"primary_metric_key":"The Eighth Conference","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"The Eighth Conference","display_name":"The Eighth Conference","sources":["LLM Stats"],"models_count":4,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fwmt23"],"reproducibility_summary":{"results_total":4,"has_reproducibility_gap_count":4,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":4,"total_groups":4,"multi_source_groups":0,"first_party_only_groups":4,"source_type_distribution":{"first_party":4,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":4,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["linguistic_core"]},{"key":"wmt24","display_name":"Wmt24","family_id":"wmt24","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":19,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fwmt24"],"reproducibility_summary":{"results_total":19,"has_reproducibility_gap_count":19,"populated_ratio_avg":0.10714285714285716},"provenance_summary":{"total_results":19,"total_groups":19,"multi_source_groups":0,"first_party_only_groups":19,"source_type_distribution":{"first_party":19,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":19,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["linguistic_core"]},{"key":"worldvqa","display_name":"Worldvqa","family_id":"worldvqa","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fworldvqa"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["multimodal"]},{"key":"writingbench","display_name":"Writingbench","family_id":"writingbench","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":8,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fwritingbench"],"reproducibility_summary":{"results_total":8,"has_reproducibility_gap_count":8,"populated_ratio_avg":0.10714285714285712},"provenance_summary":{"total_results":8,"total_groups":8,"multi_source_groups":0,"first_party_only_groups":8,"source_type_distribution":{"first_party":8,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":8,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["linguistic_core"]},{"key":"xlsum-english","display_name":"Xlsum English","family_id":"xlsum-english","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fxlsum-english"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":1,"source_type_distribution":{"first_party":1,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["linguistic_core"]},{"key":"xstest","display_name":"Xstest","family_id":"xstest","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["Not specified"],"languages":["English"],"tasks":["Responding to prompts","Assessing model compliance with safe prompts","Assessing model refusal of unsafe prompts"]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":3,"is_primary":true}],"slices":[{"key":"llm stats","display_name":"llm stats","is_bare_stem":false,"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"]}],"derivedTags":["general"]}],"summary_eval_ids":["llm-stats%2Fxstest"],"reproducibility_summary":{"results_total":3,"has_reproducibility_gap_count":3,"populated_ratio_avg":0.9285714285714285},"provenance_summary":{"total_results":3,"total_groups":3,"multi_source_groups":2,"first_party_only_groups":1,"source_type_distribution":{"first_party":3,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":3,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["safety","general"]},{"key":"zclawbench","display_name":"Zclawbench","family_id":"zclawbench","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":3,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fzclawbench"],"reproducibility_summary":{"results_total":3,"has_reproducibility_gap_count":3,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":3,"total_groups":3,"multi_source_groups":0,"first_party_only_groups":3,"source_type_distribution":{"first_party":3,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":3,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["law"]},{"key":"zebralogic","display_name":"Zebralogic","family_id":"zebralogic","is_slice":false,"is_overall":false,"primary_metric_key":"accuracy","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["LLM Stats"],"models_count":5,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fzebralogic"],"reproducibility_summary":{"results_total":5,"has_reproducibility_gap_count":5,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":5,"total_groups":5,"multi_source_groups":0,"first_party_only_groups":5,"source_type_distribution":{"first_party":5,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":5,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["logical_reasoning"]},{"key":"zerobench","display_name":"Zerobench","family_id":"zerobench","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":6,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fzerobench"],"reproducibility_summary":{"results_total":6,"has_reproducibility_gap_count":6,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":6,"total_groups":6,"multi_source_groups":0,"first_party_only_groups":6,"source_type_distribution":{"first_party":6,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":6,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["general"]},{"key":"zerobench-sub","display_name":"Zerobench Sub","family_id":"zerobench-sub","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":5,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fzerobench-sub"],"reproducibility_summary":{"results_total":5,"has_reproducibility_gap_count":5,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":5,"total_groups":5,"multi_source_groups":0,"first_party_only_groups":5,"source_type_distribution":{"first_party":5,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":5,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["general"]}],"derivedTags":["general","knowledge","applied_reasoning","multimodal","agentic","logical_reasoning","commonsense_reasoning","software_engineering","linguistic_core","mathematics","robustness","natural_sciences","humanities_and_social_sciences","law","safety","hallucination","finance"]},{"key":"math-mc","display_name":"MATH-MC","category":"General","tags":{"domains":[],"languages":[],"tasks":[]},"evals_count":5,"eval_summary_ids":["math-mc%2FLevel%201","math-mc%2FLevel%202","math-mc%2FLevel%203","math-mc%2FLevel%204","math-mc%2FLevel%205"],"reproducibility_summary":{"results_total":325,"has_reproducibility_gap_count":245,"populated_ratio_avg":0.10714285714285701},"provenance_summary":{"total_results":325,"total_groups":325,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":325,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":325,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"benchmarks":[],"standalone_benchmarks":[{"key":"math-mc","display_name":"MATH-MC","tags":{"domains":[],"languages":[],"tasks":[]},"summary_eval_ids":["math-mc%2FLevel%201","math-mc%2FLevel%202","math-mc%2FLevel%203","math-mc%2FLevel%204","math-mc%2FLevel%205"],"slices":[{"key":"math-mc","display_name":"MATH-MC","slice_key":null,"is_bare_stem":true,"metrics":[],"derivedTags":["mathematics"]},{"key":"Level 1","display_name":"Level 1","slice_key":"Level 1","is_bare_stem":false,"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["Writer, Inc."],"models_count":65,"is_primary":true}],"derivedTags":["mathematics"]},{"key":"Level 2","display_name":"Level 2","slice_key":"Level 2","is_bare_stem":false,"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["Writer, Inc."],"models_count":65,"is_primary":true}],"derivedTags":["mathematics"]},{"key":"Level 3","display_name":"Level 3","slice_key":"Level 3","is_bare_stem":false,"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["Writer, Inc."],"models_count":65,"is_primary":true}],"derivedTags":["mathematics"]},{"key":"Level 4","display_name":"Level 4","slice_key":"Level 4","is_bare_stem":false,"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["Writer, Inc."],"models_count":65,"is_primary":true}],"derivedTags":["mathematics"]},{"key":"Level 5","display_name":"Level 5","slice_key":"Level 5","is_bare_stem":false,"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["Writer, Inc."],"models_count":65,"is_primary":true}],"derivedTags":["mathematics"]}],"derivedTags":["mathematics"]}],"composites":[],"derivedTags":["mathematics"]},{"key":"mmlu-pro-leaderboard","display_name":"MMLU-PRO-Leaderboard","category":"Reasoning","tags":{"domains":["STEM","biology","business","chemistry","computer science","economics","engineering","health","history","humanities","law","math","philosophy","physics","psychology","social sciences"],"languages":["English"],"tasks":["Multiple-choice question answering"]},"evals_count":1,"eval_summary_ids":["mmlu-pro-leaderboard%2Fmmlu-pro"],"reproducibility_summary":{"results_total":223,"has_reproducibility_gap_count":223,"populated_ratio_avg":0.9285714285714283},"provenance_summary":{"total_results":223,"total_groups":223,"multi_source_groups":106,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":223,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":223,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"standalone_benchmarks":[{"key":"mmlu-pro","display_name":"MMLU-Pro","family_id":"mmlu","is_slice":false,"is_overall":false,"primary_metric_key":"accuracy","has_card":true,"tags":{"domains":["STEM","humanities","social sciences","math","physics","chemistry","law","engineering","economics","health","psychology","business","biology","philosophy","computer science","history"],"languages":["English"],"tasks":["Multiple-choice question answering"]},"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["TIGER-Lab"],"models_count":223,"is_primary":true}],"slices":[{"key":"mmlu-pro (biology)","display_name":"MMLU-Pro (Biology)","is_bare_stem":false,"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["TIGER-Lab"]}],"derivedTags":["general","knowledge"]},{"key":"mmlu-pro (business)","display_name":"MMLU-Pro (Business)","is_bare_stem":false,"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["TIGER-Lab"]}],"derivedTags":["general","knowledge"]},{"key":"mmlu-pro (chemistry)","display_name":"MMLU-Pro (Chemistry)","is_bare_stem":false,"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["TIGER-Lab"]}],"derivedTags":["general","knowledge"]},{"key":"mmlu-pro (computer science)","display_name":"MMLU-Pro (Computer Science)","is_bare_stem":false,"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["TIGER-Lab"]}],"derivedTags":["general","knowledge"]},{"key":"mmlu-pro (economics)","display_name":"MMLU-Pro (Economics)","is_bare_stem":false,"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["TIGER-Lab"]}],"derivedTags":["general","knowledge"]},{"key":"mmlu-pro (engineering)","display_name":"MMLU-Pro (Engineering)","is_bare_stem":false,"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["TIGER-Lab"]}],"derivedTags":["general","knowledge"]},{"key":"mmlu-pro (health)","display_name":"MMLU-Pro (Health)","is_bare_stem":false,"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["TIGER-Lab"]}],"derivedTags":["general","knowledge"]},{"key":"mmlu-pro (history)","display_name":"MMLU-Pro (History)","is_bare_stem":false,"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["TIGER-Lab"]}],"derivedTags":["general","knowledge"]},{"key":"mmlu-pro (law)","display_name":"MMLU-Pro (Law)","is_bare_stem":false,"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["TIGER-Lab"]}],"derivedTags":["general","knowledge"]},{"key":"mmlu-pro (math)","display_name":"MMLU-Pro (Math)","is_bare_stem":false,"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["TIGER-Lab"]}],"derivedTags":["general","knowledge"]},{"key":"mmlu-pro (other)","display_name":"MMLU-Pro (Other)","is_bare_stem":false,"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["TIGER-Lab"]}],"derivedTags":["general","knowledge"]},{"key":"mmlu-pro (overall)","display_name":"MMLU-Pro (overall)","is_bare_stem":false,"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["TIGER-Lab"]}],"derivedTags":["general","knowledge"]},{"key":"mmlu-pro (philosophy)","display_name":"MMLU-Pro (Philosophy)","is_bare_stem":false,"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["TIGER-Lab"]}],"derivedTags":["general","knowledge"]},{"key":"mmlu-pro (physics)","display_name":"MMLU-Pro (Physics)","is_bare_stem":false,"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["TIGER-Lab"]}],"derivedTags":["general","knowledge"]},{"key":"mmlu-pro (psychology)","display_name":"MMLU-Pro (Psychology)","is_bare_stem":false,"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["TIGER-Lab"]}],"derivedTags":["general","knowledge"]}],"summary_eval_ids":["mmlu-pro-leaderboard%2Fmmlu-pro"],"reproducibility_summary":{"results_total":223,"has_reproducibility_gap_count":223,"populated_ratio_avg":0.9285714285714283},"provenance_summary":{"total_results":223,"total_groups":223,"multi_source_groups":106,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":223,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":223,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":true,"derivedTags":["general","knowledge"]}],"derivedTags":["knowledge","general"]},{"key":"mmmu","display_name":"MMMU","category":"multimodal","tags":{"domains":[],"languages":[],"tasks":[]},"evals_count":2,"eval_summary_ids":["mmmu-multiple-choice%2FMMMU-Multiple-Choice","mmmu-open-ended%2FMMMU-Open-Ended"],"reproducibility_summary":{"results_total":23,"has_reproducibility_gap_count":23,"populated_ratio_avg":0.10714285714285712},"provenance_summary":{"total_results":23,"total_groups":23,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":23,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":23,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"composites":[{"key":"mmmu-multiple-choice","display_name":"MMMU-Multiple-Choice","category":"General","tags":{"domains":[],"languages":[],"tasks":[]},"evals_count":1,"benchmarks":[{"key":"MMMU-Multiple-Choice","display_name":"MMMU-Multiple-Choice","family_id":"MMMU-Multiple-Choice","is_slice":false,"is_overall":false,"primary_metric_key":"accuracy","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["Arcadia Impact"],"models_count":12,"is_primary":true}],"slices":[],"summary_eval_ids":["mmmu-multiple-choice%2FMMMU-Multiple-Choice"],"reproducibility_summary":{"results_total":12,"has_reproducibility_gap_count":12,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":12,"total_groups":12,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":12,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":12,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":true,"derivedTags":["multimodal","knowledge"]}],"reproducibility_summary":{"results_total":12,"has_reproducibility_gap_count":12,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":12,"total_groups":12,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":12,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":12,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":true,"derivedTags":["multimodal","knowledge"]},{"key":"mmmu-open-ended","display_name":"MMMU-Open-Ended","category":"General","tags":{"domains":[],"languages":[],"tasks":[]},"evals_count":1,"benchmarks":[{"key":"MMMU-Open-Ended","display_name":"MMMU-Open-Ended","family_id":"MMMU-Open-Ended","is_slice":false,"is_overall":false,"primary_metric_key":"accuracy","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["Arcadia Impact"],"models_count":11,"is_primary":true}],"slices":[],"summary_eval_ids":["mmmu-open-ended%2FMMMU-Open-Ended"],"reproducibility_summary":{"results_total":11,"has_reproducibility_gap_count":11,"populated_ratio_avg":0.10714285714285712},"provenance_summary":{"total_results":11,"total_groups":11,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":11,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":11,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["multimodal","knowledge"]}],"reproducibility_summary":{"results_total":11,"has_reproducibility_gap_count":11,"populated_ratio_avg":0.10714285714285712},"provenance_summary":{"total_results":11,"total_groups":11,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":11,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":11,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["multimodal","knowledge"]}],"derivedTags":["multimodal","knowledge"]},{"key":"mt-bench","display_name":"MT-Bench","category":"other","tags":{"domains":[],"languages":[],"tasks":[]},"evals_count":3,"eval_summary_ids":["mt-bench%2FMT-Bench%20%28overall%29","mt-bench%2FMT-Bench%20%28turn%201%29","mt-bench%2FMT-Bench%20%28turn%202%29"],"reproducibility_summary":{"results_total":96,"has_reproducibility_gap_count":96,"populated_ratio_avg":0.10714285714285716},"provenance_summary":{"total_results":96,"total_groups":96,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":96,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":96,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"benchmarks":[],"standalone_benchmarks":[{"key":"mt-bench","display_name":"MT-Bench","tags":{"domains":[],"languages":[],"tasks":[]},"summary_eval_ids":["mt-bench%2FMT-Bench%20%28overall%29","mt-bench%2FMT-Bench%20%28turn%201%29","mt-bench%2FMT-Bench%20%28turn%202%29"],"slices":[{"key":"mt-bench","display_name":"MT-Bench","slice_key":null,"is_bare_stem":true,"metrics":[],"derivedTags":["linguistic_core","applied_reasoning"]},{"key":"MT-Bench (overall)","display_name":"MT-Bench (overall)","slice_key":"MT-Bench (overall)","is_bare_stem":false,"metrics":[{"key":"score","display_name":"Score","sources":["LMSYS"],"models_count":32,"is_primary":true}],"derivedTags":["linguistic_core","applied_reasoning"]},{"key":"MT-Bench (turn 1)","display_name":"MT-Bench (turn 1)","slice_key":"MT-Bench (turn 1)","is_bare_stem":false,"metrics":[{"key":"score","display_name":"Score","sources":["LMSYS"],"models_count":32,"is_primary":true}],"derivedTags":["linguistic_core","applied_reasoning"]},{"key":"MT-Bench (turn 2)","display_name":"MT-Bench (turn 2)","slice_key":"MT-Bench (turn 2)","is_bare_stem":false,"metrics":[{"key":"score","display_name":"Score","sources":["LMSYS"],"models_count":32,"is_primary":true}],"derivedTags":["linguistic_core","applied_reasoning"]}],"derivedTags":["linguistic_core","applied_reasoning"]}],"composites":[],"derivedTags":["linguistic_core","applied_reasoning"]},{"key":"multi-swe-bench-leaderboard","display_name":"Multi-SWE-Bench-Leaderboard","category":"General","tags":{"domains":[],"languages":[],"tasks":[]},"evals_count":7,"eval_summary_ids":["multi-swe-bench-leaderboard%2FMulti-SWE-Bench%20%28c%29","multi-swe-bench-leaderboard%2FMulti-SWE-Bench%20%28c%2B%2B%29","multi-swe-bench-leaderboard%2FMulti-SWE-Bench%20%28go%29","multi-swe-bench-leaderboard%2FMulti-SWE-Bench%20%28java%29","multi-swe-bench-leaderboard%2FMulti-SWE-Bench%20%28javascript%29","multi-swe-bench-leaderboard%2FMulti-SWE-Bench%20%28rust%29","multi-swe-bench-leaderboard%2FMulti-SWE-Bench%20%28typescript%29"],"reproducibility_summary":{"results_total":84,"has_reproducibility_gap_count":84,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":84,"total_groups":84,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":84,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":84,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"benchmarks":[],"standalone_benchmarks":[{"key":"multi-swe-bench","display_name":"Multi-SWE-Bench","tags":{"domains":[],"languages":[],"tasks":[]},"summary_eval_ids":["multi-swe-bench-leaderboard%2FMulti-SWE-Bench%20%28c%29","multi-swe-bench-leaderboard%2FMulti-SWE-Bench%20%28c%2B%2B%29","multi-swe-bench-leaderboard%2FMulti-SWE-Bench%20%28go%29","multi-swe-bench-leaderboard%2FMulti-SWE-Bench%20%28java%29","multi-swe-bench-leaderboard%2FMulti-SWE-Bench%20%28javascript%29","multi-swe-bench-leaderboard%2FMulti-SWE-Bench%20%28rust%29","multi-swe-bench-leaderboard%2FMulti-SWE-Bench%20%28typescript%29"],"slices":[{"key":"c","display_name":"c","slice_key":"c","is_bare_stem":false,"metrics":[{"key":"score","display_name":"Score","sources":["ByteDance-Seed"],"models_count":12,"is_primary":true}],"derivedTags":["software_engineering"]},{"key":"c","display_name":"c++","slice_key":"c","is_bare_stem":false,"metrics":[{"key":"score","display_name":"Score","sources":["ByteDance-Seed"],"models_count":12,"is_primary":true}],"derivedTags":["software_engineering"]},{"key":"go","display_name":"go","slice_key":"go","is_bare_stem":false,"metrics":[{"key":"score","display_name":"Score","sources":["ByteDance-Seed"],"models_count":11,"is_primary":true}],"derivedTags":["software_engineering"]},{"key":"java","display_name":"java","slice_key":"java","is_bare_stem":false,"metrics":[{"key":"score","display_name":"Score","sources":["ByteDance-Seed"],"models_count":15,"is_primary":true}],"derivedTags":["software_engineering"]},{"key":"javascript","display_name":"javascript","slice_key":"javascript","is_bare_stem":false,"metrics":[{"key":"score","display_name":"Score","sources":["ByteDance-Seed"],"models_count":12,"is_primary":true}],"derivedTags":["software_engineering"]},{"key":"rust","display_name":"rust","slice_key":"rust","is_bare_stem":false,"metrics":[{"key":"score","display_name":"Score","sources":["ByteDance-Seed"],"models_count":11,"is_primary":true}],"derivedTags":["software_engineering"]},{"key":"typescript","display_name":"typescript","slice_key":"typescript","is_bare_stem":false,"metrics":[{"key":"score","display_name":"Score","sources":["ByteDance-Seed"],"models_count":11,"is_primary":true}],"derivedTags":["software_engineering"]}],"derivedTags":["software_engineering"]}],"composites":[],"derivedTags":["software_engineering"]},{"key":"openai-humaneval","display_name":"openai_humaneval","category":"Reasoning","tags":{"domains":["code generation"],"languages":["English"],"tasks":["Generating Python code from docstrings"]},"evals_count":1,"eval_summary_ids":["openai-humaneval%2Fhumaneval"],"reproducibility_summary":{"results_total":12,"has_reproducibility_gap_count":12,"populated_ratio_avg":0.9285714285714287},"provenance_summary":{"total_results":12,"total_groups":12,"multi_source_groups":3,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":12,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":12,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"standalone_benchmarks":[{"key":"humaneval","display_name":"HumanEval","family_id":"humaneval","is_slice":false,"is_overall":false,"primary_metric_key":"accuracy","has_card":true,"tags":{"domains":["code generation"],"languages":["English"],"tasks":["Generating Python code from docstrings"]},"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["Arcadia Impact"],"models_count":12,"is_primary":true}],"slices":[{"key":"openai_humaneval","display_name":"openai_humaneval","is_bare_stem":false,"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["Arcadia Impact"]}],"derivedTags":["software_engineering"]}],"summary_eval_ids":["openai-humaneval%2Fhumaneval"],"reproducibility_summary":{"results_total":12,"has_reproducibility_gap_count":12,"populated_ratio_avg":0.9285714285714287},"provenance_summary":{"total_results":12,"total_groups":12,"multi_source_groups":3,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":12,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":12,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":true,"derivedTags":["software_engineering"]}],"derivedTags":["applied_reasoning","software_engineering"]},{"key":"openeval","display_name":"openeval","category":"Safety","tags":{"domains":["Not specified","STEM","biology","business","chemistry","computer science","economics","engineering","finance","health","history","humanities","journalism","law","math","mathematics","natural language inference","natural language understanding","news articles","news media","olympiads","philosophy","physics","politics","psychology","reading comprehension","sentiment analysis","social sciences","summarization"],"languages":["Chinese","English"],"tasks":["Binary sentiment classification","Following verifiable instructions","Multiple-choice question answering","News article summarization","Not specified","Single-document extreme summarization","Solving Olympiad-level mathematical problems","Summarization","Text classification","Text generation","Text-pair classification","Yes/no question answering"]},"evals_count":19,"eval_summary_ids":["openeval%2Fbbq","openeval%2Fbold","openeval%2Fboolq","openeval%2Fcnn-dailymail","openeval%2Fculturalbench","openeval%2Fdisinformation","openeval%2Fdo-not-answer","openeval%2Femobench","openeval%2Fgpqa","openeval%2Fhi-tom","openeval%2Fifeval","openeval%2Fimdb","openeval%2Fmmlu-pro","openeval%2Fmoralbench","openeval%2Fomni-math","openeval%2Fopentom","openeval%2Fsalad-bench","openeval%2Ftruthfulqa","openeval%2Fxsum"],"reproducibility_summary":{"results_total":374,"has_reproducibility_gap_count":35,"populated_ratio_avg":0.6957601222307105},"provenance_summary":{"total_results":374,"total_groups":374,"multi_source_groups":129,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":374,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":374,"groups_with_variant_check":14,"groups_with_cross_party_check":0,"variant_divergent_count":14,"cross_party_divergent_count":0},"benchmarks":[{"key":"bbq","display_name":"BBQ","family_id":"bbq","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["Not specified"],"languages":["Chinese","English"],"tasks":["Not specified"]},"metrics":[{"key":"score","display_name":"Score","sources":["Human-Centered Eval"],"models_count":48,"is_primary":true}],"slices":[{"key":"openeval bbq","display_name":"openeval bbq","is_bare_stem":false,"metrics":[{"key":"score","display_name":"Score","sources":["Human-Centered Eval"]}],"derivedTags":["safety","commonsense_reasoning"]}],"summary_eval_ids":["openeval%2Fbbq"],"reproducibility_summary":{"results_total":48,"has_reproducibility_gap_count":0,"populated_ratio_avg":0.9285714285714287},"provenance_summary":{"total_results":48,"total_groups":48,"multi_source_groups":4,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":48,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":48,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":true,"derivedTags":["safety","commonsense_reasoning"]},{"key":"bold","display_name":"BOLD","family_id":"bold","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["Human-Centered Eval"],"models_count":14,"is_primary":true}],"slices":[],"summary_eval_ids":["openeval%2Fbold"],"reproducibility_summary":{"results_total":14,"has_reproducibility_gap_count":0,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":14,"total_groups":14,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":14,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":14,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["general"]},{"key":"boolq","display_name":"BoolQ","family_id":"boolq","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["natural language understanding","reading comprehension","natural language inference"],"languages":["English"],"tasks":["Yes/no question answering","Text-pair classification"]},"metrics":[{"key":"score","display_name":"Score","sources":["Human-Centered Eval"],"models_count":15,"is_primary":true}],"slices":[{"key":"openeval boolq","display_name":"openeval boolq","is_bare_stem":false,"metrics":[{"key":"score","display_name":"Score","sources":["Human-Centered Eval"]}],"derivedTags":["commonsense_reasoning"]}],"summary_eval_ids":["openeval%2Fboolq"],"reproducibility_summary":{"results_total":15,"has_reproducibility_gap_count":0,"populated_ratio_avg":0.9285714285714287},"provenance_summary":{"total_results":15,"total_groups":15,"multi_source_groups":15,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":15,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":15,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["commonsense_reasoning"]},{"key":"cnn-dailymail","display_name":"CNN/DailyMail","family_id":"cnn-dailymail","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["summarization","journalism","news media"],"languages":["English"],"tasks":["Summarization"]},"metrics":[{"key":"score","display_name":"Score","sources":["Human-Centered Eval"],"models_count":13,"is_primary":true}],"slices":[{"key":"openeval cnndm","display_name":"openeval cnndm","is_bare_stem":false,"metrics":[{"key":"score","display_name":"Score","sources":["Human-Centered Eval"]}],"derivedTags":["linguistic_core"]}],"summary_eval_ids":["openeval%2Fcnn-dailymail"],"reproducibility_summary":{"results_total":13,"has_reproducibility_gap_count":0,"populated_ratio_avg":0.9285714285714287},"provenance_summary":{"total_results":13,"total_groups":13,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":13,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":13,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["linguistic_core"]},{"key":"culturalbench","display_name":"CulturalBench","family_id":"culturalbench","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["Human-Centered Eval"],"models_count":12,"is_primary":true}],"slices":[],"summary_eval_ids":["openeval%2Fculturalbench"],"reproducibility_summary":{"results_total":12,"has_reproducibility_gap_count":12,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":12,"total_groups":12,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":12,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":12,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["general"]},{"key":"do-not-answer","display_name":"Do-Not-Answer","family_id":"do-not-answer","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["Human-Centered Eval"],"models_count":12,"is_primary":true}],"slices":[],"summary_eval_ids":["openeval%2Fdo-not-answer"],"reproducibility_summary":{"results_total":12,"has_reproducibility_gap_count":12,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":12,"total_groups":12,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":12,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":12,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["general"]},{"key":"emobench","display_name":"EmoBench","family_id":"emobench","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["Human-Centered Eval"],"models_count":11,"is_primary":true}],"slices":[],"summary_eval_ids":["openeval%2Femobench"],"reproducibility_summary":{"results_total":11,"has_reproducibility_gap_count":11,"populated_ratio_avg":0.10714285714285712},"provenance_summary":{"total_results":11,"total_groups":11,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":11,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":11,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["general"]},{"key":"gpqa","display_name":"GPQA","family_id":"gpqa","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["biology","physics","chemistry"],"languages":["English"],"tasks":["Multiple-choice question answering"]},"metrics":[{"key":"score","display_name":"Score","sources":["Human-Centered Eval"],"models_count":39,"is_primary":true}],"slices":[{"key":"openeval gpqa","display_name":"openeval gpqa","is_bare_stem":false,"metrics":[{"key":"score","display_name":"Score","sources":["Human-Centered Eval"]}],"derivedTags":["natural_sciences"]}],"summary_eval_ids":["openeval%2Fgpqa"],"reproducibility_summary":{"results_total":39,"has_reproducibility_gap_count":0,"populated_ratio_avg":0.9285714285714283},"provenance_summary":{"total_results":39,"total_groups":39,"multi_source_groups":32,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":39,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":39,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["natural_sciences"]},{"key":"hi-tom","display_name":"Hi-ToM","family_id":"hi-tom","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["Human-Centered Eval"],"models_count":11,"is_primary":true}],"slices":[],"summary_eval_ids":["openeval%2Fhi-tom"],"reproducibility_summary":{"results_total":11,"has_reproducibility_gap_count":0,"populated_ratio_avg":0.10714285714285712},"provenance_summary":{"total_results":11,"total_groups":11,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":11,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":11,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["general"]},{"key":"imdb","display_name":"IMDb","family_id":"imdb","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["sentiment analysis"],"languages":["English"],"tasks":["Binary sentiment classification","Text classification"]},"metrics":[{"key":"score","display_name":"Score","sources":["Human-Centered Eval"],"models_count":15,"is_primary":true}],"slices":[{"key":"openeval imdb","display_name":"openeval imdb","is_bare_stem":false,"metrics":[{"key":"score","display_name":"Score","sources":["Human-Centered Eval"]}],"derivedTags":["linguistic_core"]}],"summary_eval_ids":["openeval%2Fimdb"],"reproducibility_summary":{"results_total":15,"has_reproducibility_gap_count":0,"populated_ratio_avg":0.9285714285714287},"provenance_summary":{"total_results":15,"total_groups":15,"multi_source_groups":15,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":15,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":15,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["linguistic_core"]},{"key":"moralbench","display_name":"MoralBench","family_id":"moralbench","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["Human-Centered Eval"],"models_count":10,"is_primary":true}],"slices":[],"summary_eval_ids":["openeval%2Fmoralbench"],"reproducibility_summary":{"results_total":10,"has_reproducibility_gap_count":0,"populated_ratio_avg":0.10714285714285712},"provenance_summary":{"total_results":10,"total_groups":10,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":10,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":10,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["general"]},{"key":"omni-math","display_name":"Omni-MATH","family_id":"omni-math","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["mathematics","olympiads"],"languages":["English"],"tasks":["Solving Olympiad-level mathematical problems"]},"metrics":[{"key":"score","display_name":"Score","sources":["Human-Centered Eval"],"models_count":38,"is_primary":true}],"slices":[{"key":"openeval omni-math","display_name":"openeval omni-math","is_bare_stem":false,"metrics":[{"key":"score","display_name":"Score","sources":["Human-Centered Eval"]}],"derivedTags":["mathematics"]}],"summary_eval_ids":["openeval%2Fomni-math"],"reproducibility_summary":{"results_total":38,"has_reproducibility_gap_count":0,"populated_ratio_avg":0.9285714285714282},"provenance_summary":{"total_results":38,"total_groups":38,"multi_source_groups":3,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":38,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":38,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["mathematics"]},{"key":"opentom","display_name":"OpenToM","family_id":"opentom","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["Human-Centered Eval"],"models_count":10,"is_primary":true}],"slices":[],"summary_eval_ids":["openeval%2Fopentom"],"reproducibility_summary":{"results_total":10,"has_reproducibility_gap_count":0,"populated_ratio_avg":0.10714285714285712},"provenance_summary":{"total_results":10,"total_groups":10,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":10,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":10,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["general"]},{"key":"salad-bench","display_name":"Salad-Bench","family_id":"salad-bench","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["Human-Centered Eval"],"models_count":11,"is_primary":true}],"slices":[],"summary_eval_ids":["openeval%2Fsalad-bench"],"reproducibility_summary":{"results_total":11,"has_reproducibility_gap_count":0,"populated_ratio_avg":0.10714285714285712},"provenance_summary":{"total_results":11,"total_groups":11,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":11,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":11,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["general"]},{"key":"truthfulqa","display_name":"TruthfulQA","family_id":"truthfulqa","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["health","law","finance","politics"],"languages":["English"],"tasks":["Text generation","Multiple-choice question answering"]},"metrics":[{"key":"score","display_name":"Score","sources":["Human-Centered Eval"],"models_count":8,"is_primary":true}],"slices":[{"key":"openeval truthfulqa","display_name":"openeval truthfulqa","is_bare_stem":false,"metrics":[{"key":"score","display_name":"Score","sources":["Human-Centered Eval"]}],"derivedTags":["hallucination"]}],"summary_eval_ids":["openeval%2Ftruthfulqa"],"reproducibility_summary":{"results_total":8,"has_reproducibility_gap_count":0,"populated_ratio_avg":0.9285714285714287},"provenance_summary":{"total_results":8,"total_groups":8,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":8,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":8,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["hallucination"]},{"key":"xsum","display_name":"XSum","family_id":"xsum","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["news articles","summarization"],"languages":["English"],"tasks":["Single-document extreme summarization","News article summarization"]},"metrics":[{"key":"score","display_name":"Score","sources":["Human-Centered Eval"],"models_count":14,"is_primary":true}],"slices":[{"key":"openeval xsum","display_name":"openeval xsum","is_bare_stem":false,"metrics":[{"key":"score","display_name":"Score","sources":["Human-Centered Eval"]}],"derivedTags":["linguistic_core"]}],"summary_eval_ids":["openeval%2Fxsum"],"reproducibility_summary":{"results_total":14,"has_reproducibility_gap_count":0,"populated_ratio_avg":0.9285714285714287},"provenance_summary":{"total_results":14,"total_groups":14,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":14,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":14,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["linguistic_core"]}],"derivedTags":["general","safety","commonsense_reasoning","linguistic_core","natural_sciences","mathematics","hallucination"]},{"key":"piqa","display_name":"piqa","category":"Reasoning","tags":{"domains":["physical commonsense reasoning"],"languages":["Not specified"],"tasks":["Binary question-answering where a model must choose the correct solution to achieve a physical goal"]},"evals_count":1,"eval_summary_ids":["piqa%2Fpiqa"],"reproducibility_summary":{"results_total":10,"has_reproducibility_gap_count":10,"populated_ratio_avg":0.9285714285714286},"provenance_summary":{"total_results":10,"total_groups":10,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":10,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":10,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"standalone_benchmarks":[{"key":"piqa","display_name":"Piqa","family_id":"piqa","is_slice":false,"is_overall":true,"primary_metric_key":"accuracy","has_card":true,"tags":{"domains":["physical commonsense reasoning"],"languages":["Not specified"],"tasks":["Binary question-answering where a model must choose the correct solution to achieve a physical goal"]},"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["Arcadia Impact"],"models_count":10,"is_primary":true}],"slices":[{"key":"piqa","display_name":"piqa","is_bare_stem":true,"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["Arcadia Impact"]}],"derivedTags":["commonsense_reasoning"]}],"summary_eval_ids":["piqa%2Fpiqa"],"reproducibility_summary":{"results_total":10,"has_reproducibility_gap_count":10,"populated_ratio_avg":0.9285714285714286},"provenance_summary":{"total_results":10,"total_groups":10,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":10,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":10,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":true,"derivedTags":["commonsense_reasoning"]}],"derivedTags":["commonsense_reasoning"]},{"key":"reward-bench","display_name":"RewardBench family","category":"reward-modelling","tags":{"domains":["Factuality","Focus","Instruction Following","Math","Safety","Ties","chat","explanation generation","mathematical reasoning","reasoning","safety"],"languages":["English"],"tasks":["Classification tasks for reward model evaluation","Comparing prompt-chosen-rejected trios","Distinguishing between chosen (better) and rejected (worse) responses","Generating a final answer sequence given a math problem","Question answering","Reward model evaluation"]},"evals_count":16,"eval_summary_ids":["reward-bench%2Fmath","reward-bench%2Frewardbench","reward-bench-2%2Fmath","reward-bench-2%2Frewardbench-safety"],"reproducibility_summary":{"results_total":579,"has_reproducibility_gap_count":557,"populated_ratio_avg":0.8576363187762127},"provenance_summary":{"total_results":579,"total_groups":579,"multi_source_groups":17,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":579,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":579,"groups_with_variant_check":3,"groups_with_cross_party_check":3,"variant_divergent_count":3,"cross_party_divergent_count":3},"composites":[],"benchmarks":[{"key":"rewardbench","display_name":"RewardBench","family_id":"reward-bench","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["chat","reasoning","safety"],"languages":["English"],"tasks":["Reward model evaluation","Comparing prompt-chosen-rejected trios","Question answering"]},"metrics":[{"key":"score","display_name":"Score","sources":["Allen Institute for AI"],"models_count":300,"is_primary":true}],"slices":[{"key":"prior sets (0.5 weight)","display_name":"Prior Sets (0.5 weight)","is_bare_stem":false,"metrics":[{"key":"score","display_name":"Score","sources":["Allen Institute for AI"]}],"derivedTags":["applied_reasoning","general"]},{"key":"rewardbench","display_name":"RewardBench","is_bare_stem":true,"metrics":[{"key":"score","display_name":"Score","sources":["Allen Institute for AI"]}],"derivedTags":["applied_reasoning","general"]},{"key":"rewardbench-2","display_name":"RewardBench 2","is_bare_stem":false,"metrics":[],"derivedTags":["applied_reasoning","general"]},{"key":"rewardbench-chat","display_name":"RewardBench Chat","is_bare_stem":false,"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["Allen Institute for AI"]}],"derivedTags":["applied_reasoning","general"]},{"key":"rewardbench-chat-hard","display_name":"RewardBench Chat Hard","is_bare_stem":false,"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["Allen Institute for AI"]}],"derivedTags":["applied_reasoning","general"]},{"key":"rewardbench-reasoning","display_name":"RewardBench Reasoning","is_bare_stem":false,"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["Allen Institute for AI"]}],"derivedTags":["applied_reasoning","general"]},{"key":"rewardbench-safety","display_name":"RewardBench Safety","is_bare_stem":false,"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["Allen Institute for AI"]},{"key":"score","display_name":"Score","sources":["Allen Institute for AI"]}],"derivedTags":["applied_reasoning","general"]}],"summary_eval_ids":["reward-bench%2Frewardbench"],"reproducibility_summary":{"results_total":300,"has_reproducibility_gap_count":300,"populated_ratio_avg":0.9285714285714249},"provenance_summary":{"total_results":300,"total_groups":300,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":300,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":300,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["applied_reasoning","general"]},{"key":"rewardbench-2","display_name":"RewardBench 2","family_id":"reward-bench","is_slice":false,"is_overall":false,"primary_metric_key":null,"has_card":true,"tags":{"domains":["Factuality","Instruction Following","Math","Safety","Focus","Ties"],"languages":["English"],"tasks":["Classification tasks for reward model evaluation","Distinguishing between chosen (better) and rejected (worse) responses"]},"metrics":[],"slices":[{"key":"rewardbench-2-factuality","display_name":"RewardBench 2 Factuality","is_bare_stem":false,"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["Writer, Inc."]}],"derivedTags":["applied_reasoning","general"]},{"key":"rewardbench-2-focus","display_name":"RewardBench 2 Focus","is_bare_stem":false,"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["Writer, Inc."]}],"derivedTags":["applied_reasoning","general"]},{"key":"rewardbench-2-precise-if","display_name":"RewardBench 2 Precise IF","is_bare_stem":false,"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["Writer, Inc."]}],"derivedTags":["applied_reasoning","general"]}],"summary_eval_ids":[],"reproducibility_summary":null,"provenance_summary":null,"comparability_summary":null,"is_primary":false,"derivedTags":["applied_reasoning","general"]},{"key":"rewardbench-safety","display_name":"RewardBench Safety","family_id":"reward-bench","is_slice":false,"is_overall":false,"primary_metric_key":"accuracy","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["Writer, Inc."],"models_count":50,"is_primary":true}],"slices":[],"summary_eval_ids":["reward-bench-2%2Frewardbench-safety"],"reproducibility_summary":{"results_total":50,"has_reproducibility_gap_count":39,"populated_ratio_avg":0.10714285714285708},"provenance_summary":{"total_results":50,"total_groups":50,"multi_source_groups":3,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":50,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":50,"groups_with_variant_check":3,"groups_with_cross_party_check":3,"variant_divergent_count":3,"cross_party_divergent_count":3},"is_primary":false,"derivedTags":["applied_reasoning","general"]}],"standalone_benchmarks":[],"derivedTags":["applied_reasoning","general"]},{"key":"sciarena","display_name":"SciArena","category":"General","tags":{"domains":["Engineering","Healthcare","Humanities & Social Sciences","Natural Science"],"languages":[],"tasks":["Generating long-form responses to scientific literature-based questions","Pairwise evaluation of model responses"]},"evals_count":3,"eval_summary_ids":["sciarena%2Fsciarena"],"reproducibility_summary":{"results_total":32,"has_reproducibility_gap_count":32,"populated_ratio_avg":0.9285714285714279},"provenance_summary":{"total_results":32,"total_groups":32,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":32,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":32,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"standalone_benchmarks":[{"key":"sciarena","display_name":"SciArena","family_id":"sciarena","is_slice":false,"is_overall":true,"primary_metric_key":"rank","has_card":true,"tags":{"domains":["Natural Science","Healthcare","Humanities & Social Sciences","Engineering"],"languages":[],"tasks":["Generating long-form responses to scientific literature-based questions","Pairwise evaluation of model responses"]},"metrics":[{"key":"cost-per-task","display_name":"Cost Per Task","sources":["Ai2"],"models_count":32,"is_primary":false},{"key":"elo","display_name":"Elo Rating","sources":["Ai2"],"models_count":32,"is_primary":false},{"key":"rank","display_name":"Rank","sources":["Ai2"],"models_count":32,"is_primary":true}],"slices":[],"summary_eval_ids":["sciarena%2Fsciarena"],"reproducibility_summary":{"results_total":32,"has_reproducibility_gap_count":32,"populated_ratio_avg":0.9285714285714279},"provenance_summary":{"total_results":32,"total_groups":32,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":32,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":32,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":true,"derivedTags":["natural_sciences","applied_reasoning"]}],"derivedTags":["natural_sciences","applied_reasoning"]},{"key":"swe-polybench-leaderboard","display_name":"SWE-Polybench-Leaderboard","category":"General","tags":{"domains":[],"languages":[],"tasks":[]},"evals_count":8,"eval_summary_ids":["swe-polybench-leaderboard%2FSWE-PolyBench%20%28Java%29","swe-polybench-leaderboard%2FSWE-PolyBench%20%28JavaScript%29","swe-polybench-leaderboard%2FSWE-PolyBench%20%28Python%29","swe-polybench-leaderboard%2FSWE-PolyBench%20%28TypeScript%29","swe-polybench-leaderboard%2FSWE-PolyBench%20Verified%20%28Java%29","swe-polybench-leaderboard%2FSWE-PolyBench%20Verified%20%28JavaScript%29","swe-polybench-leaderboard%2FSWE-PolyBench%20Verified%20%28Python%29","swe-polybench-leaderboard%2FSWE-PolyBench%20Verified%20%28TypeScript%29"],"reproducibility_summary":{"results_total":16,"has_reproducibility_gap_count":16,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":16,"total_groups":16,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":16,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":16,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"benchmarks":[],"standalone_benchmarks":[{"key":"swe-polybench","display_name":"SWE-PolyBench","tags":{"domains":[],"languages":[],"tasks":[]},"summary_eval_ids":["swe-polybench-leaderboard%2FSWE-PolyBench%20%28Java%29","swe-polybench-leaderboard%2FSWE-PolyBench%20%28JavaScript%29","swe-polybench-leaderboard%2FSWE-PolyBench%20%28Python%29","swe-polybench-leaderboard%2FSWE-PolyBench%20%28TypeScript%29"],"slices":[{"key":"java","display_name":"Java","slice_key":"java","is_bare_stem":false,"metrics":[{"key":"score","display_name":"Score","sources":["AmazonScience"],"models_count":1,"is_primary":true}],"derivedTags":["software_engineering","applied_reasoning"]},{"key":"javascript","display_name":"JavaScript","slice_key":"javascript","is_bare_stem":false,"metrics":[{"key":"score","display_name":"Score","sources":["AmazonScience"],"models_count":1,"is_primary":true}],"derivedTags":["software_engineering","applied_reasoning"]},{"key":"python","display_name":"Python","slice_key":"python","is_bare_stem":false,"metrics":[{"key":"score","display_name":"Score","sources":["AmazonScience"],"models_count":1,"is_primary":true}],"derivedTags":["software_engineering","applied_reasoning"]},{"key":"typescript","display_name":"TypeScript","slice_key":"typescript","is_bare_stem":false,"metrics":[{"key":"score","display_name":"Score","sources":["AmazonScience"],"models_count":1,"is_primary":true}],"derivedTags":["software_engineering","applied_reasoning"]}],"derivedTags":["software_engineering","applied_reasoning"]},{"key":"swe-polybench-verified","display_name":"SWE-PolyBench Verified","tags":{"domains":[],"languages":[],"tasks":[]},"summary_eval_ids":["swe-polybench-leaderboard%2FSWE-PolyBench%20Verified%20%28Java%29","swe-polybench-leaderboard%2FSWE-PolyBench%20Verified%20%28JavaScript%29","swe-polybench-leaderboard%2FSWE-PolyBench%20Verified%20%28Python%29","swe-polybench-leaderboard%2FSWE-PolyBench%20Verified%20%28TypeScript%29"],"slices":[{"key":"java","display_name":"Java","slice_key":"java","is_bare_stem":false,"metrics":[{"key":"score","display_name":"Score","sources":["AmazonScience"],"models_count":3,"is_primary":true}],"derivedTags":["general"]},{"key":"javascript","display_name":"JavaScript","slice_key":"javascript","is_bare_stem":false,"metrics":[{"key":"score","display_name":"Score","sources":["AmazonScience"],"models_count":3,"is_primary":true}],"derivedTags":["general"]},{"key":"python","display_name":"Python","slice_key":"python","is_bare_stem":false,"metrics":[{"key":"score","display_name":"Score","sources":["AmazonScience"],"models_count":3,"is_primary":true}],"derivedTags":["general"]},{"key":"typescript","display_name":"TypeScript","slice_key":"typescript","is_bare_stem":false,"metrics":[{"key":"score","display_name":"Score","sources":["AmazonScience"],"models_count":3,"is_primary":true}],"derivedTags":["general"]}],"derivedTags":["general"]}],"composites":[],"derivedTags":["general","software_engineering","applied_reasoning"]},{"key":"theory-of-mind","display_name":"Theory of Mind","category":"Reasoning","tags":{"domains":["Not specified"],"languages":["Not specified"],"tasks":["Answering theory of mind questions"]},"evals_count":1,"eval_summary_ids":["theory-of-mind%2Ftheory-of-mind"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.9285714285714286},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":1,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"standalone_benchmarks":[{"key":"theory-of-mind","display_name":"Theory of Mind","family_id":"theory-of-mind","is_slice":false,"is_overall":true,"primary_metric_key":"accuracy","has_card":true,"tags":{"domains":["Not specified"],"languages":["Not specified"],"tasks":["Answering theory of mind questions"]},"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["unknown"],"models_count":1,"is_primary":true}],"slices":[],"summary_eval_ids":["theory-of-mind%2Ftheory-of-mind"],"reproducibility_summary":{"results_total":1,"has_reproducibility_gap_count":1,"populated_ratio_avg":0.9285714285714286},"provenance_summary":{"total_results":1,"total_groups":1,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":1,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":true,"derivedTags":["commonsense_reasoning"]}],"derivedTags":["commonsense_reasoning"]},{"key":"vals-ai","display_name":"Vals.ai","category":"General","tags":{"domains":["Not specified","STEM","biology","business","chemistry","code generation","computer science","computer terminal environments","economics","engineering","finance","health","history","humanities","issue resolution","law","legal","math","math word problems","mathematical problem-solving","mathematical reasoning","mathematics","medical knowledge","model training","philosophy","physics","professional medical exams","programming","programming competitions","psychology","question answering","social sciences","software engineering","system administration","web development"],"languages":["Bengali","C++","Chinese","English","French","German","Japanese","Not specified","Python","Russian","Spanish","Swahili","Thai"],"tasks":["Building an application from scratch in a sandboxed environment with access to a browser, terminal, and production services based on a text specification","Code execution","Code generation","Coding problem solving","Computer terminal environment tasks","Example tasks like training a fasttext model to meet specific requirements","Free-form multiple-choice question answering","Issue resolution","Mathematical problem solving","Multiple-choice question answering","Not specified","Open-domain question answering","Question answering","Rule-application tasks","Self-repair","Single integer answer generation (0-999 range)","Solving competitive programming problems","Solving grade-school math word problems","Tasks scaling in difficulty across unique categories","Test output prediction","Text classification","Text generation","Text generation for solving mathematical problems"]},"evals_count":34,"eval_summary_ids":["vals-ai%2Faime","vals-ai%2Fcase-law-v2","vals-ai%2Fcorp-fin-v2","vals-ai%2Fgpqa-overall","vals-ai%2Fioi","vals-ai%2Flcb","vals-ai%2Flegalbench","vals-ai%2Fmath-500","vals-ai%2Fmedcode","vals-ai%2Fmedqa","vals-ai%2Fmedscribe","vals-ai%2Fmgsm","vals-ai%2Fmmlu-pro","vals-ai%2Fmmmu","vals-ai%2Fmortgage-tax","vals-ai%2Fproof-bench","vals-ai%2Fsage-vals","vals-ai%2Fswe-bench","vals-ai%2Ftax-eval-v2","vals-ai%2Fterminal-bench","vals-ai%2Fterminal-bench-2","vals-ai%2Fvals%20ai%20finance%20agent","vals-ai%2Fvals%20ai%20gpqa","vals-ai%2Fvals%20ai%20mgsm","vals-ai%2Fvals-index","vals-ai%2Fvals-multimodal-index","vals-ai%2Fvals_ai.swebench.%3C15%20min%20fix","vals-ai%2Fvals_ai.swebench.%3E4%20hours","vals-ai%2Fvals_ai.swebench.1-4%20hours","vals-ai%2Fvals_ai.swebench.15%20min%20-%201%20hour","vals-ai%2Fvibe-code"],"reproducibility_summary":{"results_total":1856,"has_reproducibility_gap_count":1100,"populated_ratio_avg":0.5630003078817739},"provenance_summary":{"total_results":1856,"total_groups":1856,"multi_source_groups":230,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":1856,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":1856,"groups_with_variant_check":137,"groups_with_cross_party_check":0,"variant_divergent_count":121,"cross_party_divergent_count":0},"benchmarks":[{"key":"case-law-v2","display_name":"Case Law v2","family_id":"case-law-v2","is_slice":false,"is_overall":false,"primary_metric_key":"accuracy","has_card":true,"tags":{"domains":["Not specified"],"languages":["Not specified"],"tasks":["Not specified"]},"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["Vals.ai"],"models_count":48,"is_primary":true}],"slices":[],"summary_eval_ids":["vals-ai%2Fcase-law-v2"],"reproducibility_summary":{"results_total":48,"has_reproducibility_gap_count":46,"populated_ratio_avg":0.9285714285714287},"provenance_summary":{"total_results":48,"total_groups":48,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":48,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":48,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["general"]},{"key":"corp-fin-v2","display_name":"Corp Fin v2","family_id":"corp-fin-v2","is_slice":false,"is_overall":false,"primary_metric_key":"accuracy","has_card":true,"tags":{"domains":["Not specified"],"languages":["Not specified"],"tasks":["Not specified"]},"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["Vals.ai"],"models_count":84,"is_primary":true}],"slices":[],"summary_eval_ids":["vals-ai%2Fcorp-fin-v2"],"reproducibility_summary":{"results_total":84,"has_reproducibility_gap_count":82,"populated_ratio_avg":0.9285714285714294},"provenance_summary":{"total_results":84,"total_groups":84,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":84,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":84,"groups_with_variant_check":1,"groups_with_cross_party_check":0,"variant_divergent_count":1,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["finance"]},{"key":"gpqa-overall","display_name":"Gpqa overall","family_id":"gpqa-overall","is_slice":false,"is_overall":false,"primary_metric_key":"accuracy","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["Vals.ai"],"models_count":89,"is_primary":true}],"slices":[],"summary_eval_ids":["vals-ai%2Fgpqa-overall"],"reproducibility_summary":{"results_total":89,"has_reproducibility_gap_count":23,"populated_ratio_avg":0.1071428571428571},"provenance_summary":{"total_results":89,"total_groups":89,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":89,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":89,"groups_with_variant_check":6,"groups_with_cross_party_check":0,"variant_divergent_count":4,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["natural_sciences"]},{"key":"ioi","display_name":"IOI","family_id":"ioi","is_slice":false,"is_overall":false,"primary_metric_key":"accuracy","has_card":true,"tags":{"domains":["programming","computer science"],"languages":["C++"],"tasks":["Solving competitive programming problems"]},"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["Vals.ai"],"models_count":46,"is_primary":true}],"slices":[],"summary_eval_ids":["vals-ai%2Fioi"],"reproducibility_summary":{"results_total":46,"has_reproducibility_gap_count":14,"populated_ratio_avg":0.9285714285714286},"provenance_summary":{"total_results":46,"total_groups":46,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":46,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":46,"groups_with_variant_check":3,"groups_with_cross_party_check":0,"variant_divergent_count":3,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["software_engineering"]},{"key":"lcb","display_name":"LCB","family_id":"lcb","is_slice":false,"is_overall":false,"primary_metric_key":"accuracy","has_card":true,"tags":{"domains":["code generation","programming competitions"],"languages":["Not specified"],"tasks":["Code generation","Self-repair","Code execution","Test output prediction"]},"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["Vals.ai"],"models_count":96,"is_primary":true}],"slices":[],"summary_eval_ids":["vals-ai%2Flcb"],"reproducibility_summary":{"results_total":96,"has_reproducibility_gap_count":27,"populated_ratio_avg":0.9285714285714296},"provenance_summary":{"total_results":96,"total_groups":96,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":96,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":96,"groups_with_variant_check":6,"groups_with_cross_party_check":0,"variant_divergent_count":6,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["general"]},{"key":"legalbench","display_name":"LegalBench","family_id":"legalbench","is_slice":false,"is_overall":false,"primary_metric_key":"accuracy","has_card":true,"tags":{"domains":["legal","law","finance"],"languages":["English"],"tasks":["Text classification","Question answering","Text generation","Rule-application tasks"]},"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["Vals.ai"],"models_count":104,"is_primary":true}],"slices":[{"key":"vals ai legal bench","display_name":"vals ai legal bench","is_bare_stem":false,"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["Vals.ai"]}],"derivedTags":["law"]}],"summary_eval_ids":["vals-ai%2Flegalbench"],"reproducibility_summary":{"results_total":104,"has_reproducibility_gap_count":33,"populated_ratio_avg":0.9285714285714296},"provenance_summary":{"total_results":104,"total_groups":104,"multi_source_groups":27,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":104,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":104,"groups_with_variant_check":4,"groups_with_cross_party_check":0,"variant_divergent_count":4,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["law"]},{"key":"math-500","display_name":"MATH-500","family_id":"math","is_slice":false,"is_overall":false,"primary_metric_key":"accuracy","has_card":true,"tags":{"domains":["mathematics"],"languages":["English"],"tasks":["Text generation for solving mathematical problems"]},"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["Vals.ai"],"models_count":50,"is_primary":true}],"slices":[{"key":"vals ai math500","display_name":"vals ai math500","is_bare_stem":false,"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["Vals.ai"]}],"derivedTags":["mathematics"]}],"summary_eval_ids":["vals-ai%2Fmath-500"],"reproducibility_summary":{"results_total":50,"has_reproducibility_gap_count":14,"populated_ratio_avg":0.9285714285714287},"provenance_summary":{"total_results":50,"total_groups":50,"multi_source_groups":32,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":50,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":50,"groups_with_variant_check":3,"groups_with_cross_party_check":0,"variant_divergent_count":2,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["mathematics"]},{"key":"medcode","display_name":"MedCode","family_id":"medcode","is_slice":false,"is_overall":false,"primary_metric_key":"accuracy","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["Vals.ai"],"models_count":45,"is_primary":true}],"slices":[],"summary_eval_ids":["vals-ai%2Fmedcode"],"reproducibility_summary":{"results_total":45,"has_reproducibility_gap_count":13,"populated_ratio_avg":0.10714285714285712},"provenance_summary":{"total_results":45,"total_groups":45,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":45,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":45,"groups_with_variant_check":1,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["general"]},{"key":"medqa","display_name":"MedQA","family_id":"medqa","is_slice":false,"is_overall":false,"primary_metric_key":"accuracy","has_card":true,"tags":{"domains":["medical knowledge","professional medical exams"],"languages":["English"],"tasks":["Free-form multiple-choice question answering","Open-domain question answering"]},"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["Vals.ai"],"models_count":82,"is_primary":true}],"slices":[{"key":"vals ai medqa","display_name":"vals ai medqa","is_bare_stem":false,"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["Vals.ai"]}],"derivedTags":["knowledge","natural_sciences"]}],"summary_eval_ids":["vals-ai%2Fmedqa"],"reproducibility_summary":{"results_total":82,"has_reproducibility_gap_count":21,"populated_ratio_avg":0.9285714285714294},"provenance_summary":{"total_results":82,"total_groups":82,"multi_source_groups":16,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":82,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":82,"groups_with_variant_check":4,"groups_with_cross_party_check":0,"variant_divergent_count":1,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["knowledge","natural_sciences"]},{"key":"medscribe","display_name":"MedScribe","family_id":"medscribe","is_slice":false,"is_overall":false,"primary_metric_key":"accuracy","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["Vals.ai"],"models_count":45,"is_primary":true}],"slices":[],"summary_eval_ids":["vals-ai%2Fmedscribe"],"reproducibility_summary":{"results_total":45,"has_reproducibility_gap_count":44,"populated_ratio_avg":0.10714285714285712},"provenance_summary":{"total_results":45,"total_groups":45,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":45,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":45,"groups_with_variant_check":1,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["general"]},{"key":"mgsm","display_name":"MGSM","family_id":"mgsm","is_slice":false,"is_overall":false,"primary_metric_key":"accuracy","has_card":true,"tags":{"domains":["mathematical reasoning","question answering","math word problems"],"languages":["Bengali","Chinese","English","French","German","Japanese","Russian","Spanish","Swahili","Thai"],"tasks":["Solving grade-school math word problems","Text generation"]},"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["Vals.ai"],"models_count":65,"is_primary":true}],"slices":[{"key":"vals ai mgsm","display_name":"vals ai mgsm","is_bare_stem":false,"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["Vals.ai"]}],"derivedTags":["mathematics"]}],"summary_eval_ids":["vals-ai%2Fmgsm"],"reproducibility_summary":{"results_total":65,"has_reproducibility_gap_count":17,"populated_ratio_avg":0.9285714285714292},"provenance_summary":{"total_results":65,"total_groups":65,"multi_source_groups":10,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":65,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":65,"groups_with_variant_check":4,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["mathematics"]},{"key":"mortgage-tax","display_name":"Mortgage Tax","family_id":"mortgage-tax","is_slice":false,"is_overall":false,"primary_metric_key":"accuracy","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["Vals.ai"],"models_count":61,"is_primary":true}],"slices":[],"summary_eval_ids":["vals-ai%2Fmortgage-tax"],"reproducibility_summary":{"results_total":61,"has_reproducibility_gap_count":23,"populated_ratio_avg":0.10714285714285703},"provenance_summary":{"total_results":61,"total_groups":61,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":61,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":61,"groups_with_variant_check":4,"groups_with_cross_party_check":0,"variant_divergent_count":4,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["finance","applied_reasoning"]},{"key":"proof-bench","display_name":"Proof Bench","family_id":"proof-bench","is_slice":false,"is_overall":false,"primary_metric_key":"accuracy","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["Vals.ai"],"models_count":28,"is_primary":true}],"slices":[],"summary_eval_ids":["vals-ai%2Fproof-bench"],"reproducibility_summary":{"results_total":28,"has_reproducibility_gap_count":27,"populated_ratio_avg":0.10714285714285718},"provenance_summary":{"total_results":28,"total_groups":28,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":28,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":28,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["general"]},{"key":"sage-vals","display_name":"SAGE-VALS","family_id":"sage-vals","is_slice":false,"is_overall":false,"primary_metric_key":"accuracy","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["Vals.ai"],"models_count":43,"is_primary":true}],"slices":[],"summary_eval_ids":["vals-ai%2Fsage-vals"],"reproducibility_summary":{"results_total":43,"has_reproducibility_gap_count":14,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":43,"total_groups":43,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":43,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":43,"groups_with_variant_check":1,"groups_with_cross_party_check":0,"variant_divergent_count":1,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["general"]},{"key":"swe-bench","display_name":"SWE-bench","family_id":"swe-bench","is_slice":false,"is_overall":false,"primary_metric_key":"accuracy","has_card":true,"tags":{"domains":["software engineering","issue resolution","code generation"],"languages":["Python"],"tasks":["Issue resolution"]},"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["Vals.ai"],"models_count":43,"is_primary":true}],"slices":[{"key":"swe-bench","display_name":"SWE-bench","is_bare_stem":true,"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["Vals.ai"]}],"derivedTags":["software_engineering"]},{"key":"vals ai swebench","display_name":"vals ai swebench","is_bare_stem":false,"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["Vals.ai"]}],"derivedTags":["software_engineering"]}],"summary_eval_ids":["vals-ai%2Fswe-bench"],"reproducibility_summary":{"results_total":43,"has_reproducibility_gap_count":43,"populated_ratio_avg":0.9285714285714285},"provenance_summary":{"total_results":43,"total_groups":43,"multi_source_groups":2,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":43,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":43,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["software_engineering"]},{"key":"tax-eval-v2","display_name":"Tax Eval v2","family_id":"tax-eval-v2","is_slice":false,"is_overall":false,"primary_metric_key":"accuracy","has_card":true,"tags":{"domains":["Not specified"],"languages":["Not specified"],"tasks":["Not specified"]},"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["Vals.ai"],"models_count":92,"is_primary":true}],"slices":[],"summary_eval_ids":["vals-ai%2Ftax-eval-v2"],"reproducibility_summary":{"results_total":92,"has_reproducibility_gap_count":26,"populated_ratio_avg":0.9285714285714295},"provenance_summary":{"total_results":92,"total_groups":92,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":92,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":92,"groups_with_variant_check":5,"groups_with_cross_party_check":0,"variant_divergent_count":5,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["finance","applied_reasoning"]},{"key":"terminal-bench","display_name":"Terminal Bench","family_id":"terminal-bench","is_slice":false,"is_overall":false,"primary_metric_key":"accuracy","has_card":true,"tags":{"domains":["computer terminal environments","system administration","model training"],"languages":["Not specified"],"tasks":["Computer terminal environment tasks","Tasks scaling in difficulty across unique categories","Example tasks like training a fasttext model to meet specific requirements"]},"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["Vals.ai"],"models_count":64,"is_primary":true}],"slices":[{"key":"terminalbench-hard","display_name":"TerminalBench Hard","is_bare_stem":false,"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["Vals.ai"]}],"derivedTags":["agentic"]},{"key":"vals ai terminal-bench","display_name":"vals ai terminal-bench","is_bare_stem":false,"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["Vals.ai"]}],"derivedTags":["agentic"]},{"key":"vals ai vals index","display_name":"vals ai vals index","is_bare_stem":false,"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["Vals.ai"]}],"derivedTags":["agentic"]}],"summary_eval_ids":["vals-ai%2Fterminal-bench"],"reproducibility_summary":{"results_total":64,"has_reproducibility_gap_count":64,"populated_ratio_avg":0.9285714285714292},"provenance_summary":{"total_results":64,"total_groups":64,"multi_source_groups":36,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":64,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":64,"groups_with_variant_check":4,"groups_with_cross_party_check":0,"variant_divergent_count":3,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["agentic"]},{"key":"terminal-bench-2","display_name":"Terminal-Bench 2.0","family_id":"terminal-bench-2","is_slice":false,"is_overall":false,"primary_metric_key":"accuracy","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["Vals.ai"],"models_count":53,"is_primary":true}],"slices":[],"summary_eval_ids":["vals-ai%2Fterminal-bench-2"],"reproducibility_summary":{"results_total":53,"has_reproducibility_gap_count":53,"populated_ratio_avg":0.10714285714285718},"provenance_summary":{"total_results":53,"total_groups":53,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":53,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":53,"groups_with_variant_check":3,"groups_with_cross_party_check":0,"variant_divergent_count":3,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["agentic"]},{"key":"vals ai finance agent","display_name":"Vals AI Finance Agent","family_id":"vals ai finance agent","is_slice":false,"is_overall":false,"primary_metric_key":"accuracy","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["Vals.ai"],"models_count":47,"is_primary":true}],"slices":[],"summary_eval_ids":["vals-ai%2Fvals%20ai%20finance%20agent"],"reproducibility_summary":{"results_total":47,"has_reproducibility_gap_count":46,"populated_ratio_avg":0.10714285714285711},"provenance_summary":{"total_results":47,"total_groups":47,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":47,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":47,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["general"]},{"key":"vals-index","display_name":"Vals Index","family_id":"vals-index","is_slice":false,"is_overall":false,"primary_metric_key":"accuracy","has_card":true,"tags":{"domains":["Not specified"],"languages":["Not specified"],"tasks":["Not specified"]},"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["Vals.ai"],"models_count":43,"is_primary":true}],"slices":[],"summary_eval_ids":["vals-ai%2Fvals-index"],"reproducibility_summary":{"results_total":43,"has_reproducibility_gap_count":43,"populated_ratio_avg":0.9285714285714285},"provenance_summary":{"total_results":43,"total_groups":43,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":43,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":43,"groups_with_variant_check":40,"groups_with_cross_party_check":0,"variant_divergent_count":40,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["general"]},{"key":"vals-multimodal-index","display_name":"Vals Multimodal Index","family_id":"vals-multimodal-index","is_slice":false,"is_overall":false,"primary_metric_key":"accuracy","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["Vals.ai"],"models_count":29,"is_primary":true}],"slices":[],"summary_eval_ids":["vals-ai%2Fvals-multimodal-index"],"reproducibility_summary":{"results_total":29,"has_reproducibility_gap_count":29,"populated_ratio_avg":0.10714285714285718},"provenance_summary":{"total_results":29,"total_groups":29,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":29,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":29,"groups_with_variant_check":28,"groups_with_cross_party_check":0,"variant_divergent_count":28,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["multimodal"]},{"key":"vals_ai.swebench.1-4 hours","display_name":"Vals Ai.swebench.1-4 Hours","family_id":"vals_ai.swebench.1-4 hours","is_slice":false,"is_overall":false,"primary_metric_key":"accuracy","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["Vals.ai"],"models_count":43,"is_primary":true}],"slices":[],"summary_eval_ids":["vals-ai%2Fvals_ai.swebench.1-4%20hours"],"reproducibility_summary":{"results_total":43,"has_reproducibility_gap_count":42,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":43,"total_groups":43,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":43,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":43,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["general"]},{"key":"vals_ai.swebench.15 min - 1 hour","display_name":"Vals Ai.swebench.15 Min - 1 Hour","family_id":"vals_ai.swebench.15 min - 1 hour","is_slice":false,"is_overall":false,"primary_metric_key":"accuracy","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["Vals.ai"],"models_count":43,"is_primary":true}],"slices":[],"summary_eval_ids":["vals-ai%2Fvals_ai.swebench.15%20min%20-%201%20hour"],"reproducibility_summary":{"results_total":43,"has_reproducibility_gap_count":42,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":43,"total_groups":43,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":43,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":43,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["general"]},{"key":"vals_ai.swebench.\u003c15 min fix","display_name":"Vals Ai.swebench.\u003c15 Min Fix","family_id":"vals_ai.swebench.\u003c15 min fix","is_slice":false,"is_overall":false,"primary_metric_key":"accuracy","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["Vals.ai"],"models_count":43,"is_primary":true}],"slices":[],"summary_eval_ids":["vals-ai%2Fvals_ai.swebench.%3C15%20min%20fix"],"reproducibility_summary":{"results_total":43,"has_reproducibility_gap_count":42,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":43,"total_groups":43,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":43,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":43,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["general"]},{"key":"vals_ai.swebench.>4 hours","display_name":"Vals Ai.swebench.>4 Hours","family_id":"vals_ai.swebench.>4 hours","is_slice":false,"is_overall":false,"primary_metric_key":"accuracy","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["Vals.ai"],"models_count":43,"is_primary":true}],"slices":[],"summary_eval_ids":["vals-ai%2Fvals_ai.swebench.%3E4%20hours"],"reproducibility_summary":{"results_total":43,"has_reproducibility_gap_count":42,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":43,"total_groups":43,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":43,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":43,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["general"]},{"key":"vibe-code","display_name":"Vibe Code","family_id":"vibe-code","is_slice":false,"is_overall":false,"primary_metric_key":"accuracy","has_card":true,"tags":{"domains":["web development","software engineering"],"languages":["Not specified"],"tasks":["Building an application from scratch in a sandboxed environment with access to a browser, terminal, and production services based on a text specification"]},"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["Vals.ai"],"models_count":39,"is_primary":true}],"slices":[],"summary_eval_ids":["vals-ai%2Fvibe-code"],"reproducibility_summary":{"results_total":39,"has_reproducibility_gap_count":23,"populated_ratio_avg":0.9285714285714283},"provenance_summary":{"total_results":39,"total_groups":39,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":39,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":39,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["general"]}],"derivedTags":["general","finance","natural_sciences","software_engineering","law","mathematics","knowledge","applied_reasoning","agentic","multimodal"]},{"key":"wasp","display_name":"WASP","category":"Reasoning","tags":{"domains":["explanation generation","mathematical reasoning"],"languages":["English"],"tasks":["Generating a final answer sequence given a math problem"]},"evals_count":5,"eval_summary_ids":["wasp%2Fgpqa-diamond","wasp%2Fjudgebench-coding","wasp%2Fjudgebench-knowledge","wasp%2Fmath","wasp%2Frewardbench-reasoning"],"reproducibility_summary":{"results_total":263,"has_reproducibility_gap_count":199,"populated_ratio_avg":0.2633079847908745},"provenance_summary":{"total_results":263,"total_groups":263,"multi_source_groups":12,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":263,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":263,"groups_with_variant_check":3,"groups_with_cross_party_check":3,"variant_divergent_count":3,"cross_party_divergent_count":3},"benchmarks":[{"key":"gpqa-diamond","display_name":"GPQA Diamond","family_id":"gpqa","is_slice":false,"is_overall":false,"primary_metric_key":"accuracy","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["Writer, Inc."],"models_count":63,"is_primary":true}],"slices":[{"key":"gpqa diamond","display_name":"GPQA Diamond","is_bare_stem":false,"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["Writer, Inc."]}],"derivedTags":["natural_sciences"]}],"summary_eval_ids":["wasp%2Fgpqa-diamond"],"reproducibility_summary":{"results_total":63,"has_reproducibility_gap_count":47,"populated_ratio_avg":0.10714285714285701},"provenance_summary":{"total_results":63,"total_groups":63,"multi_source_groups":2,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":63,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":63,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":true,"derivedTags":["natural_sciences"]},{"key":"judgebench-coding","display_name":"JudgeBench Coding","family_id":"judgebench","is_slice":false,"is_overall":false,"primary_metric_key":"accuracy","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["Writer, Inc."],"models_count":50,"is_primary":true}],"slices":[],"summary_eval_ids":["wasp%2Fjudgebench-coding"],"reproducibility_summary":{"results_total":50,"has_reproducibility_gap_count":38,"populated_ratio_avg":0.10714285714285708},"provenance_summary":{"total_results":50,"total_groups":50,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":50,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":50,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["general"]},{"key":"judgebench-knowledge","display_name":"JudgeBench Knowledge","family_id":"judgebench","is_slice":false,"is_overall":false,"primary_metric_key":"accuracy","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["Writer, Inc."],"models_count":50,"is_primary":true}],"slices":[],"summary_eval_ids":["wasp%2Fjudgebench-knowledge"],"reproducibility_summary":{"results_total":50,"has_reproducibility_gap_count":38,"populated_ratio_avg":0.10714285714285708},"provenance_summary":{"total_results":50,"total_groups":50,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":50,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":50,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["general"]},{"key":"rewardbench-reasoning","display_name":"RewardBench Reasoning","family_id":"reward-bench","is_slice":false,"is_overall":false,"primary_metric_key":"accuracy","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["Writer, Inc."],"models_count":50,"is_primary":true}],"slices":[],"summary_eval_ids":["wasp%2Frewardbench-reasoning"],"reproducibility_summary":{"results_total":50,"has_reproducibility_gap_count":38,"populated_ratio_avg":0.10714285714285708},"provenance_summary":{"total_results":50,"total_groups":50,"multi_source_groups":3,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":50,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":50,"groups_with_variant_check":3,"groups_with_cross_party_check":3,"variant_divergent_count":3,"cross_party_divergent_count":3},"is_primary":false,"derivedTags":["general"]}],"derivedTags":["general","natural_sciences"]},{"key":"wordle-arena","display_name":"Wordle Arena","category":"Reasoning","tags":{"domains":["adversarial reasoning","deductive reasoning","word puzzle"],"languages":["English"],"tasks":["Deductive reasoning under adversarial conditions","Word-guessing game with deceptive feedback interpretation"]},"evals_count":3,"eval_summary_ids":["wordle-arena%2Fwordle-arena"],"reproducibility_summary":{"results_total":53,"has_reproducibility_gap_count":37,"populated_ratio_avg":0.9285714285714288},"provenance_summary":{"total_results":53,"total_groups":53,"multi_source_groups":30,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":53,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":53,"groups_with_variant_check":30,"groups_with_cross_party_check":30,"variant_divergent_count":23,"cross_party_divergent_count":23},"standalone_benchmarks":[{"key":"wordle-arena","display_name":"Wordle Arena","family_id":"wordle-arena","is_slice":false,"is_overall":true,"primary_metric_key":"score","has_card":true,"tags":{"domains":["adversarial reasoning","deductive reasoning","word puzzle"],"languages":["English"],"tasks":["Word-guessing game with deceptive feedback interpretation","Deductive reasoning under adversarial conditions"]},"metrics":[{"key":"mean-response-time","display_name":"Mean Response Time","sources":["Wordle Arena Project"],"models_count":42,"is_primary":false},{"key":"score","display_name":"Score","sources":["Dr. Chang Liu's Lab","Wordle Arena Project"],"models_count":51,"is_primary":true},{"key":"win-rate","display_name":"Win Rate","sources":["Dr. Chang Liu's Lab","Wordle Arena Project"],"models_count":53,"is_primary":false}],"slices":[],"summary_eval_ids":["wordle-arena%2Fwordle-arena"],"reproducibility_summary":{"results_total":53,"has_reproducibility_gap_count":37,"populated_ratio_avg":0.9285714285714288},"provenance_summary":{"total_results":53,"total_groups":53,"multi_source_groups":30,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":53,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":53,"groups_with_variant_check":30,"groups_with_cross_party_check":30,"variant_divergent_count":23,"cross_party_divergent_count":23},"is_primary":true,"derivedTags":["logical_reasoning"]}],"derivedTags":["logical_reasoning"]},{"key":"big-bench","display_name":"Big Bench","category":"other","tags":{"domains":["geometry","logical reasoning","mathematical reasoning","question answering","text generation"],"languages":["English"],"tasks":["Multiple choice","Question answering","Text generation"]},"evals_count":1,"eval_summary_ids":["big-bench-hard%2Fbig-bench-hard"],"reproducibility_summary":{"results_total":15,"has_reproducibility_gap_count":15,"populated_ratio_avg":0.9285714285714287},"provenance_summary":{"total_results":15,"total_groups":15,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":15,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":15,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"standalone_benchmarks":[{"key":"big-bench-hard","display_name":"BIG-Bench Hard","family_id":"big-bench","is_slice":false,"is_overall":false,"primary_metric_key":"accuracy","has_card":true,"tags":{"domains":["logical reasoning","mathematical reasoning","geometry","question answering","text generation"],"languages":["English"],"tasks":["Question answering","Multiple choice","Text generation"]},"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["Arcadia Impact"],"models_count":15,"is_primary":true}],"slices":[{"key":"big_bench_hard","display_name":"big_bench_hard","is_bare_stem":false,"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["Arcadia Impact"]}],"derivedTags":["general","applied_reasoning"]}],"summary_eval_ids":["big-bench-hard%2Fbig-bench-hard"],"reproducibility_summary":{"results_total":15,"has_reproducibility_gap_count":15,"populated_ratio_avg":0.9285714285714287},"provenance_summary":{"total_results":15,"total_groups":15,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":15,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":15,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":true,"derivedTags":["general","applied_reasoning"]}],"derivedTags":["general","applied_reasoning"]},{"key":"evalplus","display_name":"Evalplus","category":"other","tags":{"domains":[],"languages":[],"tasks":[]},"evals_count":1,"eval_summary_ids":["mbpp%2Fmbpp-plus"],"reproducibility_summary":{"results_total":12,"has_reproducibility_gap_count":12,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":12,"total_groups":12,"multi_source_groups":1,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":12,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":12,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"standalone_benchmarks":[{"key":"mbpp-plus","display_name":"MBPP-PLUS","family_id":"evalplus","is_slice":false,"is_overall":false,"primary_metric_key":"accuracy","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["Arcadia Impact"],"models_count":12,"is_primary":true}],"slices":[{"key":"mbpp","display_name":"mbpp","is_bare_stem":false,"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["Arcadia Impact"]}],"derivedTags":["software_engineering"]}],"summary_eval_ids":["mbpp%2Fmbpp-plus"],"reproducibility_summary":{"results_total":12,"has_reproducibility_gap_count":12,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":12,"total_groups":12,"multi_source_groups":1,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":12,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":12,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":true,"derivedTags":["software_engineering"]}],"derivedTags":["software_engineering"]},{"key":"global-mmlu","display_name":"Global Mmlu","category":"other","tags":{"domains":["multiple subjects"],"languages":["Arabic","Chinese","English","French","German","Hindi","Italian","Japanese","Korean","Portuguese","Spanish","bn","cy","id","my","sq","sw","yo"],"tasks":["Multiple-choice question answering"]},"evals_count":1,"eval_summary_ids":["global-mmlu-lite%2Fglobal-mmlu-lite"],"reproducibility_summary":{"results_total":24,"has_reproducibility_gap_count":24,"populated_ratio_avg":0.9285714285714283},"provenance_summary":{"total_results":24,"total_groups":24,"multi_source_groups":4,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":24,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":24,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"standalone_benchmarks":[{"key":"global-mmlu-lite","display_name":"Global MMLU Lite","family_id":"global-mmlu","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["multiple subjects"],"languages":["English","Arabic","bn","Spanish","French","Hindi","German","id","Italian","Japanese","Korean","my","Portuguese","Chinese","yo","sw","cy","sq"],"tasks":["Multiple-choice question answering"]},"metrics":[{"key":"score","display_name":"Score","sources":["kaggle"],"models_count":24,"is_primary":true}],"slices":[{"key":"arabic","display_name":"Arabic","is_bare_stem":false,"metrics":[{"key":"score","display_name":"Score","sources":["kaggle"]}],"derivedTags":["general"]},{"key":"bengali","display_name":"Bengali","is_bare_stem":false,"metrics":[{"key":"score","display_name":"Score","sources":["kaggle"]}],"derivedTags":["general"]},{"key":"burmese","display_name":"Burmese","is_bare_stem":false,"metrics":[{"key":"score","display_name":"Score","sources":["kaggle"]}],"derivedTags":["general"]},{"key":"chinese","display_name":"Chinese","is_bare_stem":false,"metrics":[{"key":"score","display_name":"Score","sources":["kaggle"]}],"derivedTags":["general"]},{"key":"culturally agnostic","display_name":"Culturally Agnostic","is_bare_stem":false,"metrics":[{"key":"score","display_name":"Score","sources":["kaggle"]}],"derivedTags":["general"]},{"key":"culturally sensitive","display_name":"Culturally Sensitive","is_bare_stem":false,"metrics":[{"key":"score","display_name":"Score","sources":["kaggle"]}],"derivedTags":["general"]},{"key":"english","display_name":"English","is_bare_stem":false,"metrics":[{"key":"score","display_name":"Score","sources":["kaggle"]}],"derivedTags":["general"]},{"key":"french","display_name":"French","is_bare_stem":false,"metrics":[{"key":"score","display_name":"Score","sources":["kaggle"]}],"derivedTags":["general"]},{"key":"german","display_name":"German","is_bare_stem":false,"metrics":[{"key":"score","display_name":"Score","sources":["kaggle"]}],"derivedTags":["general"]},{"key":"global mmlu lite","display_name":"Global MMLU Lite","is_bare_stem":false,"metrics":[{"key":"score","display_name":"Score","sources":["kaggle"]}],"derivedTags":["general"]},{"key":"hindi","display_name":"Hindi","is_bare_stem":false,"metrics":[{"key":"score","display_name":"Score","sources":["kaggle"]}],"derivedTags":["general"]},{"key":"indonesian","display_name":"Indonesian","is_bare_stem":false,"metrics":[{"key":"score","display_name":"Score","sources":["kaggle"]}],"derivedTags":["general"]},{"key":"italian","display_name":"Italian","is_bare_stem":false,"metrics":[{"key":"score","display_name":"Score","sources":["kaggle"]}],"derivedTags":["general"]},{"key":"japanese","display_name":"Japanese","is_bare_stem":false,"metrics":[{"key":"score","display_name":"Score","sources":["kaggle"]}],"derivedTags":["general"]},{"key":"korean","display_name":"Korean","is_bare_stem":false,"metrics":[{"key":"score","display_name":"Score","sources":["kaggle"]}],"derivedTags":["general"]},{"key":"portuguese","display_name":"Portuguese","is_bare_stem":false,"metrics":[{"key":"score","display_name":"Score","sources":["kaggle"]}],"derivedTags":["general"]},{"key":"spanish","display_name":"Spanish","is_bare_stem":false,"metrics":[{"key":"score","display_name":"Score","sources":["kaggle"]}],"derivedTags":["general"]},{"key":"swahili","display_name":"Swahili","is_bare_stem":false,"metrics":[{"key":"score","display_name":"Score","sources":["kaggle"]}],"derivedTags":["general"]},{"key":"yoruba","display_name":"Yoruba","is_bare_stem":false,"metrics":[{"key":"score","display_name":"Score","sources":["kaggle"]}],"derivedTags":["general"]}],"summary_eval_ids":["global-mmlu-lite%2Fglobal-mmlu-lite"],"reproducibility_summary":{"results_total":24,"has_reproducibility_gap_count":24,"populated_ratio_avg":0.9285714285714283},"provenance_summary":{"total_results":24,"total_groups":24,"multi_source_groups":4,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":24,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":24,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":true,"derivedTags":["general"]}],"derivedTags":["general"]},{"key":"gpqa","display_name":"GPQA family","category":"knowledge","tags":{"domains":[],"languages":[],"tasks":[]},"evals_count":1,"eval_summary_ids":["gpqa-diamond%2Fgpqa-diamond"],"reproducibility_summary":{"results_total":10,"has_reproducibility_gap_count":10,"populated_ratio_avg":0.10714285714285712},"provenance_summary":{"total_results":10,"total_groups":10,"multi_source_groups":2,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":10,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":10,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"standalone_benchmarks":[{"key":"gpqa-diamond","display_name":"GPQA Diamond","family_id":"gpqa","is_slice":false,"is_overall":false,"primary_metric_key":"accuracy","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["Arcadia Impact"],"models_count":10,"is_primary":true}],"slices":[{"key":"gpqa_diamond","display_name":"gpqa_diamond","is_bare_stem":false,"metrics":[{"key":"accuracy","display_name":"Accuracy","sources":["Arcadia Impact"]}],"derivedTags":["natural_sciences"]}],"summary_eval_ids":["gpqa-diamond%2Fgpqa-diamond"],"reproducibility_summary":{"results_total":10,"has_reproducibility_gap_count":10,"populated_ratio_avg":0.10714285714285712},"provenance_summary":{"total_results":10,"total_groups":10,"multi_source_groups":2,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":10,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":10,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":true,"derivedTags":["natural_sciences"]}],"derivedTags":["natural_sciences"]},{"key":"math","display_name":"Math","category":"other","tags":{"domains":["explanation generation","mathematics"],"languages":["English"],"tasks":["Final answer generation","Mathematical problem solving","Step-by-step solution generation","Text generation for solving mathematical problems"]},"evals_count":2,"eval_summary_ids":["artificial-analysis-llms%2Fmath-500","hfopenllm-v2%2Fmath-level-5"],"reproducibility_summary":{"results_total":4552,"has_reproducibility_gap_count":4552,"populated_ratio_avg":0.9285714285714114},"provenance_summary":{"total_results":4552,"total_groups":4552,"multi_source_groups":37,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":4552,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":4552,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"benchmarks":[{"key":"math-500","display_name":"MATH-500","family_id":"math","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["mathematics"],"languages":["English"],"tasks":["Text generation for solving mathematical problems"]},"metrics":[{"key":"score","display_name":"Score","sources":["Artificial Analysis"],"models_count":138,"is_primary":true}],"slices":[{"key":"artificial analysis","display_name":"artificial analysis","is_bare_stem":false,"metrics":[{"key":"score","display_name":"Score","sources":["Artificial Analysis"]}],"derivedTags":["general"]}],"summary_eval_ids":["artificial-analysis-llms%2Fmath-500"],"reproducibility_summary":{"results_total":138,"has_reproducibility_gap_count":138,"populated_ratio_avg":0.9285714285714298},"provenance_summary":{"total_results":138,"total_groups":138,"multi_source_groups":37,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":138,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":138,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":true,"derivedTags":["mathematics","general"]},{"key":"math-level-5","display_name":"MATH Level 5","family_id":"math","is_slice":false,"is_overall":false,"primary_metric_key":"exact-match","has_card":true,"tags":{"domains":["mathematics","explanation generation"],"languages":["English"],"tasks":["Mathematical problem solving","Step-by-step solution generation","Final answer generation"]},"metrics":[{"key":"exact-match","display_name":"Exact Match","sources":["Hugging Face"],"models_count":4414,"is_primary":true}],"slices":[],"summary_eval_ids":["hfopenllm-v2%2Fmath-level-5"],"reproducibility_summary":{"results_total":4414,"has_reproducibility_gap_count":4414,"populated_ratio_avg":0.928571428571411},"provenance_summary":{"total_results":4414,"total_groups":4414,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":4414,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":4414,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["mathematics"]}],"derivedTags":["mathematics","general"]},{"key":"swe-bench","display_name":"SWE-bench family","category":"agentic","tags":{"domains":["APIs","B2B services","Not specified","authentication","back-end systems","business applications","databases","developer tools","software engineering","user interfaces","web development"],"languages":["English","Not specified"],"tasks":["Automated GitHub issue resolution","Code generation and modification","Not specified","Producing code patches that satisfy requirements and pass tests","Resolving software issues (e.g., bug reports or feature requests)","Unit test verification"]},"evals_count":4,"eval_summary_ids":["hal%2Fswe-bench-verified-mini-hal","llm-stats%2Fswe-bench-pro","llm-stats%2Fswe-bench-verified","swe-bench-verified-mini%2Fswebench-verified-mini-mariushobbhahn"],"reproducibility_summary":{"results_total":59,"has_reproducibility_gap_count":59,"populated_ratio_avg":0.7615012106537531},"provenance_summary":{"total_results":59,"total_groups":59,"multi_source_groups":2,"first_party_only_groups":36,"source_type_distribution":{"first_party":38,"third_party":21,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":59,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"benchmarks":[{"key":"swe-bench-verified-mini-hal","display_name":"SWE-Bench-Verified-MINI-HAL","family_id":"swe-bench","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":false,"tags":{"domains":[],"languages":[],"tasks":[]},"metrics":[{"key":"score","display_name":"Score","sources":["Princeton SAgE Team"],"models_count":12,"is_primary":true}],"slices":[],"summary_eval_ids":["hal%2Fswe-bench-verified-mini-hal"],"reproducibility_summary":{"results_total":12,"has_reproducibility_gap_count":12,"populated_ratio_avg":0.10714285714285714},"provenance_summary":{"total_results":12,"total_groups":12,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":12,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":12,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["software_engineering"]},{"key":"swe-bench-pro","display_name":"SWE-Bench Pro","family_id":"swe-bench","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["software engineering","web development","back-end systems","APIs","databases","authentication","user interfaces","business applications","B2B services","developer tools"],"languages":["Not specified"],"tasks":["Resolving software issues (e.g., bug reports or feature requests)","Producing code patches that satisfy requirements and pass tests"]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":19,"is_primary":true}],"slices":[],"summary_eval_ids":["llm-stats%2Fswe-bench-pro"],"reproducibility_summary":{"results_total":19,"has_reproducibility_gap_count":19,"populated_ratio_avg":0.9285714285714285},"provenance_summary":{"total_results":19,"total_groups":19,"multi_source_groups":0,"first_party_only_groups":19,"source_type_distribution":{"first_party":19,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":19,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":true,"derivedTags":["software_engineering","agentic"]},{"key":"swe-bench-verified","display_name":"SWE-bench Verified","family_id":"swe-bench","is_slice":false,"is_overall":false,"primary_metric_key":"score","has_card":true,"tags":{"domains":["software engineering"],"languages":["English"],"tasks":["Automated GitHub issue resolution","Code generation and modification","Unit test verification"]},"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"],"models_count":19,"is_primary":true}],"slices":[{"key":"llm stats","display_name":"llm stats","is_bare_stem":false,"metrics":[{"key":"score","display_name":"Score","sources":["LLM Stats"]}],"derivedTags":["general"]}],"summary_eval_ids":["llm-stats%2Fswe-bench-verified","swe-bench-verified-leaderboard%2Fswe-bench-verified"],"reproducibility_summary":{"results_total":19,"has_reproducibility_gap_count":19,"populated_ratio_avg":0.9285714285714285},"provenance_summary":{"total_results":19,"total_groups":19,"multi_source_groups":2,"first_party_only_groups":17,"source_type_distribution":{"first_party":19,"third_party":0,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":19,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["software_engineering","general"]},{"key":"swebench-verified-mini-mariushobbhahn","display_name":"SWE-bench Verified Mini (MariusHobbhahn)","family_id":"swe-bench","is_slice":false,"is_overall":false,"primary_metric_key":"mean","has_card":true,"tags":{"domains":["Not specified"],"languages":["Not specified"],"tasks":["Not specified"]},"metrics":[{"key":"mean","display_name":"mean","sources":["Arcadia Impact"],"models_count":9,"is_primary":true},{"key":"std","display_name":"std","sources":["Arcadia Impact"],"models_count":9,"is_primary":false}],"slices":[],"summary_eval_ids":["swe-bench-verified-mini%2Fswebench-verified-mini-mariushobbhahn"],"reproducibility_summary":{"results_total":9,"has_reproducibility_gap_count":9,"populated_ratio_avg":0.9285714285714286},"provenance_summary":{"total_results":9,"total_groups":9,"multi_source_groups":0,"first_party_only_groups":0,"source_type_distribution":{"first_party":0,"third_party":9,"collaborative":0,"unspecified":0}},"comparability_summary":{"total_groups":9,"groups_with_variant_check":0,"groups_with_cross_party_check":0,"variant_divergent_count":0,"cross_party_divergent_count":0},"is_primary":false,"derivedTags":["software_engineering"]}],"derivedTags":["software_engineering","agentic","general"]}],"benchmarkIndex":[{"key":"aa-lcr","display_name":"AA-LCR","appearances":[{"family_key":"artificial-analysis","benchmark_key":"aa-lcr","eval_summary_ids":["artificial-analysis-llms%2Faa-lcr"],"is_canonical_home":false},{"family_key":"llm-stats","benchmark_key":"aa-lcr","eval_summary_ids":["llm-stats%2Faa-lcr"],"is_canonical_home":false}]},{"key":"aime","display_name":"AIME","appearances":[{"family_key":"artificial-analysis","benchmark_key":"aime","eval_summary_ids":["artificial-analysis-llms%2Faime"],"is_canonical_home":false},{"family_key":"llm-stats","benchmark_key":"aime","eval_summary_ids":["llm-stats%2Faime"],"is_canonical_home":false},{"family_key":"vals-ai","benchmark_key":"aime","eval_summary_ids":["vals-ai%2Faime"],"is_canonical_home":false}]},{"key":"aime-2024","display_name":"aime-2024","appearances":[{"family_key":"llm-stats","benchmark_key":"aime","eval_summary_ids":["llm-stats%2Faime"],"is_canonical_home":false},{"family_key":"vals-ai","benchmark_key":"aime","eval_summary_ids":["vals-ai%2Faime"],"is_canonical_home":false}]},{"key":"aime-2025","display_name":"aime-2025","appearances":[{"family_key":"artificial-analysis","benchmark_key":"aime","eval_summary_ids":["artificial-analysis-llms%2Faime"],"is_canonical_home":false},{"family_key":"llm-stats","benchmark_key":"aime","eval_summary_ids":["llm-stats%2Faime"],"is_canonical_home":false},{"family_key":"vals-ai","benchmark_key":"aime","eval_summary_ids":["vals-ai%2Faime"],"is_canonical_home":false}]},{"key":"air-bench-2024-13-harassment","display_name":"Harassment","appearances":[{"family_key":"agentharm","benchmark_key":"air-bench-2024-13-harassment","eval_summary_ids":["agentharm%2Fair-bench-2024-13-harassment"],"is_canonical_home":false},{"family_key":"helm","benchmark_key":"air-bench-2024","eval_summary_ids":["helm-air-bench%2Fair-bench-2024"],"is_canonical_home":false}]},{"key":"air-bench-2024-32-fraud","display_name":"Fraud","appearances":[{"family_key":"agentharm","benchmark_key":"air-bench-2024-32-fraud","eval_summary_ids":["agentharm%2Fair-bench-2024-32-fraud"],"is_canonical_home":false},{"family_key":"helm","benchmark_key":"air-bench-2024","eval_summary_ids":["helm-air-bench%2Fair-bench-2024"],"is_canonical_home":false}]},{"key":"apex-agents","display_name":"APEX Agents","appearances":[{"family_key":"apex-agents","benchmark_key":"apex-agents","eval_summary_ids":["apex-agents%2Fapex-agents"],"is_canonical_home":true},{"family_key":"llm-stats","benchmark_key":"apex-agents","eval_summary_ids":["llm-stats%2Fapex-agents"],"is_canonical_home":false}]},{"key":"apex-v1","display_name":"APEX v1","appearances":[{"family_key":"apex-agents","benchmark_key":"apex-v1","eval_summary_ids":["apex-agents%2Fapex-v1"],"is_canonical_home":false},{"family_key":"apex-v1","benchmark_key":"apex-v1","eval_summary_ids":["apex-v1%2Fapex-v1"],"is_canonical_home":true}]},{"key":"artificial-analysis-llms","display_name":"Artificial Analysis LLMs","appearances":[{"family_key":"artificial-analysis","benchmark_key":"artificial-analysis-llms","eval_summary_ids":["artificial-analysis-llms%2Fartificial-analysis-llms"],"is_canonical_home":false},{"family_key":"llm-stats","benchmark_key":"artificial-analysis-llms","eval_summary_ids":["llm-stats%2Fartificial-analysis-llms"],"is_canonical_home":false}]},{"key":"bbh","display_name":"BBH","appearances":[{"family_key":"hf-open-llm-v2","benchmark_key":"bbh","eval_summary_ids":["hfopenllm-v2%2Fbbh"],"is_canonical_home":false},{"family_key":"llm-stats","benchmark_key":"bbh","eval_summary_ids":["llm-stats%2Fbbh"],"is_canonical_home":false}]},{"key":"bbq","display_name":"BBQ","appearances":[{"family_key":"helm","benchmark_key":"bbq","eval_summary_ids":["helm-safety%2Fbbq"],"is_canonical_home":false},{"family_key":"openeval","benchmark_key":"bbq","eval_summary_ids":["openeval%2Fbbq"],"is_canonical_home":false}]},{"key":"big-bench-hard","display_name":"BIG-Bench Hard","appearances":[{"family_key":"llm-stats","benchmark_key":"big-bench","eval_summary_ids":["llm-stats%2Fbig-bench"],"is_canonical_home":false},{"family_key":"big-bench","benchmark_key":"big-bench-hard","eval_summary_ids":["big-bench-hard%2Fbig-bench-hard"],"is_canonical_home":false}]},{"key":"boolq","display_name":"BoolQ","appearances":[{"family_key":"helm","benchmark_key":"boolq","eval_summary_ids":["helm-classic%2Fboolq"],"is_canonical_home":false},{"family_key":"llm-stats","benchmark_key":"boolq","eval_summary_ids":["llm-stats%2Fboolq"],"is_canonical_home":false},{"family_key":"openeval","benchmark_key":"boolq","eval_summary_ids":["openeval%2Fboolq"],"is_canonical_home":false}]},{"key":"cnn-dailymail","display_name":"CNN/DailyMail","appearances":[{"family_key":"helm","benchmark_key":"cnn-dailymail","eval_summary_ids":["helm-classic%2Fcnn-dailymail"],"is_canonical_home":false},{"family_key":"openeval","benchmark_key":"cnn-dailymail","eval_summary_ids":["openeval%2Fcnn-dailymail"],"is_canonical_home":false}]},{"key":"cybench","display_name":"Cybench","appearances":[{"family_key":"cybench","benchmark_key":"cybench","eval_summary_ids":["cybench%2Fcybench"],"is_canonical_home":true},{"family_key":"llm-stats","benchmark_key":"cybench","eval_summary_ids":["llm-stats%2Fcybench"],"is_canonical_home":false}]},{"key":"disinformation","display_name":"Disinformation","appearances":[{"family_key":"agentharm","benchmark_key":"disinformation","eval_summary_ids":["agentharm%2Fdisinformation"],"is_canonical_home":false},{"family_key":"openeval","benchmark_key":"disinformation","eval_summary_ids":["openeval%2Fdisinformation"],"is_canonical_home":false}]},{"key":"facts-grounding","display_name":"Facts Grounding","appearances":[{"family_key":"facts-grounding","benchmark_key":"facts-grounding","eval_summary_ids":["facts-grounding%2Ffacts-grounding"],"is_canonical_home":true},{"family_key":"llm-stats","benchmark_key":"facts-grounding","eval_summary_ids":["llm-stats%2Ffacts-grounding"],"is_canonical_home":false}]},{"key":"global-mmlu-lite","display_name":"Global MMLU Lite","appearances":[{"family_key":"llm-stats","benchmark_key":"global-mmlu","eval_summary_ids":["llm-stats%2Fglobal-mmlu"],"is_canonical_home":false},{"family_key":"global-mmlu","benchmark_key":"global-mmlu-lite","eval_summary_ids":["global-mmlu-lite%2Fglobal-mmlu-lite"],"is_canonical_home":false}]},{"key":"gpqa","display_name":"GPQA","appearances":[{"family_key":"artificial-analysis","benchmark_key":"gpqa","eval_summary_ids":["artificial-analysis-llms%2Fgpqa"],"is_canonical_home":false},{"family_key":"helm","benchmark_key":"gpqa","eval_summary_ids":["helm-capabilities%2Fgpqa"],"is_canonical_home":false},{"family_key":"hf-open-llm-v2","benchmark_key":"gpqa","eval_summary_ids":["hfopenllm-v2%2Fgpqa"],"is_canonical_home":false},{"family_key":"llm-stats","benchmark_key":"gpqa","eval_summary_ids":["llm-stats%2Fgpqa"],"is_canonical_home":false},{"family_key":"openeval","benchmark_key":"gpqa","eval_summary_ids":["openeval%2Fgpqa"],"is_canonical_home":false}]},{"key":"gpqa-diamond","display_name":"GPQA Diamond","appearances":[{"family_key":"wasp","benchmark_key":"gpqa-diamond","eval_summary_ids":["wasp%2Fgpqa-diamond"],"is_canonical_home":false},{"family_key":"gpqa","benchmark_key":"gpqa-diamond","eval_summary_ids":["gpqa-diamond%2Fgpqa-diamond"],"is_canonical_home":false}]},{"key":"gsm8k","display_name":"GSM8K","appearances":[{"family_key":"gsm8k","benchmark_key":"gsm8k","eval_summary_ids":["gsm8k%2Fgsm8k"],"is_canonical_home":true},{"family_key":"helm","benchmark_key":"gsm8k","eval_summary_ids":["helm-lite%2Fgsm8k"],"is_canonical_home":false},{"family_key":"llm-stats","benchmark_key":"gsm8k","eval_summary_ids":["llm-stats%2Fgsm8k"],"is_canonical_home":false}]},{"key":"hellaswag","display_name":"HellaSwag","appearances":[{"family_key":"hellaswag","benchmark_key":"hellaswag","eval_summary_ids":["hellaswag%2Fhellaswag"],"is_canonical_home":true},{"family_key":"helm","benchmark_key":"hellaswag","eval_summary_ids":["helm-classic%2Fhellaswag"],"is_canonical_home":false},{"family_key":"llm-stats","benchmark_key":"hellaswag","eval_summary_ids":["llm-stats%2Fhellaswag"],"is_canonical_home":false}]},{"key":"hle","display_name":"Humanity's Last Exam","appearances":[{"family_key":"artificial-analysis","benchmark_key":"hle","eval_summary_ids":["artificial-analysis-llms%2Fhle"],"is_canonical_home":false},{"family_key":"llm-stats","benchmark_key":"hle","eval_summary_ids":["llm-stats%2Fhle"],"is_canonical_home":false}]},{"key":"humaneval","display_name":"HumanEval","appearances":[{"family_key":"llm-stats","benchmark_key":"humaneval","eval_summary_ids":["llm-stats%2Fhumaneval"],"is_canonical_home":false},{"family_key":"openai-humaneval","benchmark_key":"humaneval","eval_summary_ids":["openai-humaneval%2Fhumaneval"],"is_canonical_home":false}]},{"key":"ifbench","display_name":"IFBench","appearances":[{"family_key":"artificial-analysis","benchmark_key":"ifbench","eval_summary_ids":["artificial-analysis-llms%2Fifbench"],"is_canonical_home":false},{"family_key":"llm-stats","benchmark_key":"ifbench","eval_summary_ids":["llm-stats%2Fifbench"],"is_canonical_home":false}]},{"key":"ifeval","display_name":"IFEval","appearances":[{"family_key":"helm","benchmark_key":"ifeval","eval_summary_ids":["helm-capabilities%2Fifeval"],"is_canonical_home":false},{"family_key":"hf-open-llm-v2","benchmark_key":"ifeval","eval_summary_ids":["hfopenllm-v2%2Fifeval"],"is_canonical_home":false},{"family_key":"ifeval","benchmark_key":"ifeval","eval_summary_ids":["ifeval%2Fifeval"],"is_canonical_home":true},{"family_key":"llm-stats","benchmark_key":"ifeval","eval_summary_ids":["llm-stats%2Fifeval"],"is_canonical_home":false},{"family_key":"openeval","benchmark_key":"ifeval","eval_summary_ids":["openeval%2Fifeval"],"is_canonical_home":false}]},{"key":"imdb","display_name":"IMDb","appearances":[{"family_key":"helm","benchmark_key":"imdb","eval_summary_ids":["helm-classic%2Fimdb"],"is_canonical_home":false},{"family_key":"openeval","benchmark_key":"imdb","eval_summary_ids":["openeval%2Fimdb"],"is_canonical_home":false}]},{"key":"investment banking","display_name":"investment banking","appearances":[{"family_key":"apex-agents","benchmark_key":"apex-v1","eval_summary_ids":["apex-agents%2Fapex-v1"],"is_canonical_home":false},{"family_key":"apex-v1","benchmark_key":"apex-v1","eval_summary_ids":["apex-v1%2Fapex-v1"],"is_canonical_home":false}]},{"key":"legalbench","display_name":"LegalBench","appearances":[{"family_key":"helm","benchmark_key":"legalbench","eval_summary_ids":["helm-lite%2Flegalbench"],"is_canonical_home":false},{"family_key":"vals-ai","benchmark_key":"legalbench","eval_summary_ids":["vals-ai%2Flegalbench"],"is_canonical_home":false}]},{"key":"livebench","display_name":"LiveBench","appearances":[{"family_key":"live-bench","benchmark_key":"livebench","eval_summary_ids":["live-bench%2Flivebench"],"is_canonical_home":false},{"family_key":"llm-stats","benchmark_key":"livebench","eval_summary_ids":["llm-stats%2Flivebench"],"is_canonical_home":false}]},{"key":"livecodebench","display_name":"LiveCodeBench","appearances":[{"family_key":"artificial-analysis","benchmark_key":"livecodebench","eval_summary_ids":["artificial-analysis-llms%2Flivecodebench"],"is_canonical_home":false},{"family_key":"llm-stats","benchmark_key":"livecodebench","eval_summary_ids":["llm-stats%2Flivecodebench"],"is_canonical_home":false}]},{"key":"livecodebench-pro","display_name":"LiveCodeBench Pro","appearances":[{"family_key":"livecodebenchpro","benchmark_key":"livecodebench-pro","eval_summary_ids":["livecodebenchpro%2Flivecodebench-pro"],"is_canonical_home":false},{"family_key":"llm-stats","benchmark_key":"livecodebench","eval_summary_ids":["llm-stats%2Flivecodebench"],"is_canonical_home":false}]},{"key":"math","display_name":"MATH","appearances":[{"family_key":"helm","benchmark_key":"math","eval_summary_ids":["helm-lite%2Fmath"],"is_canonical_home":false},{"family_key":"llm-stats","benchmark_key":"math","eval_summary_ids":["llm-stats%2Fmath"],"is_canonical_home":false},{"family_key":"reward-bench","benchmark_key":"math","eval_summary_ids":["reward-bench%2Fmath"],"is_canonical_home":false},{"family_key":"reward-bench","benchmark_key":"math","eval_summary_ids":["reward-bench-2%2Fmath"],"is_canonical_home":false},{"family_key":"wasp","benchmark_key":"math","eval_summary_ids":["wasp%2Fmath"],"is_canonical_home":false}]},{"key":"math-500","display_name":"MATH-500","appearances":[{"family_key":"artificial-analysis","benchmark_key":"math-500","eval_summary_ids":["artificial-analysis-llms%2Fmath-500"],"is_canonical_home":false},{"family_key":"llm-stats","benchmark_key":"math","eval_summary_ids":["llm-stats%2Fmath"],"is_canonical_home":false},{"family_key":"vals-ai","benchmark_key":"math-500","eval_summary_ids":["vals-ai%2Fmath-500"],"is_canonical_home":false},{"family_key":"math","benchmark_key":"math-500","eval_summary_ids":["artificial-analysis-llms%2Fmath-500"],"is_canonical_home":false}]},{"key":"mbpp","display_name":"MBPP","appearances":[{"family_key":"llm-stats","benchmark_key":"mbpp","eval_summary_ids":["llm-stats%2Fmbpp"],"is_canonical_home":false},{"family_key":"evalplus","benchmark_key":"mbpp-plus","eval_summary_ids":["mbpp%2Fmbpp-plus"],"is_canonical_home":false}]},{"key":"mbpp-plus","display_name":"MBPP+","appearances":[{"family_key":"llm-stats","benchmark_key":"evalplus","eval_summary_ids":["llm-stats%2Fevalplus"],"is_canonical_home":false},{"family_key":"evalplus","benchmark_key":"mbpp-plus","eval_summary_ids":["mbpp%2Fmbpp-plus"],"is_canonical_home":false}]},{"key":"medqa","display_name":"MedQA","appearances":[{"family_key":"helm","benchmark_key":"medqa","eval_summary_ids":["helm-lite%2Fmedqa"],"is_canonical_home":false},{"family_key":"vals-ai","benchmark_key":"medqa","eval_summary_ids":["vals-ai%2Fmedqa"],"is_canonical_home":false}]},{"key":"mgsm","display_name":"MGSM","appearances":[{"family_key":"llm-stats","benchmark_key":"mgsm","eval_summary_ids":["llm-stats%2Fmgsm"],"is_canonical_home":false},{"family_key":"vals-ai","benchmark_key":"mgsm","eval_summary_ids":["vals-ai%2Fmgsm"],"is_canonical_home":false}]},{"key":"mmlu","display_name":"MMLU","appearances":[{"family_key":"helm","benchmark_key":"mmlu","eval_summary_ids":["helm-classic%2Fmmlu"],"is_canonical_home":false},{"family_key":"helm","benchmark_key":"mmlu","eval_summary_ids":["helm-lite%2Fmmlu"],"is_canonical_home":false},{"family_key":"helm","benchmark_key":"mmlu","eval_summary_ids":["helm-mmlu%2Fmmlu"],"is_canonical_home":false},{"family_key":"llm-stats","benchmark_key":"mmlu","eval_summary_ids":["llm-stats%2Fmmlu"],"is_canonical_home":false}]},{"key":"mmlu-pro","display_name":"MMLU-Pro","appearances":[{"family_key":"artificial-analysis","benchmark_key":"mmlu-pro","eval_summary_ids":["artificial-analysis-llms%2Fmmlu-pro"],"is_canonical_home":false},{"family_key":"helm","benchmark_key":"mmlu-pro","eval_summary_ids":["helm-capabilities%2Fmmlu-pro"],"is_canonical_home":false},{"family_key":"hf-open-llm-v2","benchmark_key":"mmlu-pro","eval_summary_ids":["hfopenllm-v2%2Fmmlu-pro"],"is_canonical_home":false},{"family_key":"llm-stats","benchmark_key":"mmlu-pro","eval_summary_ids":["llm-stats%2Fmmlu-pro"],"is_canonical_home":false},{"family_key":"mmlu-pro-leaderboard","benchmark_key":"mmlu-pro","eval_summary_ids":["mmlu-pro-leaderboard%2Fmmlu-pro"],"is_canonical_home":false},{"family_key":"openeval","benchmark_key":"mmlu-pro","eval_summary_ids":["openeval%2Fmmlu-pro"],"is_canonical_home":false},{"family_key":"vals-ai","benchmark_key":"mmlu-pro","eval_summary_ids":["vals-ai%2Fmmlu-pro"],"is_canonical_home":false}]},{"key":"mmmu","display_name":"MMMU","appearances":[{"family_key":"llm-stats","benchmark_key":"mmmu","eval_summary_ids":["llm-stats%2Fmmmu"],"is_canonical_home":false},{"family_key":"vals-ai","benchmark_key":"mmmu","eval_summary_ids":["vals-ai%2Fmmmu"],"is_canonical_home":false}]},{"key":"musr","display_name":"MuSR","appearances":[{"family_key":"hf-open-llm-v2","benchmark_key":"musr","eval_summary_ids":["hfopenllm-v2%2Fmusr"],"is_canonical_home":false},{"family_key":"llm-stats","benchmark_key":"musr","eval_summary_ids":["llm-stats%2Fmusr"],"is_canonical_home":false}]},{"key":"omni-math","display_name":"Omni-MATH","appearances":[{"family_key":"helm","benchmark_key":"omni-math","eval_summary_ids":["helm-capabilities%2Fomni-math"],"is_canonical_home":false},{"family_key":"openeval","benchmark_key":"omni-math","eval_summary_ids":["openeval%2Fomni-math"],"is_canonical_home":false}]},{"key":"openbookqa","display_name":"OpenbookQA","appearances":[{"family_key":"helm","benchmark_key":"openbookqa","eval_summary_ids":["helm-classic%2Fopenbookqa"],"is_canonical_home":false},{"family_key":"helm","benchmark_key":"openbookqa","eval_summary_ids":["helm-lite%2Fopenbookqa"],"is_canonical_home":false},{"family_key":"llm-stats","benchmark_key":"openbookqa","eval_summary_ids":["llm-stats%2Fopenbookqa"],"is_canonical_home":false}]},{"key":"piqa","display_name":"Piqa","appearances":[{"family_key":"llm-stats","benchmark_key":"piqa","eval_summary_ids":["llm-stats%2Fpiqa"],"is_canonical_home":false},{"family_key":"piqa","benchmark_key":"piqa","eval_summary_ids":["piqa%2Fpiqa"],"is_canonical_home":true}]},{"key":"rewardbench-reasoning","display_name":"RewardBench Reasoning","appearances":[{"family_key":"reward-bench","benchmark_key":"rewardbench","eval_summary_ids":["reward-bench%2Frewardbench"],"is_canonical_home":false},{"family_key":"wasp","benchmark_key":"rewardbench-reasoning","eval_summary_ids":["wasp%2Frewardbench-reasoning"],"is_canonical_home":false}]},{"key":"scicode","display_name":"SciCode","appearances":[{"family_key":"artificial-analysis","benchmark_key":"scicode","eval_summary_ids":["artificial-analysis-llms%2Fscicode"],"is_canonical_home":false},{"family_key":"hal","benchmark_key":"scicode","eval_summary_ids":["hal%2Fscicode"],"is_canonical_home":false},{"family_key":"llm-stats","benchmark_key":"scicode","eval_summary_ids":["llm-stats%2Fscicode"],"is_canonical_home":false}]},{"key":"swe-bench","display_name":"SWE-bench","appearances":[{"family_key":"exgentic-open-agent","benchmark_key":"swe-bench","eval_summary_ids":["exgentic-open-agent%2Fswe-bench"],"is_canonical_home":false},{"family_key":"vals-ai","benchmark_key":"swe-bench","eval_summary_ids":["vals-ai%2Fswe-bench"],"is_canonical_home":false}]},{"key":"tau-bench-airline","display_name":"Tau Bench Airline","appearances":[{"family_key":"hal","benchmark_key":"tau-bench-airline","eval_summary_ids":["hal%2Ftau-bench-airline"],"is_canonical_home":false},{"family_key":"llm-stats","benchmark_key":"tau-bench","eval_summary_ids":["llm-stats%2Ftau-bench"],"is_canonical_home":false}]},{"key":"tau2-airline","display_name":"Tau2-Bench Airline","appearances":[{"family_key":"exgentic-open-agent","benchmark_key":"tau2-airline","eval_summary_ids":["exgentic-open-agent%2Ftau2-airline"],"is_canonical_home":false},{"family_key":"llm-stats","benchmark_key":"tau2-airline","eval_summary_ids":["llm-stats%2Ftau2-airline"],"is_canonical_home":false}]},{"key":"tau2-retail","display_name":"Tau2-Bench Retail","appearances":[{"family_key":"exgentic-open-agent","benchmark_key":"tau2-retail","eval_summary_ids":["exgentic-open-agent%2Ftau2-retail"],"is_canonical_home":false},{"family_key":"llm-stats","benchmark_key":"tau2-retail","eval_summary_ids":["llm-stats%2Ftau2-retail"],"is_canonical_home":false}]},{"key":"tau2-telecom","display_name":"Tau2-Bench Telecom","appearances":[{"family_key":"exgentic-open-agent","benchmark_key":"tau2-telecom","eval_summary_ids":["exgentic-open-agent%2Ftau2-telecom"],"is_canonical_home":false},{"family_key":"llm-stats","benchmark_key":"tau2-telecom","eval_summary_ids":["llm-stats%2Ftau2-telecom"],"is_canonical_home":false}]},{"key":"terminal-bench","display_name":"Terminal Bench","appearances":[{"family_key":"llm-stats","benchmark_key":"terminal-bench","eval_summary_ids":["llm-stats%2Fterminal-bench"],"is_canonical_home":false},{"family_key":"vals-ai","benchmark_key":"terminal-bench","eval_summary_ids":["vals-ai%2Fterminal-bench"],"is_canonical_home":false}]},{"key":"terminalbench-hard","display_name":"TerminalBench Hard","appearances":[{"family_key":"artificial-analysis","benchmark_key":"terminalbench-hard","eval_summary_ids":["artificial-analysis-llms%2Fterminalbench-hard"],"is_canonical_home":false},{"family_key":"vals-ai","benchmark_key":"terminal-bench","eval_summary_ids":["vals-ai%2Fterminal-bench"],"is_canonical_home":false}]},{"key":"truthfulqa","display_name":"TruthfulQA","appearances":[{"family_key":"helm","benchmark_key":"truthfulqa","eval_summary_ids":["helm-classic%2Ftruthfulqa"],"is_canonical_home":false},{"family_key":"llm-stats","benchmark_key":"truthfulqa","eval_summary_ids":["llm-stats%2Ftruthfulqa"],"is_canonical_home":false},{"family_key":"openeval","benchmark_key":"truthfulqa","eval_summary_ids":["openeval%2Ftruthfulqa"],"is_canonical_home":false}]},{"key":"xstest","display_name":"Xstest","appearances":[{"family_key":"helm","benchmark_key":"xstest","eval_summary_ids":["helm-safety%2Fxstest"],"is_canonical_home":false},{"family_key":"llm-stats","benchmark_key":"xstest","eval_summary_ids":["llm-stats%2Fxstest"],"is_canonical_home":false}]},{"key":"xsum","display_name":"XSum","appearances":[{"family_key":"helm","benchmark_key":"xsum","eval_summary_ids":["helm-classic%2Fxsum"],"is_canonical_home":false},{"family_key":"openeval","benchmark_key":"xsum","eval_summary_ids":["openeval%2Fxsum"],"is_canonical_home":false}]}],"overlapsByFamily":[["artificial-analysis",[{"canonicalKey":"aa-lcr","canonicalDisplayName":"AA-LCR","family_key":"artificial-analysis","benchmark_key":"aa-lcr","eval_summary_ids":["artificial-analysis-llms%2Faa-lcr"],"is_canonical_home":false},{"canonicalKey":"aime","canonicalDisplayName":"AIME","family_key":"artificial-analysis","benchmark_key":"aime","eval_summary_ids":["artificial-analysis-llms%2Faime"],"is_canonical_home":false},{"canonicalKey":"aime-2025","canonicalDisplayName":"aime-2025","family_key":"artificial-analysis","benchmark_key":"aime","eval_summary_ids":["artificial-analysis-llms%2Faime"],"is_canonical_home":false},{"canonicalKey":"artificial-analysis-llms","canonicalDisplayName":"Artificial Analysis LLMs","family_key":"artificial-analysis","benchmark_key":"artificial-analysis-llms","eval_summary_ids":["artificial-analysis-llms%2Fartificial-analysis-llms"],"is_canonical_home":false},{"canonicalKey":"gpqa","canonicalDisplayName":"GPQA","family_key":"artificial-analysis","benchmark_key":"gpqa","eval_summary_ids":["artificial-analysis-llms%2Fgpqa"],"is_canonical_home":false},{"canonicalKey":"hle","canonicalDisplayName":"Humanity's Last Exam","family_key":"artificial-analysis","benchmark_key":"hle","eval_summary_ids":["artificial-analysis-llms%2Fhle"],"is_canonical_home":false},{"canonicalKey":"ifbench","canonicalDisplayName":"IFBench","family_key":"artificial-analysis","benchmark_key":"ifbench","eval_summary_ids":["artificial-analysis-llms%2Fifbench"],"is_canonical_home":false},{"canonicalKey":"livecodebench","canonicalDisplayName":"LiveCodeBench","family_key":"artificial-analysis","benchmark_key":"livecodebench","eval_summary_ids":["artificial-analysis-llms%2Flivecodebench"],"is_canonical_home":false},{"canonicalKey":"math-500","canonicalDisplayName":"MATH-500","family_key":"artificial-analysis","benchmark_key":"math-500","eval_summary_ids":["artificial-analysis-llms%2Fmath-500"],"is_canonical_home":false},{"canonicalKey":"mmlu-pro","canonicalDisplayName":"MMLU-Pro","family_key":"artificial-analysis","benchmark_key":"mmlu-pro","eval_summary_ids":["artificial-analysis-llms%2Fmmlu-pro"],"is_canonical_home":false},{"canonicalKey":"scicode","canonicalDisplayName":"SciCode","family_key":"artificial-analysis","benchmark_key":"scicode","eval_summary_ids":["artificial-analysis-llms%2Fscicode"],"is_canonical_home":false},{"canonicalKey":"terminalbench-hard","canonicalDisplayName":"TerminalBench Hard","family_key":"artificial-analysis","benchmark_key":"terminalbench-hard","eval_summary_ids":["artificial-analysis-llms%2Fterminalbench-hard"],"is_canonical_home":false}]],["llm-stats",[{"canonicalKey":"aa-lcr","canonicalDisplayName":"AA-LCR","family_key":"llm-stats","benchmark_key":"aa-lcr","eval_summary_ids":["llm-stats%2Faa-lcr"],"is_canonical_home":false},{"canonicalKey":"aime","canonicalDisplayName":"AIME","family_key":"llm-stats","benchmark_key":"aime","eval_summary_ids":["llm-stats%2Faime"],"is_canonical_home":false},{"canonicalKey":"aime-2024","canonicalDisplayName":"aime-2024","family_key":"llm-stats","benchmark_key":"aime","eval_summary_ids":["llm-stats%2Faime"],"is_canonical_home":false},{"canonicalKey":"aime-2025","canonicalDisplayName":"aime-2025","family_key":"llm-stats","benchmark_key":"aime","eval_summary_ids":["llm-stats%2Faime"],"is_canonical_home":false},{"canonicalKey":"apex-agents","canonicalDisplayName":"APEX Agents","family_key":"llm-stats","benchmark_key":"apex-agents","eval_summary_ids":["llm-stats%2Fapex-agents"],"is_canonical_home":false},{"canonicalKey":"artificial-analysis-llms","canonicalDisplayName":"Artificial Analysis LLMs","family_key":"llm-stats","benchmark_key":"artificial-analysis-llms","eval_summary_ids":["llm-stats%2Fartificial-analysis-llms"],"is_canonical_home":false},{"canonicalKey":"bbh","canonicalDisplayName":"BBH","family_key":"llm-stats","benchmark_key":"bbh","eval_summary_ids":["llm-stats%2Fbbh"],"is_canonical_home":false},{"canonicalKey":"big-bench-hard","canonicalDisplayName":"BIG-Bench Hard","family_key":"llm-stats","benchmark_key":"big-bench","eval_summary_ids":["llm-stats%2Fbig-bench"],"is_canonical_home":false},{"canonicalKey":"boolq","canonicalDisplayName":"BoolQ","family_key":"llm-stats","benchmark_key":"boolq","eval_summary_ids":["llm-stats%2Fboolq"],"is_canonical_home":false},{"canonicalKey":"cybench","canonicalDisplayName":"Cybench","family_key":"llm-stats","benchmark_key":"cybench","eval_summary_ids":["llm-stats%2Fcybench"],"is_canonical_home":false},{"canonicalKey":"facts-grounding","canonicalDisplayName":"Facts Grounding","family_key":"llm-stats","benchmark_key":"facts-grounding","eval_summary_ids":["llm-stats%2Ffacts-grounding"],"is_canonical_home":false},{"canonicalKey":"global-mmlu-lite","canonicalDisplayName":"Global MMLU Lite","family_key":"llm-stats","benchmark_key":"global-mmlu","eval_summary_ids":["llm-stats%2Fglobal-mmlu"],"is_canonical_home":false},{"canonicalKey":"gpqa","canonicalDisplayName":"GPQA","family_key":"llm-stats","benchmark_key":"gpqa","eval_summary_ids":["llm-stats%2Fgpqa"],"is_canonical_home":false},{"canonicalKey":"gsm8k","canonicalDisplayName":"GSM8K","family_key":"llm-stats","benchmark_key":"gsm8k","eval_summary_ids":["llm-stats%2Fgsm8k"],"is_canonical_home":false},{"canonicalKey":"hellaswag","canonicalDisplayName":"HellaSwag","family_key":"llm-stats","benchmark_key":"hellaswag","eval_summary_ids":["llm-stats%2Fhellaswag"],"is_canonical_home":false},{"canonicalKey":"hle","canonicalDisplayName":"Humanity's Last Exam","family_key":"llm-stats","benchmark_key":"hle","eval_summary_ids":["llm-stats%2Fhle"],"is_canonical_home":false},{"canonicalKey":"humaneval","canonicalDisplayName":"HumanEval","family_key":"llm-stats","benchmark_key":"humaneval","eval_summary_ids":["llm-stats%2Fhumaneval"],"is_canonical_home":false},{"canonicalKey":"ifbench","canonicalDisplayName":"IFBench","family_key":"llm-stats","benchmark_key":"ifbench","eval_summary_ids":["llm-stats%2Fifbench"],"is_canonical_home":false},{"canonicalKey":"ifeval","canonicalDisplayName":"IFEval","family_key":"llm-stats","benchmark_key":"ifeval","eval_summary_ids":["llm-stats%2Fifeval"],"is_canonical_home":false},{"canonicalKey":"livebench","canonicalDisplayName":"LiveBench","family_key":"llm-stats","benchmark_key":"livebench","eval_summary_ids":["llm-stats%2Flivebench"],"is_canonical_home":false},{"canonicalKey":"livecodebench","canonicalDisplayName":"LiveCodeBench","family_key":"llm-stats","benchmark_key":"livecodebench","eval_summary_ids":["llm-stats%2Flivecodebench"],"is_canonical_home":false},{"canonicalKey":"livecodebench-pro","canonicalDisplayName":"LiveCodeBench Pro","family_key":"llm-stats","benchmark_key":"livecodebench","eval_summary_ids":["llm-stats%2Flivecodebench"],"is_canonical_home":false},{"canonicalKey":"math","canonicalDisplayName":"MATH","family_key":"llm-stats","benchmark_key":"math","eval_summary_ids":["llm-stats%2Fmath"],"is_canonical_home":false},{"canonicalKey":"math-500","canonicalDisplayName":"MATH-500","family_key":"llm-stats","benchmark_key":"math","eval_summary_ids":["llm-stats%2Fmath"],"is_canonical_home":false},{"canonicalKey":"mbpp","canonicalDisplayName":"MBPP","family_key":"llm-stats","benchmark_key":"mbpp","eval_summary_ids":["llm-stats%2Fmbpp"],"is_canonical_home":false},{"canonicalKey":"mbpp-plus","canonicalDisplayName":"MBPP+","family_key":"llm-stats","benchmark_key":"evalplus","eval_summary_ids":["llm-stats%2Fevalplus"],"is_canonical_home":false},{"canonicalKey":"mgsm","canonicalDisplayName":"MGSM","family_key":"llm-stats","benchmark_key":"mgsm","eval_summary_ids":["llm-stats%2Fmgsm"],"is_canonical_home":false},{"canonicalKey":"mmlu","canonicalDisplayName":"MMLU","family_key":"llm-stats","benchmark_key":"mmlu","eval_summary_ids":["llm-stats%2Fmmlu"],"is_canonical_home":false},{"canonicalKey":"mmlu-pro","canonicalDisplayName":"MMLU-Pro","family_key":"llm-stats","benchmark_key":"mmlu-pro","eval_summary_ids":["llm-stats%2Fmmlu-pro"],"is_canonical_home":false},{"canonicalKey":"mmmu","canonicalDisplayName":"MMMU","family_key":"llm-stats","benchmark_key":"mmmu","eval_summary_ids":["llm-stats%2Fmmmu"],"is_canonical_home":false},{"canonicalKey":"musr","canonicalDisplayName":"MuSR","family_key":"llm-stats","benchmark_key":"musr","eval_summary_ids":["llm-stats%2Fmusr"],"is_canonical_home":false},{"canonicalKey":"openbookqa","canonicalDisplayName":"OpenbookQA","family_key":"llm-stats","benchmark_key":"openbookqa","eval_summary_ids":["llm-stats%2Fopenbookqa"],"is_canonical_home":false},{"canonicalKey":"piqa","canonicalDisplayName":"Piqa","family_key":"llm-stats","benchmark_key":"piqa","eval_summary_ids":["llm-stats%2Fpiqa"],"is_canonical_home":false},{"canonicalKey":"scicode","canonicalDisplayName":"SciCode","family_key":"llm-stats","benchmark_key":"scicode","eval_summary_ids":["llm-stats%2Fscicode"],"is_canonical_home":false},{"canonicalKey":"tau-bench-airline","canonicalDisplayName":"Tau Bench Airline","family_key":"llm-stats","benchmark_key":"tau-bench","eval_summary_ids":["llm-stats%2Ftau-bench"],"is_canonical_home":false},{"canonicalKey":"tau2-airline","canonicalDisplayName":"Tau2-Bench Airline","family_key":"llm-stats","benchmark_key":"tau2-airline","eval_summary_ids":["llm-stats%2Ftau2-airline"],"is_canonical_home":false},{"canonicalKey":"tau2-retail","canonicalDisplayName":"Tau2-Bench Retail","family_key":"llm-stats","benchmark_key":"tau2-retail","eval_summary_ids":["llm-stats%2Ftau2-retail"],"is_canonical_home":false},{"canonicalKey":"tau2-telecom","canonicalDisplayName":"Tau2-Bench Telecom","family_key":"llm-stats","benchmark_key":"tau2-telecom","eval_summary_ids":["llm-stats%2Ftau2-telecom"],"is_canonical_home":false},{"canonicalKey":"terminal-bench","canonicalDisplayName":"Terminal Bench","family_key":"llm-stats","benchmark_key":"terminal-bench","eval_summary_ids":["llm-stats%2Fterminal-bench"],"is_canonical_home":false},{"canonicalKey":"truthfulqa","canonicalDisplayName":"TruthfulQA","family_key":"llm-stats","benchmark_key":"truthfulqa","eval_summary_ids":["llm-stats%2Ftruthfulqa"],"is_canonical_home":false},{"canonicalKey":"xstest","canonicalDisplayName":"Xstest","family_key":"llm-stats","benchmark_key":"xstest","eval_summary_ids":["llm-stats%2Fxstest"],"is_canonical_home":false}]],["vals-ai",[{"canonicalKey":"aime","canonicalDisplayName":"AIME","family_key":"vals-ai","benchmark_key":"aime","eval_summary_ids":["vals-ai%2Faime"],"is_canonical_home":false},{"canonicalKey":"aime-2024","canonicalDisplayName":"aime-2024","family_key":"vals-ai","benchmark_key":"aime","eval_summary_ids":["vals-ai%2Faime"],"is_canonical_home":false},{"canonicalKey":"aime-2025","canonicalDisplayName":"aime-2025","family_key":"vals-ai","benchmark_key":"aime","eval_summary_ids":["vals-ai%2Faime"],"is_canonical_home":false},{"canonicalKey":"legalbench","canonicalDisplayName":"LegalBench","family_key":"vals-ai","benchmark_key":"legalbench","eval_summary_ids":["vals-ai%2Flegalbench"],"is_canonical_home":false},{"canonicalKey":"math-500","canonicalDisplayName":"MATH-500","family_key":"vals-ai","benchmark_key":"math-500","eval_summary_ids":["vals-ai%2Fmath-500"],"is_canonical_home":false},{"canonicalKey":"medqa","canonicalDisplayName":"MedQA","family_key":"vals-ai","benchmark_key":"medqa","eval_summary_ids":["vals-ai%2Fmedqa"],"is_canonical_home":false},{"canonicalKey":"mgsm","canonicalDisplayName":"MGSM","family_key":"vals-ai","benchmark_key":"mgsm","eval_summary_ids":["vals-ai%2Fmgsm"],"is_canonical_home":false},{"canonicalKey":"mmlu-pro","canonicalDisplayName":"MMLU-Pro","family_key":"vals-ai","benchmark_key":"mmlu-pro","eval_summary_ids":["vals-ai%2Fmmlu-pro"],"is_canonical_home":false},{"canonicalKey":"mmmu","canonicalDisplayName":"MMMU","family_key":"vals-ai","benchmark_key":"mmmu","eval_summary_ids":["vals-ai%2Fmmmu"],"is_canonical_home":false},{"canonicalKey":"swe-bench","canonicalDisplayName":"SWE-bench","family_key":"vals-ai","benchmark_key":"swe-bench","eval_summary_ids":["vals-ai%2Fswe-bench"],"is_canonical_home":false},{"canonicalKey":"terminal-bench","canonicalDisplayName":"Terminal Bench","family_key":"vals-ai","benchmark_key":"terminal-bench","eval_summary_ids":["vals-ai%2Fterminal-bench"],"is_canonical_home":false},{"canonicalKey":"terminalbench-hard","canonicalDisplayName":"TerminalBench Hard","family_key":"vals-ai","benchmark_key":"terminal-bench","eval_summary_ids":["vals-ai%2Fterminal-bench"],"is_canonical_home":false}]],["agentharm",[{"canonicalKey":"air-bench-2024-13-harassment","canonicalDisplayName":"Harassment","family_key":"agentharm","benchmark_key":"air-bench-2024-13-harassment","eval_summary_ids":["agentharm%2Fair-bench-2024-13-harassment"],"is_canonical_home":false},{"canonicalKey":"air-bench-2024-32-fraud","canonicalDisplayName":"Fraud","family_key":"agentharm","benchmark_key":"air-bench-2024-32-fraud","eval_summary_ids":["agentharm%2Fair-bench-2024-32-fraud"],"is_canonical_home":false},{"canonicalKey":"disinformation","canonicalDisplayName":"Disinformation","family_key":"agentharm","benchmark_key":"disinformation","eval_summary_ids":["agentharm%2Fdisinformation"],"is_canonical_home":false}]],["helm",[{"canonicalKey":"air-bench-2024-13-harassment","canonicalDisplayName":"Harassment","family_key":"helm","benchmark_key":"air-bench-2024","eval_summary_ids":["helm-air-bench%2Fair-bench-2024"],"is_canonical_home":false},{"canonicalKey":"air-bench-2024-32-fraud","canonicalDisplayName":"Fraud","family_key":"helm","benchmark_key":"air-bench-2024","eval_summary_ids":["helm-air-bench%2Fair-bench-2024"],"is_canonical_home":false},{"canonicalKey":"bbq","canonicalDisplayName":"BBQ","family_key":"helm","benchmark_key":"bbq","eval_summary_ids":["helm-safety%2Fbbq"],"is_canonical_home":false},{"canonicalKey":"boolq","canonicalDisplayName":"BoolQ","family_key":"helm","benchmark_key":"boolq","eval_summary_ids":["helm-classic%2Fboolq"],"is_canonical_home":false},{"canonicalKey":"cnn-dailymail","canonicalDisplayName":"CNN/DailyMail","family_key":"helm","benchmark_key":"cnn-dailymail","eval_summary_ids":["helm-classic%2Fcnn-dailymail"],"is_canonical_home":false},{"canonicalKey":"gpqa","canonicalDisplayName":"GPQA","family_key":"helm","benchmark_key":"gpqa","eval_summary_ids":["helm-capabilities%2Fgpqa"],"is_canonical_home":false},{"canonicalKey":"gsm8k","canonicalDisplayName":"GSM8K","family_key":"helm","benchmark_key":"gsm8k","eval_summary_ids":["helm-lite%2Fgsm8k"],"is_canonical_home":false},{"canonicalKey":"hellaswag","canonicalDisplayName":"HellaSwag","family_key":"helm","benchmark_key":"hellaswag","eval_summary_ids":["helm-classic%2Fhellaswag"],"is_canonical_home":false},{"canonicalKey":"ifeval","canonicalDisplayName":"IFEval","family_key":"helm","benchmark_key":"ifeval","eval_summary_ids":["helm-capabilities%2Fifeval"],"is_canonical_home":false},{"canonicalKey":"imdb","canonicalDisplayName":"IMDb","family_key":"helm","benchmark_key":"imdb","eval_summary_ids":["helm-classic%2Fimdb"],"is_canonical_home":false},{"canonicalKey":"legalbench","canonicalDisplayName":"LegalBench","family_key":"helm","benchmark_key":"legalbench","eval_summary_ids":["helm-lite%2Flegalbench"],"is_canonical_home":false},{"canonicalKey":"math","canonicalDisplayName":"MATH","family_key":"helm","benchmark_key":"math","eval_summary_ids":["helm-lite%2Fmath"],"is_canonical_home":false},{"canonicalKey":"medqa","canonicalDisplayName":"MedQA","family_key":"helm","benchmark_key":"medqa","eval_summary_ids":["helm-lite%2Fmedqa"],"is_canonical_home":false},{"canonicalKey":"mmlu","canonicalDisplayName":"MMLU","family_key":"helm","benchmark_key":"mmlu","eval_summary_ids":["helm-classic%2Fmmlu"],"is_canonical_home":false},{"canonicalKey":"mmlu","canonicalDisplayName":"MMLU","family_key":"helm","benchmark_key":"mmlu","eval_summary_ids":["helm-lite%2Fmmlu"],"is_canonical_home":false},{"canonicalKey":"mmlu","canonicalDisplayName":"MMLU","family_key":"helm","benchmark_key":"mmlu","eval_summary_ids":["helm-mmlu%2Fmmlu"],"is_canonical_home":false},{"canonicalKey":"mmlu-pro","canonicalDisplayName":"MMLU-Pro","family_key":"helm","benchmark_key":"mmlu-pro","eval_summary_ids":["helm-capabilities%2Fmmlu-pro"],"is_canonical_home":false},{"canonicalKey":"omni-math","canonicalDisplayName":"Omni-MATH","family_key":"helm","benchmark_key":"omni-math","eval_summary_ids":["helm-capabilities%2Fomni-math"],"is_canonical_home":false},{"canonicalKey":"openbookqa","canonicalDisplayName":"OpenbookQA","family_key":"helm","benchmark_key":"openbookqa","eval_summary_ids":["helm-classic%2Fopenbookqa"],"is_canonical_home":false},{"canonicalKey":"openbookqa","canonicalDisplayName":"OpenbookQA","family_key":"helm","benchmark_key":"openbookqa","eval_summary_ids":["helm-lite%2Fopenbookqa"],"is_canonical_home":false},{"canonicalKey":"truthfulqa","canonicalDisplayName":"TruthfulQA","family_key":"helm","benchmark_key":"truthfulqa","eval_summary_ids":["helm-classic%2Ftruthfulqa"],"is_canonical_home":false},{"canonicalKey":"xstest","canonicalDisplayName":"Xstest","family_key":"helm","benchmark_key":"xstest","eval_summary_ids":["helm-safety%2Fxstest"],"is_canonical_home":false},{"canonicalKey":"xsum","canonicalDisplayName":"XSum","family_key":"helm","benchmark_key":"xsum","eval_summary_ids":["helm-classic%2Fxsum"],"is_canonical_home":false}]],["apex-agents",[{"canonicalKey":"apex-agents","canonicalDisplayName":"APEX Agents","family_key":"apex-agents","benchmark_key":"apex-agents","eval_summary_ids":["apex-agents%2Fapex-agents"],"is_canonical_home":true},{"canonicalKey":"apex-v1","canonicalDisplayName":"APEX v1","family_key":"apex-agents","benchmark_key":"apex-v1","eval_summary_ids":["apex-agents%2Fapex-v1"],"is_canonical_home":false},{"canonicalKey":"investment banking","canonicalDisplayName":"investment banking","family_key":"apex-agents","benchmark_key":"apex-v1","eval_summary_ids":["apex-agents%2Fapex-v1"],"is_canonical_home":false}]],["apex-v1",[{"canonicalKey":"apex-v1","canonicalDisplayName":"APEX v1","family_key":"apex-v1","benchmark_key":"apex-v1","eval_summary_ids":["apex-v1%2Fapex-v1"],"is_canonical_home":true},{"canonicalKey":"investment banking","canonicalDisplayName":"investment banking","family_key":"apex-v1","benchmark_key":"apex-v1","eval_summary_ids":["apex-v1%2Fapex-v1"],"is_canonical_home":false}]],["hf-open-llm-v2",[{"canonicalKey":"bbh","canonicalDisplayName":"BBH","family_key":"hf-open-llm-v2","benchmark_key":"bbh","eval_summary_ids":["hfopenllm-v2%2Fbbh"],"is_canonical_home":false},{"canonicalKey":"gpqa","canonicalDisplayName":"GPQA","family_key":"hf-open-llm-v2","benchmark_key":"gpqa","eval_summary_ids":["hfopenllm-v2%2Fgpqa"],"is_canonical_home":false},{"canonicalKey":"ifeval","canonicalDisplayName":"IFEval","family_key":"hf-open-llm-v2","benchmark_key":"ifeval","eval_summary_ids":["hfopenllm-v2%2Fifeval"],"is_canonical_home":false},{"canonicalKey":"mmlu-pro","canonicalDisplayName":"MMLU-Pro","family_key":"hf-open-llm-v2","benchmark_key":"mmlu-pro","eval_summary_ids":["hfopenllm-v2%2Fmmlu-pro"],"is_canonical_home":false},{"canonicalKey":"musr","canonicalDisplayName":"MuSR","family_key":"hf-open-llm-v2","benchmark_key":"musr","eval_summary_ids":["hfopenllm-v2%2Fmusr"],"is_canonical_home":false}]],["openeval",[{"canonicalKey":"bbq","canonicalDisplayName":"BBQ","family_key":"openeval","benchmark_key":"bbq","eval_summary_ids":["openeval%2Fbbq"],"is_canonical_home":false},{"canonicalKey":"boolq","canonicalDisplayName":"BoolQ","family_key":"openeval","benchmark_key":"boolq","eval_summary_ids":["openeval%2Fboolq"],"is_canonical_home":false},{"canonicalKey":"cnn-dailymail","canonicalDisplayName":"CNN/DailyMail","family_key":"openeval","benchmark_key":"cnn-dailymail","eval_summary_ids":["openeval%2Fcnn-dailymail"],"is_canonical_home":false},{"canonicalKey":"disinformation","canonicalDisplayName":"Disinformation","family_key":"openeval","benchmark_key":"disinformation","eval_summary_ids":["openeval%2Fdisinformation"],"is_canonical_home":false},{"canonicalKey":"gpqa","canonicalDisplayName":"GPQA","family_key":"openeval","benchmark_key":"gpqa","eval_summary_ids":["openeval%2Fgpqa"],"is_canonical_home":false},{"canonicalKey":"ifeval","canonicalDisplayName":"IFEval","family_key":"openeval","benchmark_key":"ifeval","eval_summary_ids":["openeval%2Fifeval"],"is_canonical_home":false},{"canonicalKey":"imdb","canonicalDisplayName":"IMDb","family_key":"openeval","benchmark_key":"imdb","eval_summary_ids":["openeval%2Fimdb"],"is_canonical_home":false},{"canonicalKey":"mmlu-pro","canonicalDisplayName":"MMLU-Pro","family_key":"openeval","benchmark_key":"mmlu-pro","eval_summary_ids":["openeval%2Fmmlu-pro"],"is_canonical_home":false},{"canonicalKey":"omni-math","canonicalDisplayName":"Omni-MATH","family_key":"openeval","benchmark_key":"omni-math","eval_summary_ids":["openeval%2Fomni-math"],"is_canonical_home":false},{"canonicalKey":"truthfulqa","canonicalDisplayName":"TruthfulQA","family_key":"openeval","benchmark_key":"truthfulqa","eval_summary_ids":["openeval%2Ftruthfulqa"],"is_canonical_home":false},{"canonicalKey":"xsum","canonicalDisplayName":"XSum","family_key":"openeval","benchmark_key":"xsum","eval_summary_ids":["openeval%2Fxsum"],"is_canonical_home":false}]],["big-bench",[{"canonicalKey":"big-bench-hard","canonicalDisplayName":"BIG-Bench Hard","family_key":"big-bench","benchmark_key":"big-bench-hard","eval_summary_ids":["big-bench-hard%2Fbig-bench-hard"],"is_canonical_home":false}]],["cybench",[{"canonicalKey":"cybench","canonicalDisplayName":"Cybench","family_key":"cybench","benchmark_key":"cybench","eval_summary_ids":["cybench%2Fcybench"],"is_canonical_home":true}]],["facts-grounding",[{"canonicalKey":"facts-grounding","canonicalDisplayName":"Facts Grounding","family_key":"facts-grounding","benchmark_key":"facts-grounding","eval_summary_ids":["facts-grounding%2Ffacts-grounding"],"is_canonical_home":true}]],["global-mmlu",[{"canonicalKey":"global-mmlu-lite","canonicalDisplayName":"Global MMLU Lite","family_key":"global-mmlu","benchmark_key":"global-mmlu-lite","eval_summary_ids":["global-mmlu-lite%2Fglobal-mmlu-lite"],"is_canonical_home":false}]],["wasp",[{"canonicalKey":"gpqa-diamond","canonicalDisplayName":"GPQA Diamond","family_key":"wasp","benchmark_key":"gpqa-diamond","eval_summary_ids":["wasp%2Fgpqa-diamond"],"is_canonical_home":false},{"canonicalKey":"math","canonicalDisplayName":"MATH","family_key":"wasp","benchmark_key":"math","eval_summary_ids":["wasp%2Fmath"],"is_canonical_home":false},{"canonicalKey":"rewardbench-reasoning","canonicalDisplayName":"RewardBench Reasoning","family_key":"wasp","benchmark_key":"rewardbench-reasoning","eval_summary_ids":["wasp%2Frewardbench-reasoning"],"is_canonical_home":false}]],["gpqa",[{"canonicalKey":"gpqa-diamond","canonicalDisplayName":"GPQA Diamond","family_key":"gpqa","benchmark_key":"gpqa-diamond","eval_summary_ids":["gpqa-diamond%2Fgpqa-diamond"],"is_canonical_home":false}]],["gsm8k",[{"canonicalKey":"gsm8k","canonicalDisplayName":"GSM8K","family_key":"gsm8k","benchmark_key":"gsm8k","eval_summary_ids":["gsm8k%2Fgsm8k"],"is_canonical_home":true}]],["hellaswag",[{"canonicalKey":"hellaswag","canonicalDisplayName":"HellaSwag","family_key":"hellaswag","benchmark_key":"hellaswag","eval_summary_ids":["hellaswag%2Fhellaswag"],"is_canonical_home":true}]],["openai-humaneval",[{"canonicalKey":"humaneval","canonicalDisplayName":"HumanEval","family_key":"openai-humaneval","benchmark_key":"humaneval","eval_summary_ids":["openai-humaneval%2Fhumaneval"],"is_canonical_home":false}]],["ifeval",[{"canonicalKey":"ifeval","canonicalDisplayName":"IFEval","family_key":"ifeval","benchmark_key":"ifeval","eval_summary_ids":["ifeval%2Fifeval"],"is_canonical_home":true}]],["live-bench",[{"canonicalKey":"livebench","canonicalDisplayName":"LiveBench","family_key":"live-bench","benchmark_key":"livebench","eval_summary_ids":["live-bench%2Flivebench"],"is_canonical_home":false}]],["livecodebenchpro",[{"canonicalKey":"livecodebench-pro","canonicalDisplayName":"LiveCodeBench Pro","family_key":"livecodebenchpro","benchmark_key":"livecodebench-pro","eval_summary_ids":["livecodebenchpro%2Flivecodebench-pro"],"is_canonical_home":false}]],["reward-bench",[{"canonicalKey":"math","canonicalDisplayName":"MATH","family_key":"reward-bench","benchmark_key":"math","eval_summary_ids":["reward-bench%2Fmath"],"is_canonical_home":false},{"canonicalKey":"math","canonicalDisplayName":"MATH","family_key":"reward-bench","benchmark_key":"math","eval_summary_ids":["reward-bench-2%2Fmath"],"is_canonical_home":false},{"canonicalKey":"rewardbench-reasoning","canonicalDisplayName":"RewardBench Reasoning","family_key":"reward-bench","benchmark_key":"rewardbench","eval_summary_ids":["reward-bench%2Frewardbench"],"is_canonical_home":false}]],["math",[{"canonicalKey":"math-500","canonicalDisplayName":"MATH-500","family_key":"math","benchmark_key":"math-500","eval_summary_ids":["artificial-analysis-llms%2Fmath-500"],"is_canonical_home":false}]],["evalplus",[{"canonicalKey":"mbpp","canonicalDisplayName":"MBPP","family_key":"evalplus","benchmark_key":"mbpp-plus","eval_summary_ids":["mbpp%2Fmbpp-plus"],"is_canonical_home":false},{"canonicalKey":"mbpp-plus","canonicalDisplayName":"MBPP+","family_key":"evalplus","benchmark_key":"mbpp-plus","eval_summary_ids":["mbpp%2Fmbpp-plus"],"is_canonical_home":false}]],["mmlu-pro-leaderboard",[{"canonicalKey":"mmlu-pro","canonicalDisplayName":"MMLU-Pro","family_key":"mmlu-pro-leaderboard","benchmark_key":"mmlu-pro","eval_summary_ids":["mmlu-pro-leaderboard%2Fmmlu-pro"],"is_canonical_home":false}]],["piqa",[{"canonicalKey":"piqa","canonicalDisplayName":"Piqa","family_key":"piqa","benchmark_key":"piqa","eval_summary_ids":["piqa%2Fpiqa"],"is_canonical_home":true}]],["hal",[{"canonicalKey":"scicode","canonicalDisplayName":"SciCode","family_key":"hal","benchmark_key":"scicode","eval_summary_ids":["hal%2Fscicode"],"is_canonical_home":false},{"canonicalKey":"tau-bench-airline","canonicalDisplayName":"Tau Bench Airline","family_key":"hal","benchmark_key":"tau-bench-airline","eval_summary_ids":["hal%2Ftau-bench-airline"],"is_canonical_home":false}]],["exgentic-open-agent",[{"canonicalKey":"swe-bench","canonicalDisplayName":"SWE-bench","family_key":"exgentic-open-agent","benchmark_key":"swe-bench","eval_summary_ids":["exgentic-open-agent%2Fswe-bench"],"is_canonical_home":false},{"canonicalKey":"tau2-airline","canonicalDisplayName":"Tau2-Bench Airline","family_key":"exgentic-open-agent","benchmark_key":"tau2-airline","eval_summary_ids":["exgentic-open-agent%2Ftau2-airline"],"is_canonical_home":false},{"canonicalKey":"tau2-retail","canonicalDisplayName":"Tau2-Bench Retail","family_key":"exgentic-open-agent","benchmark_key":"tau2-retail","eval_summary_ids":["exgentic-open-agent%2Ftau2-retail"],"is_canonical_home":false},{"canonicalKey":"tau2-telecom","canonicalDisplayName":"Tau2-Bench Telecom","family_key":"exgentic-open-agent","benchmark_key":"tau2-telecom","eval_summary_ids":["exgentic-open-agent%2Ftau2-telecom"],"is_canonical_home":false}]]]};
-
-const meta = document.getElementById("meta");
-meta.textContent = [
-  DATA.schemaVersion ? "schema=" + DATA.schemaVersion : null,
-  DATA.generatedAt ? "generated=" + new Date(DATA.generatedAt).toISOString().slice(0, 19) : null,
-  "cleaner: " + DATA.cleanerStatus,
-].filter(Boolean).join(" · ");
-
-const statsEl = document.getElementById("stats");
-const statRows = [
-  ["Families", DATA.stats.families],
-  ["Benchmarks", DATA.stats.benchmarks],
-  ["Cross-suite overlaps", DATA.stats.overlaps],
-  ["Composites", DATA.stats.composite_count],
-  ["Splits", DATA.stats.slice_count],
-  ["Metrics", DATA.stats.metric_count],
-].filter(([, v]) => v != null);
-statsEl.innerHTML = statRows.map(([k, v]) => "<span><b>" + v + "</b>" + k + "</span>").join("");
-
-// Categories on a parent (family / composite) are the simple union of its
-// children's curated tags. cleanHierarchy already does this union via
-// decorateHierarchyDerivedTags, so reading derivedTags directly gives us
-// the full set. Fall back to the legacy category field only when the
-// node has no derivedTags at all.
-function nodeCategories(node) {
-  if (Array.isArray(node?.derivedTags) && node.derivedTags.length > 0) {
-    return node.derivedTags.slice();
-  }
-  return node?.category ? [node.category] : [];
-}
-
-// Populate category filter with the union across every family.
-const categories = Array.from(
-  new Set(DATA.families.flatMap(nodeCategories))
-).filter(Boolean).sort();
-const catSelect = document.getElementById("categoryFilter");
-for (const cat of categories) {
-  const opt = document.createElement("option");
-  opt.value = cat;
-  opt.textContent = cat;
-  catSelect.appendChild(opt);
-}
-
-// Acronyms preserved verbatim in upper-case when prettifying slugs.
-const ACRONYMS = new Set([
-  "AI","API","ARC","BBH","BFCL","CTF","CVE","CYSE","ELO","GAIA","GPQA","GSM",
-  "HELM","HF","HLE","IFEVAL","IMO","JSON","LLM","ML","MMLU","MMMU","MT",
-  "OPENAI","RLHF","SWE","TAU","USACO","VQA","WASP","CSV","API","SQL",
-  "AIME","GSM8K","BOOLQ","HRM","MCP","MQA","XML","CV","PDF",
-]);
-
-// Title-case a single word/segment, preserving known acronyms in caps.
-function prettifySegment(seg) {
-  if (!seg) return seg;
-  const upper = seg.toUpperCase();
-  if (ACRONYMS.has(upper)) return upper;
-  if (/^v\d+(?:\.\d+)?$/i.test(seg)) return seg.toLowerCase();
-  if (/^\d/.test(seg)) return seg;
-  // Already mixed-case (probably a CamelCase or Acronym + suffix) — keep
-  // verbatim. e.g. "APEX" stays "APEX", "iOS" stays "iOS".
-  if (seg !== seg.toLowerCase()) return seg;
-  return seg.charAt(0).toUpperCase() + seg.slice(1);
-}
-
-function titleCase(s) {
-  if (!s) return "";
-  // Split on whitespace, underscores, hyphens, AND slashes so path-like
-  // slice names ("livebench/coding/coding_completion") flatten to
-  // "Livebench Coding Coding Completion". Em-dashes / en-dashes are
-  // preserved as visible separators (warehouse already uses " — "
-  // between benchmark and slice in some display names).
-  return String(s)
-    .trim()
-    .split(/[\s_/-]+/)
-    .filter(Boolean)
-    .map((tok) => (tok === "—" || tok === "–" ? "—" : prettifySegment(tok)))
-    .join(" ");
-}
-
-function prettifyKey(key) { return titleCase(key); }
-
-// Pick the best human name for a node. Title-cases everything to a
-// uniform "Words With Spaces" form. Falls back to the prettified key
-// when the upstream display_name leaked from another row (e.g. gsm-mc
-// once arrived labelled with WASP's name) — detected by zero token
-// overlap with the key. Trailing contributor handles in parens are
-// stripped.
-function cleanDisplayName(name, key) {
-  const hasTokenOverlap = (a, b) => {
-    if (!a || !b) return false;
-    const tokens = (s) => new Set(String(s).toLowerCase().match(/[a-z0-9]+/g) || []);
-    const at = tokens(a), bt = tokens(b);
-    for (const t of at) if (t.length >= 2 && bt.has(t)) return true;
-    return false;
-  };
-  let candidate = name && String(name).trim();
-  if (!candidate || (key && !hasTokenOverlap(candidate, key))) {
-    candidate = key || "";
-  }
-  candidate = String(candidate || "")
-    .replace(/\s*\(([A-Za-z][A-Za-z0-9_-]*)\)\s*$/u, (_, tok) => {
-      if (/\s/.test(tok)) return " (" + tok + ")";
-      if (/[0-9]/.test(tok)) return " (" + tok + ")";
-      if (tok === tok.toUpperCase() && tok.length <= 6) return " (" + tok + ")";
-      return "";
-    })
-    .trim();
-  return titleCase(candidate);
-}
-
-function isMeaningfulTag(t) {
-  if (!t) return false;
-  const norm = String(t).trim().toLowerCase();
-  return norm.length > 0 && norm !== "not specified" && norm !== "n/a" && norm !== "unknown";
-}
-
-function tagBlock(tags) {
-  if (!tags) return "";
-  const out = [];
-  for (const k of ["domains", "languages", "tasks"]) {
-    const xs = (tags[k] || []).filter(isMeaningfulTag);
-    for (const x of xs) out.push('<span class="tag-pill">' + escapeHtml(x) + "</span>");
-  }
-  if (Array.isArray(tags.derivedTags)) {
-    for (const x of tags.derivedTags.filter(isMeaningfulTag)) {
-      out.push('<span class="tag-pill" style="background:var(--accent);color:white">' + escapeHtml(x) + "</span>");
-    }
-  }
-  return out.join("");
-}
-
-function escapeHtml(s) {
-  return String(s).replace(/[&<>"']/g, c => ({"&":"&amp;","<":"&lt;",">":"&gt;",'"':"&quot;","'":"&#39;"}[c]));
-}
-
-function renderBench(b, level, parentFamily) {
-  // Schema: a benchmark carries slices[], each slice has key,
-  // display_name, is_bare_stem, metrics[]. Metrics use key and
-  // display_name. The "root" slice is the one with is_bare_stem true
-  // (its metrics live at benchmark scope rather than under a sub-slice).
-  const slices = b.slices || [];
-  const sliceName = (s) => s.display_name ?? s.slice_name ?? s.key ?? s.slice_key;
-  const sliceKey = (s) => s.key ?? s.slice_key;
-  const metricName = (m) => m.display_name ?? m.metric_name ?? m.key ?? m.metric_key;
-  const metricKey = (m) => m.key ?? m.metric_key;
-  const metricLowerBetter = (m) => m.lower_is_better === true;
-  const isMetric = (m) => m && (metricName(m) || metricKey(m))
-  // Normalise slice/family/benchmark keys to a canonical form so a slice
-  // labelled "vals ai aime" matches the synthesised "vals-ai aime"
-  // family+benchmark concatenation.
-  const norm = (s) => String(s || "").toLowerCase().replace(/[_\s/-]+/g, " ").trim();
-  const isRolledUpSlice = (s) => {
-    if (s.is_bare_stem === true) return true;
-    if (s.slice_key == null && s.is_bare_stem !== false) return true;
-    const k = norm(sliceKey(s));
-    if (!k) return false;
-    if (k === norm(b.key)) return true;
-    if (parentFamily && k === norm(parentFamily.key + " " + b.key)) return true;
-    if (parentFamily && k === norm(parentFamily.key)) return true;
-    return false;
-  };
-  const root = slices.find(isRolledUpSlice) ?? null;
-  const sliceItems = slices.filter(s => s !== root);
-  const metricsRaw = (root?.metrics ?? b.metrics ?? []);
-  const metrics = metricsRaw.filter(isMetric);
-  const primaryMetricKey = b.primary_metric_key;
-  const evalIds = b.summary_eval_ids || [];
-  const benchCats = nodeCategories(b);
-  const benchName = cleanDisplayName(b.display_name, b.key);
-  // level is one of: "benchmark" (under composite), "standalone", "single"
-  // (sole benchmark in a family with no composite), "slice-promoted".
-  const levelLabel = level || "benchmark";
-
-  return `
-  <details class="bench">
-    <summary class="bench-head">
-      <span class="star">${b.is_overall ? "★" : "·"}</span>
-      <span class="name">${escapeHtml(benchName)}</span>
-      <span class="key">${escapeHtml(b.key)}</span>
-      <span class="badges">
-        <span class="badge level">${escapeHtml(levelLabel)}</span>
-        ${benchCats.map(c => `<span class="badge cat">${escapeHtml(c)}</span>`).join("")}
-        ${b.is_primary ? '<span class="badge cat">primary</span>' : ""}
-        <span class="badge size">${slices.length} slice${slices.length === 1 ? "" : "s"}</span>
-        <span class="badge size">${metrics.length} metric${metrics.length === 1 ? "" : "s"}</span>
-      </span>
-    </summary>
-    <div class="bench-detail">
-      <dl>
-        ${b.family_id ? `<dt>Family ID</dt><dd>${escapeHtml(b.family_id)}</dd>` : ""}
-        ${b.has_card != null ? `<dt>Has card</dt><dd>${b.has_card ? "yes" : "no"}</dd>` : ""}
-        ${primaryMetricKey ? `<dt>Primary metric</dt><dd>${escapeHtml(primaryMetricKey)}</dd>` : ""}
-        ${evalIds.length ? `<dt>Eval summary IDs</dt><dd style="font:10px ui-monospace,Menlo,monospace;color:var(--fg-muted);word-break:break-all;">${evalIds.map(escapeHtml).join("<br>")}</dd>` : ""}
-      </dl>
-      ${metrics.length ? `<div style="margin-top:6px"><b style="font-size:11px;text-transform:uppercase;letter-spacing:0.1em;color:var(--fg-subtle);font-weight:600">Metrics (root scope)</b><br>${metrics.map(m => `<span class="metric-pill ${metricKey(m) === primaryMetricKey ? "primary" : ""}" title="metric · ${escapeHtml(metricName(m) || metricKey(m) || "")}${metricLowerBetter(m) ? " · lower is better" : ""}"><span class="pill-level">metric</span>${escapeHtml(metricName(m) || metricKey(m) || "—")}</span>`).join("")}</div>` : ""}
-      ${sliceItems.length ? `<div style="margin-top:8px"><b style="font-size:11px;text-transform:uppercase;letter-spacing:0.1em;color:var(--fg-subtle);font-weight:600">Splits</b><ul class="slice-list">${sliceItems.map(s => `<li><span class="badge level slice-tag">split</span><span class="slice-name">${escapeHtml(cleanDisplayName(sliceName(s), sliceKey(s)) || "—")}</span><span class="slice-key">${escapeHtml(sliceKey(s) || "")}</span> ${(s.metrics||[]).filter(isMetric).map(m => `<span class="metric-pill" title="metric · ${escapeHtml(metricName(m) || metricKey(m) || "")}"><span class="pill-level">metric</span>${escapeHtml(metricName(m) || metricKey(m) || "—")}</span>`).join("")}</li>`).join("")}</ul></div>` : ""}
-    </div>
-  </details>`;
-}
-
-function categoryBadges(node) {
-  const cats = nodeCategories(node);
-  if (!cats.length) return "";
-  return cats.map(c => `<span class="badge cat">${escapeHtml(c)}</span>`).join("");
-}
-
-function renderComposite(c, parentFamily) {
-  return `
-  <details class="composite" open>
-    <summary class="composite-head">
-      <span class="badge level">composite</span>
-      <span class="name">${escapeHtml(cleanDisplayName(c.display_name, c.key))}</span>
-      <span class="key">${escapeHtml(c.key)}</span>
-      <span class="badges" style="margin-left:auto">
-        ${categoryBadges(c)}
-        <span class="badge size">${(c.benchmarks||[]).length} bench${(c.benchmarks||[]).length === 1 ? "" : "s"}</span>
-      </span>
-    </summary>
-    <div class="indent">
-      ${[...(c.benchmarks||[])].sort((a,b) => (a.display_name||a.key||"").toLowerCase().localeCompare((b.display_name||b.key||"").toLowerCase())).map(b => renderBench(b, "benchmark", parentFamily)).join("")}
-    </div>
-  </details>`;
-}
-
-function renderFamily(f) {
-  const byName = (a, b) => (a.display_name || a.key || "").toLowerCase().localeCompare((b.display_name || b.key || "").toLowerCase());
-  const compositesHtml = [...(f.composites||[])].sort(byName).map(c => renderComposite(c, f)).join("");
-  // Family-direct children. "standalone" = the family carries a single
-  // overall benchmark (family.standalone_benchmarks). "single" = the
-  // family's only direct child is a lone benchmark (no composite). "direct"
-  // = family carries multiple direct benchmarks (no composite wrapper).
-  const standalonesHtml = [...(f.standalone_benchmarks||[])].sort(byName).map(b => renderBench(b, "standalone", f)).join("");
-  const totalDirect = (f.benchmarks||[]).length + (f.composites||[]).length + (f.standalone_benchmarks||[]).length;
-  const directLevel = totalDirect === 1 ? "single benchmark" : "benchmark";
-  const directHtml = [...(f.benchmarks||[])].sort(byName).map(b => renderBench(b, directLevel, f)).join("");
-  const benchmarkCount = flattenLen(f);
-  const cats = nodeCategories(f);
-  return `
-  <details class="fam" data-key="${escapeHtml(f.key)}" data-categories="${escapeHtml(cats.join("|"))}" data-search="${escapeHtml(searchableText(f))}">
-    <summary class="fam-head">
-      <span class="badge level">family</span>
-      <span class="name">${escapeHtml(cleanDisplayName(f.display_name, f.key))}</span>
-      <span class="key">${escapeHtml(f.key)}</span>
-      <span class="badges">
-        ${categoryBadges(f)}
-        <span class="badge size">${benchmarkCount} bench${benchmarkCount === 1 ? "" : "s"}</span>
-        <span class="badge size">${(f.eval_summary_ids||[]).length} eval ids</span>
-      </span>
-    </summary>
-    <div class="indent">
-      ${compositesHtml}
-      ${standalonesHtml}
-      ${directHtml}
-    </div>
-  </details>`;
-}
-
-function flattenLen(f) {
-  return (f.benchmarks||[]).length
-    + (f.standalone_benchmarks||[]).length
-    + (f.composites||[]).reduce((s, c) => s + (c.benchmarks||[]).length, 0);
-}
-
-function searchableText(f) {
-  const parts = [f.key, f.display_name, f.category];
-  for (const c of f.composites || []) {
-    parts.push(c.key, c.display_name);
-    for (const b of c.benchmarks || []) parts.push(b.key, b.display_name);
-  }
-  for (const b of f.standalone_benchmarks || []) parts.push(b.key, b.display_name);
-  for (const b of f.benchmarks || []) parts.push(b.key, b.display_name);
-  return parts.filter(Boolean).join(" ").toLowerCase();
-}
-
-function renderHierarchy() {
-  const pane = document.getElementById("pane-hierarchy");
-  // Alphabetical sort by *cleaned* display name (case-insensitive) so
-  // the reader can audit one by one.
-  const sortedFamilies = [...DATA.families].sort((a, b) => {
-    const an = cleanDisplayName(a.display_name, a.key).toLowerCase();
-    const bn = cleanDisplayName(b.display_name, b.key).toLowerCase();
-    return an.localeCompare(bn);
-  });
-  pane.innerHTML = sortedFamilies.map(renderFamily).join("");
-}
-
-function renderOverlaps() {
-  const pane = document.getElementById("pane-overlaps");
-  if (DATA.benchmarkIndex.length === 0) {
-    pane.innerHTML = '<div class="empty">No cross-suite overlaps detected</div>';
-    return;
-  }
-  pane.innerHTML = DATA.benchmarkIndex.map(entry => {
-    const apps = (entry.appearances || []).map(a =>
-      `<span><b>${escapeHtml(a.family_key)}</b>${escapeHtml(a.benchmark_key || "")}</span>`
-    ).join("");
-    return `
-    <div class="overlap-row" data-search="${escapeHtml(((entry.key||"") + " " + (entry.display_name||"") + " " + (entry.appearances||[]).map(a => a.family_key + " " + a.benchmark_key).join(" ")).toLowerCase())}">
-      <div>
-        <div class="canonical-name">${escapeHtml(entry.display_name || entry.key)}</div>
-        <div class="canonical-key">${escapeHtml(entry.key)}</div>
-      </div>
-      <div class="n">${(entry.appearances||[]).length}</div>
-      <div class="apps">${apps}</div>
-    </div>`;
-  }).join("");
-}
-
-renderHierarchy();
-renderOverlaps();
-
-const searchEl = document.getElementById("search");
-const catEl = document.getElementById("categoryFilter");
-const countEl = document.getElementById("resultCount");
-
-function applyFilters() {
-  const q = searchEl.value.trim().toLowerCase();
-  const cat = catEl.value;
-  let visibleFam = 0;
-  for (const fam of document.querySelectorAll(".fam")) {
-    const matchesQ = !q || fam.dataset.search.includes(q);
-    const famCats = (fam.dataset.categories || "").split("|").filter(Boolean);
-    const matchesC = !cat || famCats.includes(cat);
-    const visible = matchesQ && matchesC;
-    fam.classList.toggle("hidden", !visible);
-    if (visible) visibleFam++;
-  }
-  let visibleOverlap = 0;
-  for (const row of document.querySelectorAll(".overlap-row")) {
-    const visible = !q || row.dataset.search.includes(q);
-    row.classList.toggle("hidden", !visible);
-    if (visible) visibleOverlap++;
-  }
-  countEl.textContent = `${visibleFam} families · ${visibleOverlap} overlaps shown`;
-}
-
-searchEl.addEventListener("input", applyFilters);
-catEl.addEventListener("change", applyFilters);
-applyFilters();
-
-document.getElementById("expandAll").addEventListener("click", () => {
-  for (const d of document.querySelectorAll("details")) d.open = true;
-});
-document.getElementById("collapseAll").addEventListener("click", () => {
-  for (const d of document.querySelectorAll("details")) d.open = false;
-});
-
-for (const tab of document.querySelectorAll(".tab-bar button")) {
-  tab.addEventListener("click", () => {
-    for (const t of document.querySelectorAll(".tab-bar button")) t.classList.toggle("on", t === tab);
-    for (const p of document.querySelectorAll(".pane")) p.classList.toggle("on", p.id === "pane-" + tab.dataset.tab);
-  });
-}
-</script>
-</body>
-</html>
