Benchmark leaderboard by family + author region/institution

data/wam.db

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:afa5119bc335d2ad494cd871b265586d52be5b9803ebfd1e9c428e52b48812c7
-size 1961984
+oid sha256:18a23de4441d7a5fb39a67aa4cf714137d9d373f573c5e0119f9796660bfd241
+size 2027520

src/wam/pipeline/extract.py

@@ -22,43 +22,62 @@ from wam.store import papers as ps
 log = get_logger("pipeline.extract")
 
 SYSTEM = (
-    "Extract structured experimental results from this World Action Models paper. Rules:\n"
+    "Extract structured experimental results from this embodied-AI / World-Action-Model paper. "
+    "Rules:\n"
     "- Only report numbers ACTUALLY stated in the text — never estimate or invent.\n"
-    "- Each model/system is identified by (model_name, training_dataset). The SAME name "
-    "trained/finetuned on a different dataset is a DIFFERENT variant — always capture the "
-    "training/finetune dataset when stated.\n"
-    "- For each result give the benchmark, task, metric name+value, and any inference "
-    "speed/cost with their units and hardware as reported.\n"
-    "- Set claimed_by_authors=false for numbers quoted from OTHER papers (baselines/"
-    "comparisons), true for the paper's own results.\n"
-    "- BE CONCISE: report only this paper's headline results — at most ~15 benchmark rows "
-    "and ~6 model variants (the proposed method + key baselines). Keep every 'notes' field "
-    "under 12 words; omit notes if not needed.\n"
-    "- Empty lists are fine if the paper reports no quantitative results.")
+    "- Use the benchmark's STANDARD NAME and the variant when given, e.g. LIBERO, LIBERO-Long, "
+    "LIBERO-Goal, CALVIN (ABC->D), RoboTwin, SimplerEnv, RLBench, Meta-World, ManiSkill, "
+    "RoboCasa, Open-X, ALFWorld, VBench/VBench-Long. NEVER use a table caption ('Table 2'), a "
+    "section title, or a single task name ('Put Cube into Cup') as the benchmark — those tasks "
+    "go in the 'task' field, with the benchmark in 'benchmark'.\n"
+    "- Prefer the HEADLINE/aggregate number per (model, benchmark) (e.g. overall success rate "
+    "/ average), not every per-task row. Avoid generic names like 'Average' or 'Table N'.\n"
+    "- Each model is identified by (model_name, training_dataset). The SAME name trained/"
+    "finetuned on a different dataset is a DIFFERENT variant — capture the training dataset.\n"
+    "- Give metric name+value and any inference speed/cost with units + hardware as reported.\n"
+    "- claimed_by_authors=false for numbers quoted from OTHER papers (baselines), true for the "
+    "paper's own results.\n"
+    "- BE CONCISE: at most ~15 rows and ~8 model variants (proposed method + key baselines on "
+    "the main benchmarks). Notes under 12 words or omit. Empty lists are fine.")
+
+
+def _extract_one(cfg: Config, client: LLMClient, row) -> tuple[str, object | None]:
+    """Worker: fetch PDF text + extract. Returns (paper_id, ExtractionResult|None). No DB."""
+    links = json.loads(row["links_json"] or "{}")
+    text = pdf.get_text(cfg, row["id"], links.get("pdf") or "", max_chars=35000)
+    body = text or row["abstract"] or ""
+    user = (f"Title: {row['title']}\n\nAbstract: {row['abstract'] or ''}\n\n"
+            f"Paper text (may be truncated):\n{body}")
+    try:
+        res = client.complete_json("extract", SYSTEM, user, ExtractionResult,
+                                   label="extract", max_tokens=6000)
+        return row["id"], res
+    except Exception as e:  # noqa: BLE001
+        log.warning("extract failed for %s: %s", row["id"], e)
+        return row["id"], None
 
 
 def run(cfg: Config, client: LLMClient, conn: sqlite3.Connection,
         limit: int | None = None) -> int:
+    from concurrent.futures import ThreadPoolExecutor, as_completed
     cap = limit or int(cfg.get("constants.analyze_cap", 40))
     todo = ps.needs_extract(conn, limit=cap)
-    log.info("extracting benchmarks for %d core papers (cap=%d)", len(todo), cap)
+    workers = int(cfg.get("constants.extract_workers", 6))
+    log.info("extracting benchmarks for %d papers (%d workers)", len(todo), workers)
     done = 0
-    for row in todo:
-        links = json.loads(row["links_json"] or "{}")
-        # 35k chars (~9k tokens) comfortably covers intro+results+tables while staying fast.
-        text = pdf.get_text(cfg, row["id"], links.get("pdf") or "", max_chars=35000)
-        body = text or row["abstract"] or ""
-        user = (f"Title: {row['title']}\n\nAbstract: {row['abstract'] or ''}\n\n"
-                f"Paper text (may be truncated):\n{body}")
-        try:
-            res = client.complete_json("extract", SYSTEM, user, ExtractionResult,
-                                       label="extract", max_tokens=12000)
-        except Exception as e:  # noqa: BLE001
-            log.warning("extract failed for %s: %s", row["id"], e)
-            continue
-        n_models, n_rows = bm.store_extraction(conn, row["id"], res.models, res.benchmarks)
-        done += 1
-        log.debug("%s -> %d variants, %d benchmark rows", row["id"], n_models, n_rows)
-        conn.commit()
+    # LLM calls run concurrently (I/O-bound); each result is committed in the main thread AS
+    # IT COMPLETES (as_completed, not map) so a slow/stuck paper never blocks committing others.
+    with ThreadPoolExecutor(max_workers=workers) as ex:
+        futs = {ex.submit(_extract_one, cfg, client, r): r["id"] for r in todo}
+        for fut in as_completed(futs):
+            pid, res = fut.result()
+            if res is not None:
+                bm.store_extraction(conn, pid, res.models, res.benchmarks)
+            else:
+                conn.execute("UPDATE papers SET benchmarks_extracted=1 WHERE id=?", (pid,))
+            conn.commit()
+            done += 1
+            if done % 20 == 0:
+                log.info("extract progress: %d/%d", done, len(todo))
     log.info("extracted benchmarks from %d papers", done)
     return done

src/wam/pipeline/people.py

@@ -25,9 +25,13 @@ from wam.store.people import slug
 log = get_logger("pipeline.people")
 
 
-class Directions(BaseModel):
+class AuthorProfile(BaseModel):
     directions: str = Field(description="1-2 sentences on this author's main WAM-related "
                                         "research directions, grounded in the listed papers")
+    affiliation: str | None = Field(default=None, description="the author's institution as "
+                                    "stated in the papers' author/affiliation block, else null")
+    region: str | None = Field(default=None, description="country/region of that institution "
+                               "(e.g. 'China', 'USA', 'Singapore'), else null")
 
 
 def run(cfg: Config, client: LLMClient, conn: sqlite3.Connection,
@@ -72,21 +76,35 @@ def run(cfg: Config, client: LLMClient, conn: sqlite3.Connection,
     conn.execute("DELETE FROM authors")
     conn.execute("DELETE FROM groups")
 
+    from wam.sources import pdf
     kept = set(ranked)
     for aid in ranked:
         e = influential[aid]
         titles = "\n".join(f"- {meta[p]['title']}: {meta[p]['tldr']}" for p in e["pids"]
                            if p in meta)
+        # Author/affiliation block lives on the PDF first page — give the model that text
+        # (cached only, no download) so institution + region are grounded, not guessed.
+        blocks = []
+        for p in sorted(e["pids"])[:2]:
+            txt = pdf.get_text(cfg, p, "", max_chars=2500)
+            if txt:
+                blocks.append(f"[{meta.get(p,{}).get('title','')}]\n{txt}")
+        ctx = (f"Author: {e['name']}\nTracked papers:\n{titles}\n\n"
+               f"Author/affiliation blocks (first pages):\n" + "\n---\n".join(blocks))
+        aff, region, directions = e["affiliation"], None, None
         try:
-            directions = client.complete_json(
-                "cheap", "Summarize an author's research directions for a WAM digest. Write "
-                "in English.", f"Author: {e['name']}\nTracked papers:\n{titles}", Directions,
-                label="author-directions", max_tokens=2000).directions
+            prof = client.complete_json(
+                "cheap", "For the given author, summarize their WAM research directions and, "
+                "FROM the author/affiliation blocks only, extract their institution and "
+                "country/region. If a field isn't in the text, use null. Write in English.",
+                ctx, AuthorProfile, label="author-profile", max_tokens=2000)
+            directions = prof.directions
+            aff = prof.affiliation or e["affiliation"]
+            region = prof.region
         except Exception as ex:  # noqa: BLE001
-            log.warning("directions failed for %s: %s", e["name"], ex)
-            directions = None
+            log.warning("profile failed for %s: %s", e["name"], ex)
         store.upsert_author(
-            conn, author_id=aid, name=e["name"], affiliation=e["affiliation"],
+            conn, author_id=aid, name=e["name"], affiliation=aff, region=region,
             s2_url=f"https://www.semanticscholar.org/author/{aid}", citations=e["citations"],
             h_index=e["h_index"], paper_ids=sorted(e["pids"]), directions=directions)
     conn.commit()

src/wam/render/readme.py

@@ -79,20 +79,43 @@ def _core_table(conn: sqlite3.Connection, limit: int = 50) -> str:
     return "\n".join(out) + "\n"
 
 
-def _leaderboard(conn: sqlite3.Connection, limit: int = 40) -> str:
+def _leaderboard(conn: sqlite3.Connection, top_per: int = 10) -> str:
+    """Per-benchmark leaderboards grouped by canonical family (key embodied benchmarks first)."""
+    from wam.store.benchmarks import BENCH_FAMILIES, normalize_benchmark
+
     rows = conn.execute(
         "SELECT model_name, training_dataset, benchmark, task, metric_name, metric_value, "
-        "claimed_by_authors FROM benchmarks WHERE metric_value IS NOT NULL "
-        "ORDER BY benchmark, metric_value DESC LIMIT ?", (limit,)).fetchall()
+        "claimed_by_authors FROM benchmarks WHERE metric_value IS NOT NULL").fetchall()
     if not rows:
         return "_No benchmark results extracted yet._\n"
-    out = ["| Benchmark | Task | Model (training data) | Metric | Value | Source |",
-           "|-----------|------|-----------------------|--------|------:|:------:|"]
+    fam: dict[str, list] = {}
     for r in rows:
-        td = f" _({r['training_dataset']})_" if r["training_dataset"] else ""
-        src = "authors" if r["claimed_by_authors"] else "3rd-party"
-        out.append(f"| {r['benchmark']} | {r['task'] or '—'} | {r['model_name']}{td} | "
-                   f"{r['metric_name'] or '—'} | {r['metric_value']} | {src} |")
+        f = normalize_benchmark(r["benchmark"])
+        if f:
+            fam.setdefault(f, []).append(r)
+    key = [f for f in BENCH_FAMILIES if f in fam]
+    other = sorted((f for f in fam if f not in BENCH_FAMILIES), key=lambda f: -len(fam[f]))
+    out = ["_Model identity = (model, training data); same name on different data is a distinct "
+           "row. `authors` = self-reported, `3rd-party` = quoted. Higher is better for "
+           "success-rate-style metrics._\n"]
+    for f in key + other[:12]:
+        seen, items = set(), []
+        for r in sorted(fam[f], key=lambda x: x["metric_value"], reverse=True):
+            k = (r["model_name"], r["training_dataset"], r["metric_name"])
+            if k in seen:
+                continue
+            seen.add(k)
+            items.append(r)
+            if len(items) >= top_per:
+                break
+        out.append(f"\n#### {f}  ·  _{len(fam[f])} results_\n")
+        out.append("| Model (training data) | Task | Metric | Value | Source |")
+        out.append("|-----------------------|------|--------|------:|:------:|")
+        for r in items:
+            td = f" _({r['training_dataset']})_" if r["training_dataset"] else ""
+            src = "authors" if r["claimed_by_authors"] else "3rd-party"
+            out.append(f"| {r['model_name']}{td} | {r['task'] or '—'} | {r['metric_name'] or '—'} "
+                       f"| {r['metric_value']} | {src} |")
     return "\n".join(out) + "\n"
 
 

src/wam/store/benchmarks.py

@@ -15,6 +15,39 @@ from datetime import date, datetime
 from wam.pipeline.schemas import BenchmarkRow, ModelVariant
 
 
+# Canonical embodied-AI benchmark families (raw name -> family via alias substring match).
+BENCH_FAMILIES: dict[str, list[str]] = {
+    "LIBERO": ["libero"],
+    "CALVIN": ["calvin"],
+    "RoboTwin": ["robotwin", "robo twin"],
+    "SimplerEnv": ["simpler"],
+    "RLBench": ["rlbench"],
+    "Meta-World": ["meta-world", "metaworld", "meta world"],
+    "ManiSkill": ["maniskill", "mani-skill", "mani skill"],
+    "RoboCasa": ["robocasa"],
+    "Open-X / RT": ["open-x", "openx", "open x", "rt-1", "rt-2", "rt-x", "bridgedata", "bridge data"],
+    "ALFWorld": ["alfworld"],
+    "VBench": ["vbench"],
+    "AgiBot / GENIE": ["agibot", "genie"],
+    "Habitat": ["habitat"],
+    "BEHAVIOR": ["behavior-1k", "behavior1k", "behavior "],
+}
+# Junk that the extractor sometimes emits as a "benchmark".
+_BENCH_JUNK = re.compile(r"^(table\s*\d|fig\.?\s*\d|average|overall|search|real[\s-]?robot|"
+                         r"main results?|ablation|results?|n/?a|simulation|sim)\b", re.I)
+
+
+def normalize_benchmark(raw: str | None) -> str | None:
+    """Map a raw benchmark string to a canonical family; return None for junk/empty."""
+    if not raw or not raw.strip() or _BENCH_JUNK.match(raw.strip()):
+        return None
+    low = raw.lower()
+    for family, aliases in BENCH_FAMILIES.items():
+        if any(a in low for a in aliases):
+            return family
+    return raw.strip()
+
+
 def variant_key(model_name: str, training_dataset: str | None) -> str:
     base = f"{model_name}|{training_dataset or 'unknown'}".lower()
     return re.sub(r"[^a-z0-9]+", "-", base).strip("-")

src/wam/store/db.py

@@ -36,6 +36,9 @@ def _migrate(conn: sqlite3.Connection) -> None:
     cols = {r["name"] for r in conn.execute("PRAGMA table_info(papers)")}
     if "benchmarks_extracted" not in cols:
         conn.execute("ALTER TABLE papers ADD COLUMN benchmarks_extracted INTEGER DEFAULT 0")
+    acols = {r["name"] for r in conn.execute("PRAGMA table_info(authors)")}
+    if "region" not in acols:
+        conn.execute("ALTER TABLE authors ADD COLUMN region TEXT")
     conn.commit()
 
 

src/wam/store/papers.py

@@ -87,10 +87,11 @@ def needs_innovation(conn: sqlite3.Connection, limit: int | None = None) -> list
 
 
 def needs_extract(conn: sqlite3.Connection, limit: int | None = None) -> list[sqlite3.Row]:
-    # core papers that have been analyzed but not yet benchmark-extracted
+    # core + adjacent papers not yet benchmark-extracted (VLA/world-model papers report
+    # standard embodied benchmarks too). No analysis requirement — extraction reads the PDF.
     return conn.execute(_limit_clause(
-        "SELECT id, source, title, abstract, links_json FROM papers WHERE track='core' "
-        "AND analysis_json IS NOT NULL AND benchmarks_extracted=0", limit)).fetchall()
+        "SELECT id, source, title, abstract, links_json FROM papers WHERE "
+        "track IN ('core','adjacent') AND benchmarks_extracted=0", limit)).fetchall()
 
 
 # --- stage writers -----------------------------------------------------------

src/wam/store/people.py

@@ -14,21 +14,23 @@ def slug(name: str) -> str:
 
 def upsert_author(conn: sqlite3.Connection, *, author_id: str, name: str,
                   affiliation: str | None, s2_url: str | None, citations: int | None,
-                  h_index: int | None, paper_ids: list[str], directions: str | None) -> None:
+                  h_index: int | None, paper_ids: list[str], directions: str | None,
+                  region: str | None = None) -> None:
     conn.execute(
         """INSERT INTO authors
-           (id, name, affiliation, s2_url, citations, h_index, paper_ids_json, directions,
-            updated_at)
-           VALUES (?,?,?,?,?,?,?,?,?)
+           (id, name, affiliation, region, s2_url, citations, h_index, paper_ids_json,
+            directions, updated_at)
+           VALUES (?,?,?,?,?,?,?,?,?,?)
            ON CONFLICT(id) DO UPDATE SET
              affiliation=COALESCE(excluded.affiliation, affiliation),
+             region=COALESCE(excluded.region, region),
              s2_url=COALESCE(excluded.s2_url, s2_url),
              citations=COALESCE(excluded.citations, citations),
              h_index=COALESCE(excluded.h_index, h_index),
              paper_ids_json=excluded.paper_ids_json,
              directions=COALESCE(excluded.directions, directions),
              updated_at=excluded.updated_at""",
-        (author_id, name, affiliation, s2_url, citations, h_index, json.dumps(paper_ids),
+        (author_id, name, affiliation, region, s2_url, citations, h_index, json.dumps(paper_ids),
          directions, datetime.now().isoformat(timespec="seconds")),
     )
 

src/wam/store/schema.sql

@@ -75,6 +75,7 @@ CREATE TABLE IF NOT EXISTS authors (
     id              TEXT PRIMARY KEY,                -- semantic scholar id or slug
     name            TEXT NOT NULL,
     affiliation     TEXT,
+    region          TEXT,                            -- country/region of the affiliation
     s2_url          TEXT,
     citations       INTEGER,
     h_index         INTEGER,

src/wam/webapp/dashboard.py

@@ -95,13 +95,14 @@ def render(cfg: Config) -> None:
     # --- Authors ---
     st.subheader("👥 Influential authors")
     arows = con.execute(
-        "SELECT name, citations, h_index, paper_ids_json, directions, s2_url FROM authors "
-        "ORDER BY json_array_length(paper_ids_json) DESC").fetchall()
+        "SELECT name, region, affiliation, citations, h_index, paper_ids_json, directions, "
+        "s2_url FROM authors ORDER BY json_array_length(paper_ids_json) DESC").fetchall()
     if arows:
         adf = pd.DataFrame([{
-            "author": n, "papers": len(json.loads(pj or "[]")), "citations": cit,
-            "h_index": h, "directions": (d or "")[:200], "s2": url or ""}
-            for n, cit, h, pj, d, url in arows])
+            "author": n, "region": reg or "", "institution": aff or "",
+            "papers": len(json.loads(pj or "[]")), "citations": cit, "h_index": h,
+            "directions": (d or "")[:200], "s2": url or ""}
+            for n, reg, aff, cit, h, pj, d, url in arows])
         st.dataframe(adf, use_container_width=True, hide_index=True,
                      column_config={"s2": st.column_config.LinkColumn("s2")})
     else: