"""Ecosystem Lineage Explorer — FastAPI + SQLite (read-only) + vis-network. - Overview: analysis dashboard (relationship-type mix, most-copied, most-derived, most-translated, most-prolific producers) over the inferred lineage graph. - Explorer: search/pick a repo -> interactive lineage graph + related lists. - API: GET /related?dataset=&kind=, /api/graph, /analysis, /examples, /stats. Nodes are (id, kind) where kind is 'dataset' or 'model'; edges carry the kind of each endpoint. The schema is auto-detected: a dataset-only db (no kind columns) is served exactly as before with everything defaulting to kind='dataset'. DB is opened read-only; lineage.db is baked into the repo (small, robust). Edward Tufte-inspired styling. """ import json import os import sqlite3 from functools import lru_cache from pathlib import Path from fastapi import FastAPI, HTTPException, Query, Request from fastapi.middleware.gzip import GZipMiddleware from fastapi.responses import HTMLResponse, JSONResponse # DB selection order: # 1. LINEAGE_DB env var (dev: point at a sample/full db without editing anything) # 2. the bucket-mounted DB (/data/lineage.db, persistent + can grow) # 3. the baked-in copy (cold-mount fail, local dev) so the Space stays up either way. ENV_DB = os.environ.get("LINEAGE_DB") BUCKET_DB = Path("/data/lineage.db") LOCAL_DB = Path(__file__).parent / "lineage.db" if ENV_DB: DB = Path(ENV_DB) elif BUCKET_DB.exists(): DB = BUCKET_DB else: DB = LOCAL_DB print(f"using DB: {DB} (env={bool(ENV_DB)} bucket-mount={BUCKET_DB.exists()})", flush=True) # High safety ceiling only — stops a depth-2 hub exploding to tens of thousands # of nodes. The per-node degree cap (see graph()) is the primary anti-hairball. MAX_NODES = 2500 # Composite node key delimiter (id + kind). Unit-separator can't occur in a repo id. SEP = "\x1f" app = FastAPI(title="Ecosystem Lineage Explorer") app.add_middleware(GZipMiddleware, minimum_size=500) _con = sqlite3.connect(f"file:{DB}?mode=ro", uri=True, check_same_thread=False) _con.row_factory = sqlite3.Row def _cols(table: str) -> set[str]: return {r[1] for r in _con.execute(f"PRAGMA table_info({table})").fetchall()} # Schema auto-detection: the model extension adds a `kind` column to nodes and # `child_kind`/`parent_kind` to edges. When absent (dataset-only db) every query # falls back to the original form and synthesises kind='dataset'. _NODE_COLS = _cols("nodes") _EDGE_COLS = _cols("edges") _TABLES = {r[0] for r in _con.execute("SELECT name FROM sqlite_master WHERE type='table'").fetchall()} HAS_NODE_KIND = "kind" in _NODE_COLS HAS_EDGE_KIND = "child_kind" in _EDGE_COLS and "parent_kind" in _EDGE_COLS # Optional precomputed downstream-footprint table (dataset -> models trained on it -> # their derivative closure). When absent (or empty), the UI surfaces that use it hide. HAS_IMPACT = "dataset_impact" in _TABLES # Optional precomputed trending-enablement table (top trending models + their upward # provenance chain as JSON). Absent -> the Overview table hides. HAS_TRENDING = "trending_enablement" in _TABLES # Time-evolution data: nodes.created_at (ISO, nullable) drives the Explorer scrubber; # the precomputed `timeline` table drives the Overview "Over time" charts. Either # absent -> the corresponding surface hides. HAS_CREATED_AT = "created_at" in _NODE_COLS HAS_TIMELINE = "timeline" in _TABLES # Optional one-row build-provenance table (built_at, cutoff) stamped by build_db. # Absent on older dbs -> the "data as of" line hides. HAS_META = "meta" in _TABLES print(f"schema: node_kind={HAS_NODE_KIND} edge_kind={HAS_EDGE_KIND} impact={HAS_IMPACT} " f"trending={HAS_TRENDING} created_at={HAS_CREATED_AT} timeline={HAS_TIMELINE} " f"meta={HAS_META}", flush=True) # node identity is (id, kind); kind comes from user query params, so validate it — # an unbounded set of kinds would otherwise poison the bounded per-kind lru_caches. VALID_KINDS = ("dataset", "model") def _check_kind(kind: str) -> str: if kind not in VALID_KINDS: raise HTTPException(status_code=400, detail="kind must be 'dataset' or 'model'") return kind def _built_meta() -> dict: if not HAS_META: return {} return {r[0]: r[1] for r in _con.execute("SELECT key, value FROM meta").fetchall()} @lru_cache(maxsize=2048) def _impact(dataset: str) -> dict | None: """Downstream model footprint for one dataset, or None (no table / not found).""" if not HAS_IMPACT: return None r = _con.execute( "SELECT direct_models, descendant_models, descendant_downloads " "FROM dataset_impact WHERE dataset_id=?", (dataset,) ).fetchone() return dict(r) if r else None @app.middleware("http") async def cache_headers(request: Request, call_next): resp = await call_next(request) if request.url.path.startswith(("/api", "/related", "/examples", "/stats", "/analysis")): resp.headers["Cache-Control"] = "public, max-age=86400" return resp @lru_cache(maxsize=8192) def _meta(dataset: str, kind: str = "dataset") -> dict: if HAS_NODE_KIND: r = _con.execute("SELECT * FROM nodes WHERE dataset_id=? AND kind=?", (dataset, kind)).fetchone() else: r = _con.execute("SELECT * FROM nodes WHERE dataset_id=?", (dataset,)).fetchone() if r: d = dict(r) d.setdefault("kind", kind) return d return {"dataset_id": dataset, "kind": kind, "downloads": 0, "likes": 0, "author": ""} @lru_cache(maxsize=8192) def parents(dataset: str, kind: str = "dataset") -> list[dict]: if HAS_EDGE_KIND: rows = _con.execute( "SELECT parent AS id, parent_kind AS kind, primary_type, confidence, tags, size_ratio, source " "FROM edges WHERE child=? AND child_kind=? ORDER BY confidence DESC", (dataset, kind) ).fetchall() else: rows = _con.execute( "SELECT parent AS id, 'dataset' AS kind, primary_type, confidence, tags, size_ratio, source " "FROM edges WHERE child=? ORDER BY confidence DESC", (dataset,) ).fetchall() return [dict(r) for r in rows] @lru_cache(maxsize=8192) def children(dataset: str, kind: str = "dataset") -> list[dict]: if HAS_EDGE_KIND: rows = _con.execute( "SELECT child AS id, child_kind AS kind, primary_type, confidence, tags, size_ratio, source " "FROM edges WHERE parent=? AND parent_kind=? ORDER BY confidence DESC", (dataset, kind) ).fetchall() else: rows = _con.execute( "SELECT child AS id, 'dataset' AS kind, primary_type, confidence, tags, size_ratio, source " "FROM edges WHERE parent=? ORDER BY confidence DESC", (dataset,) ).fetchall() return [dict(r) for r in rows] def siblings(dataset: str, kind: str = "dataset") -> list[dict]: if HAS_EDGE_KIND: rows = _con.execute( "SELECT DISTINCT e2.child AS id, e2.child_kind AS kind, e2.parent AS via, e2.primary_type " "FROM edges e1 JOIN edges e2 ON e1.parent=e2.parent AND e1.parent_kind=e2.parent_kind " "WHERE e1.child=? AND e1.child_kind=? AND NOT (e2.child=? AND e2.child_kind=?) LIMIT 80", (dataset, kind, dataset, kind) ).fetchall() else: rows = _con.execute( "SELECT DISTINCT e2.child AS id, 'dataset' AS kind, e2.parent AS via, e2.primary_type " "FROM edges e1 JOIN edges e2 ON e1.parent=e2.parent " "WHERE e1.child=? AND e2.child!=? LIMIT 80", (dataset, dataset) ).fetchall() return [dict(r) for r in rows] def related(dataset: str, kind: str = "dataset") -> dict: out = {"dataset": dataset, "kind": kind, "meta": _meta(dataset, kind), "parents": parents(dataset, kind), "children": children(dataset, kind), "siblings": siblings(dataset, kind)} if kind == "dataset": # only datasets carry a downstream footprint imp = _impact(dataset) if imp: out["impact"] = imp return out @app.get("/related") def related_endpoint( dataset: str = Query(..., description="repo id, e.g. tatsu-lab/alpaca"), kind: str = Query("dataset", description="dataset | model"), ): """JSON of parents/children/siblings — for metadata workflows.""" return JSONResponse(related(dataset, _check_kind(kind))) @app.get("/api/graph") def graph(dataset: str, kind: str = "dataset", depth: int = 2, cap: int = 25): """Bounded lineage graph. `cap` = max edges rendered per node, per direction, PER counterpart-kind (top-N by confidence within each kind bucket). Each truncated bucket collapses into its own muted "+K more s" aggregate node, so a mixed family always shows BOTH kinds — thousands of models trained on a dataset can't crowd its dataset-derivatives off the canvas. Nodes are keyed by (id, kind); edges reference those keys.""" _check_kind(kind) cap = min(max(1, cap), 200) # clamp: a huge cap would render every edge of a hub node depth = min(max(1, depth), 4) def nk(i: str, k: str) -> str: return k + SEP + i focus = nk(dataset, kind) nodes: dict[str, tuple[str, str]] = {focus: (dataset, kind)} seen_edges: list[tuple] = [] # (direction, bucket_kind, attach_key) -> count of hidden tail edges for that kind aggregates: dict[tuple[str, str, str], int] = {} def expand(frontier, get_fn, direction): nxt = set() for d, dk in frontier: if len(nodes) >= MAX_NODES: break buckets: dict[str, list] = {} for r in get_fn(d, dk): # already ORDER BY confidence DESC buckets.setdefault(r["kind"], []).append(r) for bkind, rows in buckets.items(): for r in rows[:cap]: rk = nk(r["id"], r["kind"]) a, b = (rk, nk(d, dk)) if direction == "up" else (nk(d, dk), rk) seen_edges.append((a, b, r["primary_type"], r["confidence"], r["source"])) if rk not in nodes and len(nodes) < MAX_NODES: nodes[rk] = (r["id"], r["kind"]) nxt.add((r["id"], r["kind"])) if len(rows) > cap: aggregates[(direction, bkind, nk(d, dk))] = len(rows) - cap return nxt frontier = {(dataset, kind)} for _ in range(max(1, min(depth, 4))): frontier = expand(frontier, parents, "up") if len(nodes) >= MAX_NODES: break frontier = {(dataset, kind)} for _ in range(max(1, min(depth, 4))): frontier = expand(frontier, children, "down") if len(nodes) >= MAX_NODES: break edges_seen = {(a, b): (t, cf, s) for a, b, t, cf, s in seen_edges} node_list = [] for key, (i, kk) in nodes.items(): m = _meta(i, kk) node_list.append({"key": key, "id": i, "kind": kk, "label": i.split("/")[-1], "title": f"{i} ({m['downloads']:,} dl)", "downloads": m["downloads"], "created_at": m.get("created_at"), "focus": key == focus}) edge_list = [{"from": a, "to": b, "type": t, "confidence": cf, "source": s} for (a, b), (t, cf, s) in edges_seen.items() if a in nodes and b in nodes] # One synthetic aggregate node per truncated (direction, kind) bucket # (not clickable client-side); its label names the kind so both are legible. for (direction, bkind, akey), count in aggregates.items(): aid = "agg" + SEP + direction + SEP + bkind + SEP + akey node_list.append({"key": aid, "id": aid, "kind": "aggregate", "agg": True, "aggkind": bkind, "label": f"+{count:,} more {bkind}s", "count": count, "title": f"{count:,} more {bkind}s not shown (top {cap} by confidence rendered)", "downloads": 0, "focus": False}) if direction == "up": edge_list.append({"from": aid, "to": akey, "type": "aggregate", "confidence": 0, "source": "aggregate"}) else: edge_list.append({"from": akey, "to": aid, "type": "aggregate", "confidence": 0, "source": "aggregate"}) return {"nodes": node_list, "edges": edge_list} @lru_cache(maxsize=1) def _stats(): n = _con.execute("SELECT count(*) FROM nodes").fetchone()[0] e = _con.execute("SELECT count(*) FROM edges").fetchone()[0] inf = _con.execute("SELECT count(*) FROM edges WHERE source='inferred'").fetchone()[0] deriv = _con.execute("SELECT count(DISTINCT child) FROM edges WHERE source='inferred'").fetchone()[0] # Datasets that actually appear in the inferred lineage (as parent or child) — # the population the derivative-rate is measured against. NOT all dataset nodes: # the model layer drags in trained_on-only parents that were never analysed for # derivation, so measuring the rate against them would understate it. lineage_datasets = _con.execute( "SELECT count(*) FROM (SELECT parent d FROM edges WHERE source='inferred' " "UNION SELECT child FROM edges WHERE source='inferred')" ).fetchone()[0] if HAS_NODE_KIND: dataset_nodes = _con.execute("SELECT count(*) FROM nodes WHERE kind='dataset'").fetchone()[0] model_nodes = _con.execute("SELECT count(*) FROM nodes WHERE kind='model'").fetchone()[0] else: dataset_nodes, model_nodes = n, 0 declared_edges = _con.execute("SELECT count(*) FROM edges WHERE source='declared_tag'").fetchone()[0] bm = _built_meta() # built_at / cutoff when the db carries them (else absent) return {"nodes": n, "edges": e, "inferred": inf, "derivatives": deriv, "lineage_datasets": lineage_datasets, "dataset_nodes": dataset_nodes, "model_nodes": model_nodes, "declared_edges": declared_edges, "built_at": bm.get("built_at"), "cutoff": bm.get("cutoff")} @lru_cache(maxsize=8) def _examples(n: int): if HAS_EDGE_KIND: rows = _con.execute( "SELECT parent AS id, parent_kind AS kind, count(*) AS derivatives FROM edges " "WHERE source='inferred' GROUP BY parent, parent_kind ORDER BY derivatives DESC LIMIT ?", (n,) ).fetchall() else: rows = _con.execute( "SELECT parent AS id, 'dataset' AS kind, count(*) AS derivatives FROM edges " "WHERE source='inferred' GROUP BY parent ORDER BY derivatives DESC LIMIT ?", (n,) ).fetchall() return [dict(r) for r in rows] @lru_cache(maxsize=1) def _analysis(): def top(sql, *a): return [dict(r) for r in _con.execute(sql, a).fetchall()] res = { "type_dist": top("SELECT primary_type t, count(*) n FROM edges WHERE source='inferred' " "GROUP BY t ORDER BY n DESC"), "most_copied": top("SELECT parent id, count(*) n FROM edges WHERE primary_type='exact_copy' " "GROUP BY parent ORDER BY n DESC LIMIT 12"), "most_derived": top("SELECT parent id, count(*) n FROM edges WHERE source='inferred' " "GROUP BY parent ORDER BY n DESC LIMIT 12"), "most_translated": top("SELECT parent id, count(*) n FROM edges WHERE primary_type='translation' " "GROUP BY parent ORDER BY n DESC LIMIT 10"), "top_orgs": top("SELECT substr(child,1,instr(child,'/')-1) id, count(*) n FROM edges " "WHERE source='inferred' AND instr(child,'/')>0 GROUP BY id ORDER BY n DESC LIMIT 12"), } if HAS_IMPACT: # datasets whose trained-on model closure has the largest download footprint res["downstream_footprint"] = top( "SELECT dataset_id id, direct_models, descendant_models, descendant_downloads " "FROM dataset_impact ORDER BY descendant_downloads DESC LIMIT 12") if HAS_TRENDING: # top trending models + their upward provenance chain (ancestry JSON) rows = _con.execute( "SELECT model_id, trending_score, ancestry FROM trending_enablement " "ORDER BY trending_score DESC LIMIT 12").fetchall() out = [] for r in rows: try: anc = json.loads(r["ancestry"]) if r["ancestry"] else [] except (ValueError, TypeError): anc = [] out.append({"id": r["model_id"], "trending_score": r["trending_score"], "ancestry": anc}) res["trending_enablement"] = out if HAS_TIMELINE: # tidy long table (period, series, category, count) -> parallel arrays aligned # on a shared sorted period axis: {periods, nodes:{kind:[..]}, edges:{type:[..]}} try: rows = _con.execute("SELECT period, series, category, count FROM timeline").fetchall() periods = sorted({r["period"] for r in rows}) pidx = {p: i for i, p in enumerate(periods)} nodes_s: dict[str, list] = {} edges_s: dict[str, list] = {} for r in rows: tgt = nodes_s if r["series"] == "node" else edges_s tgt.setdefault(r["category"], [0] * len(periods))[pidx[r["period"]]] = r["count"] if periods: res["timeline"] = {"periods": periods, "nodes": nodes_s, "edges": edges_s} except sqlite3.Error: pass # unexpected timeline shape -> surface hides, everything else fine return res @lru_cache(maxsize=1) def _map(): """Overview map: multi-level chains (a derivative that is itself derived-from) + the top fan-out hubs. Nodes sized by derivative count. This view is the inferred dataset-lineage backbone, so its nodes are datasets.""" counts = dict(_con.execute( "SELECT parent, count(*) FROM edges WHERE source='inferred' GROUP BY parent" ).fetchall()) backbone = _con.execute( "SELECT parent, child, primary_type, confidence FROM edges WHERE source='inferred' " "AND child IN (SELECT DISTINCT parent FROM edges WHERE source='inferred')" ).fetchall() nodeset, edges = set(), [] for r in backbone: nodeset.add(r["parent"]) nodeset.add(r["child"]) edges.append({"from": r["parent"], "to": r["child"], "type": r["primary_type"], "confidence": r["confidence"]}) for pid, _n in sorted(counts.items(), key=lambda x: -x[1])[:60]: nodeset.add(pid) nodes = [{"id": n, "kind": "dataset", "label": n.split("/")[-1], "derivatives": counts.get(n, 0), "downloads": _meta(n)["downloads"], "created_at": _meta(n).get("created_at")} for n in nodeset] return {"nodes": nodes, "edges": edges} @app.get("/api/map") def api_map(): return _map() @lru_cache(maxsize=4096) def _search(q: str) -> list[dict]: if HAS_NODE_KIND: rows = _con.execute( "SELECT n.dataset_id AS id, n.kind AS kind, n.downloads, " "(SELECT count(*) FROM edges WHERE parent=n.dataset_id AND parent_kind=n.kind) AS derivatives " "FROM nodes n WHERE n.dataset_id LIKE ? " "ORDER BY n.downloads DESC LIMIT 12", (f"%{q}%",) ).fetchall() else: rows = _con.execute( "SELECT n.dataset_id AS id, 'dataset' AS kind, n.downloads, " "(SELECT count(*) FROM edges WHERE parent=n.dataset_id) AS derivatives " "FROM nodes n WHERE n.dataset_id LIKE ? " "ORDER BY n.downloads DESC LIMIT 12", (f"%{q}%",) ).fetchall() return [dict(r) for r in rows] @app.get("/api/search") def api_search(q: str = Query(..., min_length=2, max_length=120)): """Substring search over the repos in the lineage graph.""" return _search(q.strip()) @app.get("/stats") def stats(): return _stats() @app.get("/examples") def examples(n: int = 24): return _examples(n) @app.get("/analysis") def analysis(): """Aggregate analysis of the inferred lineage graph.""" return _analysis() @app.get("/", response_class=HTMLResponse) def index(): return (INDEX_HTML .replace("__STATS__", json.dumps(_stats())) .replace("__EXAMPLES__", json.dumps(_examples(24))) .replace("__ANALYSIS__", json.dumps(_analysis()))) INDEX_HTML = """ Ecosystem Lineage Explorer

Ecosystem Lineage Explorer

""" if __name__ == "__main__": import uvicorn uvicorn.run(app, host="0.0.0.0", port=7860)