import marimo __generated_with = "0.10.9" app = marimo.App(width="medium") @app.cell def _(): import marimo as mo return (mo,) @app.cell def _(mo): mo.md( """ # DOAB Metadata Extraction: VLM vs Text **Can Vision-Language Models extract metadata from book covers better than text extraction?** This dashboard compares VLM (vision) and text-based approaches for extracting metadata from academic book covers in the [DOAB dataset](https://huggingface.co/datasets/biglam/doab-metadata-extraction). - **Title Extraction**: Extract just the book title (simpler task) - **Full Metadata**: Extract title, subtitle, publisher, year, ISBN (harder task) 📊 **Evaluation logs**: [davanstrien/doab-title-extraction-evals](https://huggingface.co/datasets/davanstrien/doab-title-extraction-evals) """ ) return @app.cell def _(): import pandas as pd import altair as alt from inspect_ai.analysis import evals_df return alt, evals_df, pd @app.cell def _(evals_df): # Load evaluation results from HuggingFace df_raw = evals_df("hf://datasets/davanstrien/doab-title-extraction-evals", quiet=True) # Add metadata columns df_raw["approach"] = df_raw["task_name"].apply(lambda x: "VLM" if "vlm" in x else "Text") df_raw["model_short"] = df_raw["model"].apply(lambda x: x.split("/")[-1]) # Determine task category def get_task_category(task_name): if "llm_judge" in task_name: return "Full Metadata" else: return "Title Extraction" df_raw["task_category"] = df_raw["task_name"].apply(get_task_category) # Convert score to percentage df_raw["accuracy"] = df_raw["score_headline_value"] * 100 # Parameter sizes (manual mapping) param_sizes = { "hf-inference-providers/Qwen/Qwen3-VL-8B-Instruct": 8, "hf-inference-providers/Qwen/Qwen3-VL-30B-A3B-Thinking": 30, "hf-inference-providers/zai-org/GLM-4.6V-Flash": 9, "hf-inference-providers/openai/gpt-oss-20b": 20, "hf-inference-providers/Qwen/Qwen3-4B-Instruct-2507": 4, "hf-inference-providers/allenai/Olmo-3-7B-Instruct": 7, } df_raw["param_size_b"] = df_raw["model"].map(param_sizes) df_raw return df_raw, get_task_category, param_sizes @app.cell def _(df_raw, mo): # Task selector task_selector = mo.ui.dropdown( options=["Title Extraction", "Full Metadata"], value="Title Extraction", label="Task", ) return (task_selector,) @app.cell def _(df_raw, mo, task_selector): # Filter by selected task df = df_raw[df_raw["task_category"] == task_selector.value].copy() # Calculate summary stats vlm_avg = df[df["approach"] == "VLM"]["accuracy"].mean() text_avg = df[df["approach"] == "Text"]["accuracy"].mean() diff = vlm_avg - text_avg task_desc = "book titles" if task_selector.value == "Title Extraction" else "full metadata (title, subtitle, publisher, year, ISBN)" mo.vstack([ task_selector, mo.md( f""" ## Key Results: {task_selector.value} | Approach | Average Accuracy | |----------|-----------------| | **VLM (Vision)** | **{vlm_avg:.0f}%** | | Text Extraction | {text_avg:.0f}% | **VLM advantage: +{diff:.0f} percentage points** VLMs {'significantly ' if diff > 15 else ''}outperform text extraction for extracting {task_desc} from book covers. """ ) ]) return df, diff, task_desc, text_avg, vlm_avg @app.cell def _(mo): mo.md("## Model Size vs Accuracy") return @app.cell def _(alt, df, mo): # Interactive scatter plot: model size vs accuracy scatter = alt.Chart(df).mark_circle(size=150).encode( x=alt.X("param_size_b:Q", title="Parameters (Billions)", scale=alt.Scale(zero=False)), y=alt.Y("accuracy:Q", title="Accuracy (%)", scale=alt.Scale(domain=[50, 105])), color=alt.Color("approach:N", title="Approach", scale=alt.Scale(domain=["VLM", "Text"], range=["#1f77b4", "#ff7f0e"])), tooltip=[ alt.Tooltip("model_short:N", title="Model"), alt.Tooltip("approach:N", title="Approach"), alt.Tooltip("param_size_b:Q", title="Params (B)"), alt.Tooltip("accuracy:Q", title="Accuracy", format=".1f"), ], ).properties( width=500, height=300, ).interactive() # Add text labels text = alt.Chart(df).mark_text( align="left", baseline="middle", dx=10, fontSize=11, ).encode( x="param_size_b:Q", y="accuracy:Q", text="model_short:N", color=alt.Color("approach:N", scale=alt.Scale(domain=["VLM", "Text"], range=["#1f77b4", "#ff7f0e"])), ) chart = (scatter + text).configure_axis( labelFontSize=12, titleFontSize=14, ) mo.ui.altair_chart(chart) return chart, scatter, text @app.cell def _(mo): mo.md("## Model Leaderboard") return @app.cell def _(df, mo): # Filter selector for approach approach_filter = mo.ui.dropdown( options=["All", "VLM", "Text"], value="All", label="Filter by approach", ) return (approach_filter,) @app.cell def _(approach_filter, df, mo): # Filter data based on selection if approach_filter.value == "All": filtered_df = df else: filtered_df = df[df["approach"] == approach_filter.value] # Create leaderboard leaderboard = ( filtered_df[["model_short", "approach", "param_size_b", "accuracy"]] .sort_values("accuracy", ascending=False) .reset_index(drop=True) ) leaderboard.columns = ["Model", "Approach", "Params (B)", "Accuracy (%)"] leaderboard["Accuracy (%)"] = leaderboard["Accuracy (%)"].round(1) mo.vstack([ approach_filter, mo.ui.table(leaderboard, selection=None), ]) return filtered_df, leaderboard @app.cell def _(mo): mo.md( """ ## About This Evaluation **Task**: Extract metadata from academic book cover images **Dataset**: [DOAB Metadata Extraction](https://huggingface.co/datasets/biglam/doab-metadata-extraction) - 50 samples **Evaluation Framework**: [Inspect AI](https://inspect.aisi.org.uk/) **Scoring**: - *Title Extraction*: Custom flexible matching (case-insensitive, handles subtitles) - *Full Metadata*: LLM-as-judge with partial credit ### Models Evaluated **VLM (Vision-Language Models)**: - Qwen3-VL-8B-Instruct (8B params) - Qwen3-VL-30B-A3B-Thinking (30B params) - GLM-4.6V-Flash (9B params) **Text Extraction** (OCR → LLM): - gpt-oss-20b (20B params) - Qwen3-4B-Instruct-2507 (4B params) - Olmo-3-7B-Instruct (7B params) - Qwen3-VL-8B-Instruct as text-only LLM (8B params) ### Why VLMs Win Book covers are **visually structured**: - Titles appear in specific locations (usually top/center) - Typography indicates importance (larger = more likely title) - Layout provides context that pure text loses Text extraction flattens this structure, losing valuable spatial information. --- *Built with [Marimo](https://marimo.io) | Evaluation framework: [Inspect AI](https://inspect.aisi.org.uk/)* """ ) return if __name__ == "__main__": app.run()