import gradio as gr import pandas as pd ARROW_RENAME = { "CER": "CER ↓", "WER": "WER ↓", "Equation TER": "Equation TER ↓", "BoW F1": "BoW F1 ↑", "Diacritic Micro F1": "Diacritic Micro F1 ↑", "Diacritic Macro F1": "Diacritic Macro F1 ↑", "TEDS": "TEDS ↑", "Figure Recall": "Figure Recall ↑", "Score": "Category Score ↑", } ARROW_RENAME_INV = {v: k for k, v in ARROW_RENAME.items()} # (type, full_display_name, url) MODEL_INFO = { "Claude Opus 4.7": ("api", "Claude Opus 4.7", "https://docs.anthropic.com/en/docs/about-claude/models/overview"), "Gemini 3.1 Pro": ("api", "Gemini 3.1 Pro", "https://ai.google.dev/gemini-api/docs/models/gemini-3.1-pro-preview"), "GPT-5.5": ("api", "GPT-5.5", "https://developers.openai.com/api/docs/models/gpt-5.5"), "GPT-5.4": ("api", "GPT-5.4", "https://developers.openai.com/api/docs/models/gpt-5.4"), "Mistral OCR 3": ("api", "Mistral OCR 3", "https://docs.mistral.ai/models/ocr-3-25-12"), "Kimi-K2.6": ("vlm", "Kimi-K2.6", "https://huggingface.co/moonshotai/Kimi-K2.6"), "Qwen3.5": ("vlm", "Qwen3.5-397B-A17B", "https://huggingface.co/Qwen/Qwen3.5-397B-A17B"), "Qwen3-VL": ("vlm", "Qwen3-VL-8B-Instruct", "https://huggingface.co/Qwen/Qwen3-VL-8B-Instruct"), "Gemma-3": ("vlm", "Gemma-3-12B-IT", "https://huggingface.co/google/gemma-3-12b-it"), "Phi-4-MM": ("vlm", "Phi-4-Multimodal-Instruct", "https://huggingface.co/microsoft/Phi-4-multimodal-instruct"), "Ministral-3": ("vlm", "Ministral-3-8B-Instruct", "https://huggingface.co/mistralai/Ministral-3-8B-Instruct-2512"), "InternVL3.5": ("vlm", "InternVL3.5-8B", "https://huggingface.co/OpenGVLab/InternVL3_5-8B"), "GaMS-3-beta": ("vlm", "GaMS3-12B-Multimodal", "https://huggingface.co/GaMS-Beta/GaMS3-12B-Multimodal"), "Nanonets-OCR2": ("ocr", "Nanonets-OCR2-3B", "https://huggingface.co/nanonets/Nanonets-OCR2-3B"), "GLM-OCR": ("ocr", "GLM-OCR", "https://huggingface.co/zai-org/GLM-OCR"), "olmOCR-2": ("ocr", "olmOCR-2-7B", "https://huggingface.co/allenai/olmOCR-2-7B-1025"), "Nemotron-Parse": ("ocr", "Nemotron-Parse v1.2", "https://huggingface.co/nvidia/NVIDIA-Nemotron-Parse-v1.2"), "Docling": ("pipeline", "Docling", "https://github.com/DS4SD/docling"), "Marker": ("pipeline", "Marker", "https://github.com/VikParuchuri/marker"), "MinerU": ("pipeline", "MinerU", "https://github.com/opendatalab/MinerU"), } TYPE_EMOJI = {"api": "🔴", "vlm": "🟢", "ocr": "🟡", "pipeline": "🔵"} TYPE_LABEL = { "api": "Proprietary API systems", "vlm": "Open-source VLMs", "ocr": "OCR-specialized models", "pipeline": "Traditional pipelines", } JS = """ () => { function createTip(text, el) { removeTip(); const tip = document.createElement('div'); tip.id = 'slo-tip'; tip.textContent = text; Object.assign(tip.style, { position: 'fixed', background: '#1e1e1e', color: '#fff', padding: '5px 10px', borderRadius: '4px', fontSize: '0.78em', lineHeight: '1.4', whiteSpace: 'nowrap', zIndex: '99999', boxShadow: '0 2px 6px rgba(0,0,0,0.3)', pointerEvents: 'none' }); document.body.appendChild(tip); const r = el.getBoundingClientRect(); tip.style.left = r.left + 'px'; tip.style.top = (r.top - tip.offsetHeight - 6) + 'px'; } function removeTip() { const t = document.getElementById('slo-tip'); if (t) t.remove(); } function attachBadges() { document.querySelectorAll('.model-badge').forEach(el => { if (el.dataset.tipAttached) return; el.dataset.tipAttached = '1'; const label = el.dataset.label; if (!label) return; el.addEventListener('mouseenter', () => createTip(label, el)); el.addEventListener('mouseleave', removeTip); }); } new MutationObserver(attachBadges).observe(document.body, { childList: true, subtree: true }); setInterval(attachBadges, 500); } """ def fmt_model(name): if name in MODEL_INFO: mtype, full_name, url = MODEL_INFO[name] emoji = TYPE_EMOJI[mtype] label = TYPE_LABEL[mtype] return ( f'{emoji} ' f'{full_name}' ) return name def load_overall(): df = pd.read_csv("data/overall_averages.csv") score_cols = [c for c in df.columns if c not in ("Rank", "Model")] for col in score_cols: df[col] = (df[col] * 100).round(1) df = df.rename(columns={ "academic": "Academic ↑", "textbook": "Textbook ↑", "handwriting": "Handwriting ↑", "billboard": "Billboard ↑", "historical": "Historical ↑", "web": "Web ↑", "Overall": "Overall ↑", }) df["Model"] = df["Model"].apply(fmt_model) return df[["Rank", "Model", "Academic ↑", "Historical ↑", "Handwriting ↑", "Billboard ↑", "Textbook ↑", "Web ↑", "Overall ↑"]] def load_category(name): df = pd.read_csv(f"data/{name}_averages.csv") score_cols = [c for c in df.columns if c not in ("Rank", "Model")] for col in score_cols: df[col] = (df[col] * 100).round(1) df = df.rename(columns={c: v for c, v in ARROW_RENAME.items() if c in df.columns}) df["Model"] = df["Model"].apply(fmt_model) non_score = [c for c in df.columns if c not in ("Rank", "Model", "Category Score ↑")] return df[["Rank", "Model"] + non_score + ["Category Score ↑"]] def dtypes_for(df, is_ci): if is_ci: return ["html" if c == "Model" else ("number" if c == "Rank" else "str") for c in df.columns] return ["html" if c == "Model" else ("number" if df[c].dtype != object else "str") for c in df.columns] def make_dataframe(df): return gr.Dataframe(value=df, datatype=dtypes_for(df, False), wrap=False, interactive=False, elem_classes=["leaderboard-table"], max_height=10000) def load_overall_ci(): plain = pd.read_csv("data/overall_averages.csv") cat_map = { "Academic ↑": "academic", "Historical ↑": "historical", "Handwriting ↑": "handwriting", "Billboard ↑": "billboard", "Textbook ↑": "textbook", "Web ↑": "web", } col_order = ["Academic ↑", "Historical ↑", "Handwriting ↑", "Billboard ↑", "Textbook ↑", "Web ↑", "Overall ↑"] overall_ci = pd.read_csv("data/overall_cis.csv").set_index("Model") result = plain[["Rank", "Model"]].copy() for col in col_order: if col in cat_map: ci_raw = pd.read_csv(f"data/{cat_map[col]}_cis.csv").set_index("Model") result[col] = [ f"[{ci_raw.loc[m, 'Score_lower'] * 100:.1f}, {ci_raw.loc[m, 'Score_upper'] * 100:.1f}]" if m in ci_raw.index else "" for m in plain["Model"] ] else: result[col] = [ f"[{overall_ci.loc[m, 'Overall_lower'] * 100:.1f}, {overall_ci.loc[m, 'Overall_upper'] * 100:.1f}]" if m in overall_ci.index else "" for m in plain["Model"] ] result["Model"] = result["Model"].apply(fmt_model) return result[["Rank", "Model"] + col_order] def load_category_ci(name): plain = pd.read_csv(f"data/{name}_averages.csv") ci_raw = pd.read_csv(f"data/{name}_cis.csv").set_index("Model") plain = plain.rename(columns={c: v for c, v in ARROW_RENAME.items() if c in plain.columns}) result = plain[["Rank", "Model"]].copy() non_score = [c for c in plain.columns if c not in ("Rank", "Model", "Category Score ↑")] col_order = non_score + ["Category Score ↑"] for col in col_order: base = ARROW_RENAME_INV.get(col, col) lower_col = f"{base}_lower" upper_col = f"{base}_upper" if lower_col in ci_raw.columns and upper_col in ci_raw.columns: result[col] = [ f"[{ci_raw.loc[m, lower_col] * 100:.1f}, {ci_raw.loc[m, upper_col] * 100:.1f}]" if m in ci_raw.index else "" for m in plain["Model"] ] else: result[col] = "" result["Model"] = result["Model"].apply(fmt_model) return result[["Rank", "Model"] + col_order] HEADER = """

slOCR: Slovenian-Language OCR Benchmark 🏔️

A comprehensive benchmark evaluating OCR models, Vision Language Models (VLMs), and traditional pipelines on 632 Slovene document pages with reference Markdown files.

1. Document categories:
  • Academic (207 pages): PDF pages from university theses, research papers, and technical reports; includes tables, equations, and figures
  • Handwriting (154 pages): scanned PDF pages of handwritten Slovene student essays from the Šolar corpus, spanning primary and secondary school levels
  • Historical (154 pages): archival scans of historical Slovene books and older printed materials, in both PDF and JPG format
  • Billboard (43 pages): PNG photographs of real-world Slovene posters, signs, and billboards with sparse, unstructured text
  • Web (42 pages): screenshots of Slovene web pages including news articles, forums, and blogs, in PNG format
  • Textbook (32 pages): PDF pages from Slovene school textbooks with dense text and educational content
2. Accuracy metrics:
  • Metrics are reported as percentages averaged across document pages, ↑ means higher is better and ↓ means lower is better
  • CER (Character Error Rate) / WER (Word Error Rate): edit distance between extracted and reference text at character or word level, clipped to [0, 1] per page
  • Diacritic Micro / Macro F1: F1 for Slovene diacritics (č, š, ž), micro-averaged across all occurrences or macro-averaged per diacritic type
  • TEDS (Tree Edit Distance Similarity): structural similarity between extracted and reference HTML table trees
  • Equation TER (Translation Edit Rate): token-level edit rate for LaTeX display equations
  • Figure Recall: fraction of reference figures present in the extracted document
  • BoW (Bag of Words) F1: F1 over unordered word multisets
  • Category Score: average of all relevant metrics per category, with error metrics inverted as (1 - metric). Diacritic Macro F1 is excluded
  • Overall: average of all 6 Category Scores

3. Evaluated models:   🔴 Proprietary API systems    🟢 Open-source VLMs    🟡 OCR-specialized models    🔵 Traditional pipelines

""" CSS = """ .header-cols { display: flex; flex-wrap: wrap; column-gap: 40px; row-gap: 16px; font-size: 0.97em; margin-bottom: 16px; } .header-cols > div { flex: 1; min-width: 300px; } .leaderboard-table table th:last-child, .leaderboard-table table td:last-child { background-color: #f5f5f5; font-weight: 600; border-left: 1px solid #d0d0d0; } @media (prefers-color-scheme: dark) { .leaderboard-table table th:last-child, .leaderboard-table table td:last-child { background-color: #2d2d2d; border-left-color: #555; } } footer { display: none !important; } .leaderboard-table td a { text-decoration: underline; color: inherit; } .model-badge { position: relative; display: inline-block; cursor: help; } """ CATEGORY_TABS = [ ("📚 Academic", "academic"), ("📜 Historical", "historical"), ("✍️ Handwriting", "handwriting"), ("🪧 Billboard", "billboard"), ("📖 Textbook", "textbook"), ("🌐 Web", "web"), ] TABS = [("🏆 Overall", load_overall(), load_overall_ci())] + [ (label, load_category(name), load_category_ci(name)) for label, name in CATEGORY_TABS ] FOOTER = """

Want your OCR model evaluated? Open a new discussion with a link to its HuggingFace or GitHub repository. 📬

Developed as part of the FRI Data Science Project Competition 2026, mentored by Valira AI.

""" with gr.Blocks(css=CSS, js=JS) as demo: gr.HTML(HEADER) ci_toggle = gr.Radio( choices=["Averages", "95% Confidence Intervals"], value="Averages", label="Display mode:", info="ℹ️ Confidence intervals were obtained via 10,000-sample bootstrap resampling of documents per category.", ) tab_comps = [] with gr.Tabs(): for label, plain_df, ci_df in TABS: with gr.TabItem(label): tab_comps.append((make_dataframe(plain_df), plain_df, ci_df)) def toggle_ci(mode): show_ci = mode == "95% Confidence Intervals" updates = [] for _, plain_df, ci_df in tab_comps: target = (ci_df if ci_df is not None else plain_df) if show_ci else plain_df is_ci = show_ci and ci_df is not None updates.append(gr.update(value=target, datatype=dtypes_for(target, is_ci), wrap=False)) return updates ci_toggle.change(toggle_ci, inputs=ci_toggle, outputs=[c for c, _, _ in tab_comps]) gr.HTML(FOOTER) demo.launch()