Community quant of baidu/Unlimited-OCR (uocr-quant)

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Baidu
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

QUANT_PROVENANCE.json

@@ -0,0 +1,16 @@
+{
+  "method": "GPTQ NVFP4, offloaded hessians, llmcompressor 0.11.0",
+  "calibration": "32 OCR-domain text sequences (see calibration/)",
+  "ignore": [
+    "lm_head",
+    "re:.*embed_tokens.*",
+    "re:.*sam_model.*",
+    "re:.*vision_model.*",
+    "re:.*projector.*",
+    "re:.*mlp\\.gate$",
+    "re:.*norm.*"
+  ],
+  "base_model": "baidu/Unlimited-OCR",
+  "base_model_revision": "07dea832e22aefee32ad281d4b80551282e1c168",
+  "dead_expert_repair": "3/768 experts unrouted during calibration; data-free params transplanted, input_global_scale from sibling means (repair_dead_experts.py, issue #6)"
+}

README.md

@@ -6,6 +6,32 @@ pipeline_tag: image-text-to-text
 inference: false
 quantized_by: shadowrock-io
 library_name: vllm
+metrics:
+  - cer
+model-index:
+  - name: Unlimited-OCR-Community-NVFP4
+    results:
+      - task:
+          type: image-to-text
+          name: Grounded document OCR (parity vs BF16)
+        dataset:
+          name: uocr-quant synthetic document fixtures (invoice, memo, table report)
+          type: shadowrock/uocr-quant-fixtures
+          config: default
+          split: test
+        metrics:
+          - type: cer_vs_bf16_mean
+            name: Mean CER vs BF16 (greedy, grounding prompt)
+            value: 0.0068
+          - type: cer_vs_bf16_max
+            name: Max per-fixture CER vs BF16
+            value: 0.0204
+          - type: decode_tok_per_s
+            name: Decode throughput (tok/s, vLLM 0.26.0, RTX 5070 Ti)
+            value: 40.84
+        source:
+          name: ShadowRock eval (raw JSON)
+          url: https://huggingface.co/shadowrock-io/Unlimited-OCR-Community-NVFP4/tree/main/evidence
 tags:
   - nvfp4
   - fp4
@@ -34,7 +60,8 @@ language:
 
 **Unofficial community quantization — not a Baidu release.**
 
-Calibrated NVFP4 build of [baidu/Unlimited-OCR](https://huggingface.co/baidu/Unlimited-OCR), the
+Calibrated NVFP4 build of [baidu/Unlimited-OCR](https://huggingface.co/baidu/Unlimited-OCR) (revision
+[`07dea832`](https://huggingface.co/baidu/Unlimited-OCR/commit/07dea832e22aefee32ad281d4b80551282e1c168)), the
 3.34B DeepSeek-V2-style MoE OCR model. All credit for the model belongs to Baidu; this repo
 changes only the numeric precision of the text-decoder weights. Checkpoint size drops from
 6.7 GB to ~2.8 GB.

UPSTREAM_README.md

@@ -0,0 +1,335 @@
+---
+pipeline_tag: image-text-to-text
+language:
+- multilingual
+tags:
+- baidu
+- vision-language
+- ocr
+- custom_code
+license: mit
+library_name: transformers
+---
+<p align="center">
+  <img src="assets/baidu.png" width="55%" alt="Baidu Inc." />
+</p>
+
+<hr>
+
+<h1 align="center">Unlimited OCR Works</h1>
+
+<div align="center">
+
+  <a href="https://trendshift.io/repositories/62053?utm_source=trendshift-badge&amp;utm_medium=badge&amp;utm_campaign=badge-trendshift-62053" target="_blank" rel="noopener noreferrer"><img src="https://trendshift.io/api/badge/trendshift/repositories/62053/daily" alt="baidu%2FUnlimited-OCR | Trendshift" width="250" height="55"/></a>
+  
+  <a href="https://github.com/baidu/Unlimited-OCR">
+    <img alt="GitHub" src="https://img.shields.io/badge/GitHub-Code-181717?logo=github&logoColor=white" />
+  </a>
+  <a href="https://huggingface.co/baidu/Unlimited-OCR">
+    <img alt="Hugging Face" src="https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-Model-ffc107?color=ffc107&logoColor=white" />
+  </a>
+</div>
+
+<div align="center">
+    <a href="https://arxiv.org/abs/2606.23050">
+    <img alt="arXiv" src="https://img.shields.io/badge/arXiv-Unlimited OCR Works-b31b1b?logo=arxiv&logoColor=white" />
+  </a>
+  <a href="https://x.com/Baidu_Inc" target="_blank">
+    <img alt="Twitter Follow" src="https://img.shields.io/badge/Twitter-Baidu Inc.-white?logo=x&logoColor=white" />
+  </a>
+</div>
+
+<h3 align="center">Welcome the Era of One-shot Long-horizon Parsing.</h3>
+
+<p align="center">
+    <img src="assets/Unlimited-OCR.png" width="1000" alt="Unlimited OCR overview" />
+</p>
+
+
+## Release
+- [2026/07/21] 🤝 Thanks to the [ms-swift community](https://github.com/modelscope/ms-swift) for their support, our model now supports training with [ms-swift](https://github.com/modelscope/ms-swift).
+- [2026/07/03] 🤝 Thanks to the Baidu Cloud team for their support. Our model is now available on [Baidu Cloud](https://cloud.baidu.com/doc/OCR/s/fmr1p39gb).
+- [2026/06/28] 🤝 Thanks to the [vLLM community](https://github.com/vllm-project/vllm) and [Tianyu Guo](https://github.com/gty111) for their support, our model now supports vLLM inference.
+- [2026/06/24] 🤝 Thanks to [AK](https://x.com/_akhaliq) for creating a demo for us. It is now available at [Hugging Face Spaces](https://huggingface.co/spaces/baidu/Unlimited-OCR).
+- [2026/06/23] 📄 Our paper is now available on [arXiv](https://arxiv.org/abs/2606.23050).
+- [2026/06/23] 🤝 Thanks to the [ModelScope community](https://github.com/modelscope) for their support. Our model is now available at [ModelScope](https://modelscope.cn/models/PaddlePaddle/Unlimited-OCR).
+- [2026/06/22] 🚀 We present [Unlimited-OCR](https://github.com/baidu/Unlimited-OCR), aiming to push [Deepseek-OCR](https://github.com/deepseek-ai/DeepSeek-OCR) one step further.
+
+## Inference
+
+### Transformers
+Inference using Huggingface transformers on NVIDIA GPUs. Requirements tested on python 3.12.3 + CUDA12.9：
+
+```
+torch==2.10.0
+torchvision==0.25.0
+transformers==4.57.1
+Pillow==12.1.1
+matplotlib==3.10.8
+einops==0.8.2
+addict==2.4.0
+easydict==1.13
+pymupdf==1.27.2.2
+psutil==7.2.2
+```
+
+```python
+import os
+import torch
+from transformers import AutoModel, AutoTokenizer
+
+model_name = 'baidu/Unlimited-OCR'
+
+tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
+model = AutoModel.from_pretrained(
+    model_name,
+    trust_remote_code=True,
+    use_safetensors=True,
+    torch_dtype=torch.bfloat16,
+)
+model = model.eval().cuda()
+
+# ── Single image supports two configs: gundam or base ──
+# gundam: base_size=1024, image_size=640, crop_mode=True
+# base: base_size=1024, image_size=1024, crop_mode=False
+model.infer(
+    tokenizer,
+    prompt='<image>document parsing.',
+    image_file='your_image.jpg',
+    output_path='your/output/dir',
+    base_size=1024, image_size=640, crop_mode=True,
+    max_length=32768,
+    no_repeat_ngram_size=35, ngram_window=128,
+    save_results=True,
+)
+
+# ── Multi page / PDF only uses base (image_size=1024) ──
+model.infer_multi(
+    tokenizer,
+    prompt='<image>Multi page parsing.',
+    image_files=['page1.png', 'page2.png', 'page3.png'],
+    output_path='your/output/dir',
+    image_size=1024,
+    max_length=32768,
+    no_repeat_ngram_size=35, ngram_window=1024,
+    save_results=True,
+)
+
+# ── PDF (convert pages to images, then multi-page parsing) ──
+import tempfile, fitz  # PyMuPDF
+
+def pdf_to_images(pdf_path, dpi=300):
+    doc = fitz.open(pdf_path)
+    tmp_dir = tempfile.mkdtemp(prefix='pdf_ocr_')
+    mat = fitz.Matrix(dpi / 72, dpi / 72)
+    paths = []
+    for i, page in enumerate(doc):
+        out = os.path.join(tmp_dir, f'page_{i+1:04d}.png')
+        page.get_pixmap(matrix=mat).save(out)
+        paths.append(out)
+    doc.close()
+    return paths
+
+model.infer_multi(
+    tokenizer,
+    prompt='<image>Multi page parsing.',
+    image_files=pdf_to_images('your_doc.pdf', dpi=300),
+    output_path='your/output/dir',
+    image_size=1024,
+    max_length=32768,
+    no_repeat_ngram_size=35, ngram_window=1024,
+    save_results=True,
+)
+```
+
+### vLLM
+
+Please refer to the official vLLM recipe for deployment details:
+
+**Recipe:** [https://recipes.vllm.ai/baidu/Unlimited-OCR](https://recipes.vllm.ai/baidu/Unlimited-OCR)
+
+##### Docker Images
+Use the following Docker images depending on your GPU platform:
+
+**Default (CUDA 13.0):**
+```bash
+docker pull vllm/vllm-openai:unlimited-ocr
+```
+**For Hopper GPUs (CUDA 12.9)**
+```bash
+docker pull vllm/vllm-openai:unlimited-ocr-cu129
+```
+
+### SGLang
+
+Set up the environment (uv-managed virtualenv). Install the local SGLang wheel first,
+then pin `kernels==0.9.0` and install PyMuPDF for PDF-to-image conversion:
+```shell
+uv venv --python 3.12
+source .venv/bin/activate
+
+uv pip install wheel/sglang-0.0.0.dev11416+g92e8bb79e-py3-none-any.whl
+uv pip install kernels==0.11.7
+uv pip install pymupdf==1.27.2.2
+```
+
+Start the SGLang server:
+```shell
+python -m sglang.launch_server \
+    --model baidu/Unlimited-OCR \
+    --served-model-name Unlimited-OCR \
+    --attention-backend fa3 \
+    --page-size 1 \
+    --mem-fraction-static 0.8 \
+    --context-length 32768 \
+    --enable-custom-logit-processor \
+    --disable-overlap-schedule \
+    --skip-server-warmup \
+    --host 0.0.0.0 \
+    --port 10000
+```
+
+Send streaming requests to the OpenAI-compatible API:
+```python
+import base64
+import json
+import os
+import tempfile
+
+import fitz
+import requests
+from sglang.srt.sampling.custom_logit_processor import DeepseekOCRNoRepeatNGramLogitProcessor
+
+server_url = "http://127.0.0.1:10000"
+
+session = requests.Session()
+session.trust_env = False
+
+
+def pdf_to_images(pdf_path, dpi=300):
+    doc = fitz.open(pdf_path)
+    tmp_dir = tempfile.mkdtemp(prefix="pdf_ocr_")
+    mat = fitz.Matrix(dpi / 72, dpi / 72)
+    image_paths = []
+    for i, page in enumerate(doc):
+        image_path = os.path.join(tmp_dir, f"page_{i + 1:04d}.png")
+        page.get_pixmap(matrix=mat).save(image_path)
+        image_paths.append(image_path)
+    doc.close()
+    return image_paths
+
+
+def encode_image(image_path):
+    ext = os.path.splitext(image_path)[1].lower()
+    mime = "image/jpeg" if ext in (".jpg", ".jpeg") else f"image/{ext.lstrip('.')}"
+    with open(image_path, "rb") as f:
+        data = base64.b64encode(f.read()).decode("utf-8")
+    return {"type": "image_url", "image_url": {"url": f"data:{mime};base64,{data}"}}
+
+
+def build_content(prompt, image_paths):
+    return [{"type": "text", "text": prompt}] + [encode_image(path) for path in image_paths]
+
+
+def generate(prompt, image_paths, image_mode, ngram_window):
+    payload = {
+        "model": "Unlimited-OCR",
+        "messages": [{"role": "user", "content": build_content(prompt, image_paths)}],
+        "temperature": 0,
+        "skip_special_tokens": False,
+        "images_config": {"image_mode": image_mode},
+        "custom_logit_processor": DeepseekOCRNoRepeatNGramLogitProcessor.to_str(),
+        "custom_params": {
+            "ngram_size": 35,
+            "window_size": ngram_window,
+        },
+        "stream": True,
+    }
+    response = session.post(
+        f"{server_url}/v1/chat/completions",
+        headers={"Content-Type": "application/json"},
+        data=json.dumps(payload),
+        timeout=1200,
+        stream=True,
+    )
+    response.raise_for_status()
+
+    chunks = []
+    for line in response.iter_lines(chunk_size=1, decode_unicode=True):
+        if not line or not line.startswith("data: "):
+            continue
+        data = line[len("data: "):]
+        if data == "[DONE]":
+            break
+        event = json.loads(data)
+        delta = event["choices"][0].get("delta", {}).get("content", "")
+        if delta:
+            print(delta, end="", flush=True)
+            chunks.append(delta)
+    print()
+    return "".join(chunks)
+
+
+# Single image supports two configs: gundam or base. Example below uses gundam.
+generate("document parsing.", ["your_image.jpg"], image_mode="gundam", ngram_window=128)
+
+# Multi image (base only)
+generate("Multi page parsing.", ["page1.png", "page2.png"], image_mode="base", ngram_window=1024)
+
+# PDF (base only)
+generate("Multi page parsing.", pdf_to_images("your_doc.pdf", dpi=300), image_mode="base", ngram_window=1024)
+```
+
+For OmniDocBench evaluation, you need to perform the following post-processing.
+```python
+DET_RE = re.compile(r'<\|det\|>([^<\s]+)(?:\s*\[[^\]]*\])?\s*<\|/det\|>(.*)', re.DOTALL)
+
+def remove_det(raw: str) -> str:
+    """
+    Strip <|det|>type [bbox]<|/det|> markers, group lines belonging to the
+    same block with \\n, and separate different blocks with \\n\\n.
+    """
+    blocks = []
+    cur = None
+    for line in raw.splitlines():
+        line = line.rstrip()
+        if not line:
+            continue
+        m = DET_RE.match(line)
+        if m:
+            category, content = m.group(1).strip(), m.group(2).strip()
+            if category == 'image':
+                continue
+            if cur is not None:
+                blocks.append(cur)
+            cur = [content] if content else []
+            continue
+        if cur is None:
+            cur = []
+        cur.append(line)
+    if cur is not None:
+        blocks.append(cur)
+    text = '\n\n'.join('\n'.join(b) for b in blocks).strip()
+    return text
+```
+
+
+## Visualization
+
+<img src="assets/long-horizon-ocr.gif" width="100%" alt="Long-horizon OCR demo" />
+
+## Acknowledgement
+
+We would like to thank [Deepseek-OCR](https://github.com/deepseek-ai/DeepSeek-OCR), [Deepseek-OCR-2](https://github.com/deepseek-ai/DeepSeek-OCR-2), [PaddleOCR](https://github.com/PaddlePaddle/PaddleOCR) for their valuable models and ideas.
+
+## Citation
+```bibtex
+@misc{yin2026unlimitedocrworks,
+      title={Unlimited OCR Works}, 
+      author={Youyang Yin and Huanhuan Liu and YY and Qunyi Xie and Chaorun Liu and Shiqi Yang and Shaohua Wang and Zhanlong Liu and Hao Zou and Jinyue Chen and Shu Wei and Jingjing Wu and Mingxin Huang and Zhen Wu and Guibin Wang and Tengyu Du and Lei Jia},
+      year={2026},
+      eprint={2606.23050},
+      archivePrefix={arXiv},
+      primaryClass={cs.CV},
+      url={https://arxiv.org/abs/2606.23050}, 
+}

evidence/bf16.json

@@ -1,5 +1,16 @@
 {
   "tag": "bf16",
+  "meta": {
+    "harness": "https://git.srk.rest/shadowrock/uocr-quant @ 1f584d7",
+    "date": "2026-08-17",
+    "base_model": "baidu/Unlimited-OCR",
+    "base_model_revision": "07dea832e22aefee32ad281d4b80551282e1c168",
+    "engine": "vLLM 0.26.0 (enforce_eager, max_model_len 8192)",
+    "torch": "2.11.0+cu130",
+    "gpu": "NVIDIA GeForce RTX 5070 Ti (SM120), driver 610.43.03",
+    "decoding": "greedy (temperature 0.0), max_tokens 1024",
+    "cer_definition": "Levenshtein/len(ref) on whitespace-normalized text after stripping grounding/box tags, vs BF16 reference transcript"
+  },
   "model_dir": "models/Unlimited-OCR",
   "prompt": "<image>\n<|grounding|>OCR this image.",
   "decode_tok_per_s": 45.34,

evidence/fp8.json

@@ -1,5 +1,16 @@
 {
   "tag": "fp8",
+  "meta": {
+    "harness": "https://git.srk.rest/shadowrock/uocr-quant @ 1f584d7",
+    "date": "2026-08-17",
+    "base_model": "baidu/Unlimited-OCR",
+    "base_model_revision": "07dea832e22aefee32ad281d4b80551282e1c168",
+    "engine": "vLLM 0.26.0 (enforce_eager, max_model_len 8192)",
+    "torch": "2.11.0+cu130",
+    "gpu": "NVIDIA GeForce RTX 5070 Ti (SM120), driver 610.43.03",
+    "decoding": "greedy (temperature 0.0), max_tokens 1024",
+    "cer_definition": "Levenshtein/len(ref) on whitespace-normalized text after stripping grounding/box tags, vs BF16 reference transcript"
+  },
   "model_dir": "exports/Unlimited-OCR-FP8-Dynamic",
   "prompt": "<image>\n<|grounding|>OCR this image.",
   "decode_tok_per_s": 39.36,

evidence/nvfp4.json

@@ -1,5 +1,16 @@
 {
   "tag": "nvfp4",
+  "meta": {
+    "harness": "https://git.srk.rest/shadowrock/uocr-quant @ 1f584d7",
+    "date": "2026-08-17",
+    "base_model": "baidu/Unlimited-OCR",
+    "base_model_revision": "07dea832e22aefee32ad281d4b80551282e1c168",
+    "engine": "vLLM 0.26.0 (enforce_eager, max_model_len 8192)",
+    "torch": "2.11.0+cu130",
+    "gpu": "NVIDIA GeForce RTX 5070 Ti (SM120), driver 610.43.03",
+    "decoding": "greedy (temperature 0.0), max_tokens 1024",
+    "cer_definition": "Levenshtein/len(ref) on whitespace-normalized text after stripping grounding/box tags, vs BF16 reference transcript"
+  },
   "model_dir": "exports/Unlimited-OCR-NVFP4",
   "prompt": "<image>\n<|grounding|>OCR this image.",
   "decode_tok_per_s": 40.84,

evidence/probe-nvfp4.json

@@ -1,5 +1,16 @@
 {
   "tag": "probe-nvfp4",
+  "meta": {
+    "harness": "https://git.srk.rest/shadowrock/uocr-quant @ 1f584d7",
+    "date": "2026-08-17",
+    "base_model": "baidu/Unlimited-OCR",
+    "base_model_revision": "07dea832e22aefee32ad281d4b80551282e1c168",
+    "engine": "vLLM 0.26.0 (enforce_eager, max_model_len 8192)",
+    "torch": "2.11.0+cu130",
+    "gpu": "NVIDIA GeForce RTX 5070 Ti (SM120), driver 610.43.03",
+    "decoding": "greedy (temperature 0.0), max_tokens 1024",
+    "cer_definition": "Levenshtein/len(ref) on whitespace-normalized text after stripping grounding/box tags, vs BF16 reference transcript"
+  },
   "model_dir": "models/probe-nvfp4",
   "prompt": "<image>\n<|grounding|>OCR this image.",
   "decode_tok_per_s": 64.24,