evidence/fp8.json

{
  "tag": "fp8",
  "meta": {
    "harness": "https://git.srk.rest/shadowrock/uocr-quant @ 1f584d7",
    "date": "2026-08-17",
    "base_model": "baidu/Unlimited-OCR",
    "base_model_revision": "07dea832e22aefee32ad281d4b80551282e1c168",
    "engine": "vLLM 0.26.0 (enforce_eager, max_model_len 8192)",
    "torch": "2.11.0+cu130",
    "gpu": "NVIDIA GeForce RTX 5070 Ti (SM120), driver 610.43.03",
    "decoding": "greedy (temperature 0.0), max_tokens 1024",
    "cer_definition": "Levenshtein/len(ref) on whitespace-normalized text after stripping grounding/box tags, vs BF16 reference transcript"
  },
  "model_dir": "exports/Unlimited-OCR-FP8-Dynamic",
  "prompt": "<image>\n<|grounding|>OCR this image.",
  "decode_tok_per_s": 39.36,
  "peak_vram_gib_client": null,
  "fixtures": {
    "invoice.png": {
      "raw": "title [54, 80, 403, 128]INVOICE #2026-0817\ntext [54, 262, 457, 310]Bill To: ShadowRock LLC\ntext [54, 354, 577, 406]Item: GPU Quantization Services\ntext [54, 445, 403, 501]Qty: 2 Unit: $150.00\ntext [54, 628, 356, 675]Total Due: $300.00\ntext [54, 718, 527, 768]Due Date: September 1, 2026",
      "ntok": 145,
      "seconds": 6.071
    },
    "memo.png": {
      "raw": "title [35, 42, 303, 85]MEMORANDUM\ntext [34, 140, 306, 180]To: Engineering Team\ntext [35, 211, 253, 249]From: Operations\ntext [35, 280, 310, 318]Date: August 17, 2026\ntext [35, 350, 501, 387]Re: GPU cluster maintenance window\ntext [32, 489, 648, 808]The primary GPU cluster will be offline for scheduled maintenance from 02:00 to 06:00 UTC on August 24. Batch jobs should be drained before 01:30 UTC. Contact ops@example.com with any questions.",
      "ntok": 169,
      "seconds": 2.843
    },
    "table_report.png": {
      "raw": "text [56, 48, 520, 98]Quarterly Revenue Report\ntable [56, 158, 941, 454]<table>QuarterRevenueGrowthQ1 2026$4.2M12%Q2 2026$4.9M16%Q3 2026$5.1M4%</table>\ntext [53, 513, 681, 604]Operating expenses decreased 3% QoQ, driven by reduced cloud infrastructure spending.",
      "ntok": 144,
      "seconds": 2.722
    }
  },
  "cer_vs_bf16": {
    "invoice.png": 0.0,
    "memo.png": 0.0,
    "table_report.png": 0.0
  },
  "mean_cer_vs_bf16": 0.0
}