Publish artifacts for multimodal-document-retrieval-20260802

README.md

@@ -0,0 +1,61 @@
+---
+license: mit
+library_name: custom
+pipeline_tag: visual-document-retrieval
+datasets:
+- RKB109/multimodal-document-retrieval-20260802-dataset
+tags:
+- synthetic-data
+- transparent-baseline
+- multimodal-ai
+- visual-document-retrieval
+- document-question-answering
+- image-to-text
+- feature-extraction
+metrics:
+- accuracy
+---
+
+# Multimodal Document Retrieval Baseline Baseline Model
+
+## Model Description
+
+This repository contains a small, transparent prototype model for
+**Business documents contain meaning in text, tables, layout, and imagery that text-only retrieval can miss.**
+
+The model combines per-label token weights with IDF-weighted evidence
+retrieval. It was generated for reproducible architecture demonstrations and
+does not call a hosted LLM.
+
+## Evaluation
+
+- Held-out synthetic examples: 4
+- Accuracy: 1
+- Intended metrics: retrieval_accuracy, modality_coverage, recall_at_3
+
+## Intended Use
+
+- Architecture prototyping
+- CI and evaluation examples
+- Local baseline comparisons
+- Educational experimentation
+
+## Hugging Face Task Coverage
+
+- `visual-document-retrieval`
+- `document-question-answering`
+- `image-to-text`
+- `feature-extraction`
+
+## Limitations and Risks
+
+The starter dataset contains synthetic textual modality descriptors, not sensitive scanned documents.
+
+The dataset is synthetic and small. Do not use this model for consequential
+decisions without representative data, expert review, and production-grade
+evaluation.
+
+## Reproducibility
+
+The linked GitHub repository includes `train.py`, the exact dataset split,
+evaluation code, and the model JSON format.

evaluation.json

@@ -0,0 +1,5 @@
+{
+  "test_examples": 4,
+  "accuracy": 1,
+  "synthetic_evaluation": true
+}

inference.py

@@ -0,0 +1,80 @@
+"""Transparent baseline pipeline for the generated AI project."""
+
+from __future__ import annotations
+
+import json
+import math
+import re
+from pathlib import Path
+
+
+def tokenize(value: str) -> list[str]:
+    return re.findall(r"[a-z0-9]+", value.lower())
+
+
+class Pipeline:
+    def __init__(self, model: dict):
+        self.model = model
+
+    @classmethod
+    def from_file(cls, path: str | Path) -> "Pipeline":
+        return cls(json.loads(Path(path).read_text(encoding="utf-8")))
+
+    def classify(self, text: str) -> tuple[str, float]:
+        tokens = tokenize(text)
+        scores = {
+            label: sum(weights.get(token, 0) for token in tokens)
+            for label, weights in self.model["prototypes"].items()
+        }
+        ranked = sorted(scores.items(), key=lambda item: (-item[1], item[0]))
+        label, best = ranked[0]
+        total = sum(max(score, 0) for _, score in ranked) or 1
+        return label, best / total
+
+    def search(self, query: str, limit: int = 3) -> list[dict]:
+        query_tokens = set(tokenize(query))
+        ranked = []
+        for document in self.model["documents"]:
+            document_tokens = set(tokenize(document["text"]))
+            lexical = sum(
+                self.model["idf"].get(token, 1.0)
+                for token in query_tokens & document_tokens
+            )
+            ranked.append({**document, "score": round(lexical, 6)})
+        return sorted(ranked, key=lambda item: (-item["score"], item["id"]))[:limit]
+
+    def graph_evidence(self, text: str) -> list[dict]:
+        tokens = set(tokenize(text))
+        matches = []
+        for subject, relation, target in self.model.get("graph_edges", []):
+            edge_tokens = set(tokenize(f"{subject} {relation} {target}"))
+            overlap = len(tokens & edge_tokens)
+            if overlap:
+                matches.append(
+                    {
+                        "subject": subject,
+                        "relation": relation,
+                        "target": target,
+                        "overlap": overlap,
+                    }
+                )
+        return sorted(matches, key=lambda item: -item["overlap"])
+
+    def run(self, text: str) -> dict:
+        label, confidence = self.classify(text)
+        evidence = self.search(text)
+        result = {
+            "prediction": label,
+            "confidence": round(confidence, 4),
+            "requires_review": confidence < self.model["confidence_threshold"],
+            "evidence": evidence,
+        }
+        if self.model["mode"] == "graph":
+            result["graph_evidence"] = self.graph_evidence(text)
+        if self.model["mode"] == "agent":
+            result["proposed_tool"] = label
+            result["approval_required"] = label in {
+                "request-approval",
+                "request-human-help",
+            }
+        return result

model.json

@@ -0,0 +1,197 @@
+{
+  "format": "daily-project-prototype-v1",
+  "project": "multimodal-document-retrieval",
+  "title": "Multimodal Document Retrieval Baseline",
+  "domain": "multimodal-ai",
+  "mode": "retrieval",
+  "labels": [
+    "invoice",
+    "technical-diagram",
+    "policy-form"
+  ],
+  "prototypes": {
+    "invoice": {
+      "80": 3,
+      "184": 2,
+      "1480": 3,
+      "find": 3,
+      "the": 3,
+      "invoice": 7,
+      "with": 3,
+      "a": 5,
+      "shipping": 6,
+      "surcharge": 6,
+      "table": 8,
+      "ocr": 5,
+      "total": 5,
+      "image": 3,
+      "company": 3,
+      "seal": 3,
+      "in": 2,
+      "an": 3,
+      "operations": 2,
+      "review": 2,
+      "for": 1,
+      "evaluation": 1,
+      "case": 1,
+      "which": 2,
+      "contains": 2,
+      "tax": 4,
+      "adjustment": 4,
+      "subtotal": 2,
+      "final": 2
+    },
+    "technical-diagram": {
+      "in": 2,
+      "an": 4,
+      "operations": 2,
+      "review": 2,
+      "locate": 2,
+      "the": 5,
+      "architecture": 2,
+      "diagram": 7,
+      "showing": 2,
+      "a": 5,
+      "queue": 4,
+      "caption": 5,
+      "service": 5,
+      "labels": 5,
+      "producer": 2,
+      "consumer": 2,
+      "for": 2,
+      "evaluation": 2,
+      "case": 2,
+      "show": 3,
+      "network": 6,
+      "with": 3,
+      "gateway": 6,
+      "topology": 3,
+      "client": 3
+    },
+    "policy-form": {
+      "find": 2,
+      "the": 2,
+      "signed": 2,
+      "privacy": 4,
+      "exception": 4,
+      "request": 4,
+      "ocr": 4,
+      "layout": 2,
+      "signature": 4,
+      "block": 2,
+      "approved": 4,
+      "for": 2,
+      "an": 3,
+      "evaluation": 2,
+      "case": 2,
+      "in": 1,
+      "operations": 1,
+      "review": 1,
+      "which": 2,
+      "form": 2,
+      "includes": 2,
+      "retention": 4,
+      "approval": 2,
+      "checkbox": 2,
+      "present": 2
+    }
+  },
+  "idf": {
+    "80": 2.252763,
+    "184": 2.252763,
+    "1480": 2.252763,
+    "ocr": 1.336472,
+    "total": 1.847298,
+    "table": 1.847298,
+    "shipping": 2.252763,
+    "surcharge": 2.252763,
+    "image": 2.252763,
+    "company": 2.252763,
+    "seal": 2.252763,
+    "caption": 1.847298,
+    "service": 1.847298,
+    "diagram": 2.252763,
+    "labels": 1.847298,
+    "producer": 2.252763,
+    "queue": 2.252763,
+    "consumer": 2.252763,
+    "privacy": 2.252763,
+    "exception": 2.252763,
+    "layout": 2.252763,
+    "signature": 1.847298,
+    "block": 2.252763,
+    "approved": 1.847298,
+    "invoice": 2.252763,
+    "subtotal": 2.252763,
+    "tax": 2.252763,
+    "adjustment": 2.252763,
+    "final": 2.252763,
+    "network": 2.252763,
+    "topology": 2.252763,
+    "client": 2.252763,
+    "gateway": 2.252763,
+    "retention": 2.252763,
+    "request": 2.252763,
+    "checkbox": 2.252763,
+    "present": 2.252763
+  },
+  "documents": [
+    {
+      "id": "doc-01",
+      "label": "invoice",
+      "text": "OCR total 1480; table shipping surcharge 80; image company seal.",
+      "metadata": {
+        "synthetic": true,
+        "domain": "multimodal-ai"
+      }
+    },
+    {
+      "id": "doc-02",
+      "label": "technical-diagram",
+      "text": "Caption service diagram; labels producer queue consumer.",
+      "metadata": {
+        "synthetic": true,
+        "domain": "multimodal-ai"
+      }
+    },
+    {
+      "id": "doc-03",
+      "label": "policy-form",
+      "text": "OCR privacy exception; layout signature block approved.",
+      "metadata": {
+        "synthetic": true,
+        "domain": "multimodal-ai"
+      }
+    },
+    {
+      "id": "doc-04",
+      "label": "invoice",
+      "text": "OCR invoice 184; table subtotal tax adjustment final total.",
+      "metadata": {
+        "synthetic": true,
+        "domain": "multimodal-ai"
+      }
+    },
+    {
+      "id": "doc-05",
+      "label": "technical-diagram",
+      "text": "Caption network topology; labels client gateway service.",
+      "metadata": {
+        "synthetic": true,
+        "domain": "multimodal-ai"
+      }
+    },
+    {
+      "id": "doc-06",
+      "label": "policy-form",
+      "text": "OCR retention request; checkbox approved; signature present.",
+      "metadata": {
+        "synthetic": true,
+        "domain": "multimodal-ai"
+      }
+    }
+  ],
+  "graph_edges": [],
+  "confidence_threshold": 0.18,
+  "trained_on_synthetic_data": true
+}

project.json

@@ -0,0 +1,52 @@
+{
+  "name": "Multimodal Document Retrieval Baseline",
+  "problem": "Business documents contain meaning in text, tables, layout, and imagery that text-only retrieval can miss.",
+  "domain": "multimodal-ai",
+  "architecture": "retrieval",
+  "hugging_face_tasks": [
+    "visual-document-retrieval",
+    "document-question-answering",
+    "image-to-text",
+    "feature-extraction"
+  ],
+  "recommended_stack": [
+    "FastAPI for ingestion and multimodal search",
+    "Transformers with LayoutLMv3 or ColPali-style encoders",
+    "OCR adapter plus image captioning",
+    "PostgreSQL plus pgvector for fused embeddings",
+    "Object storage for source documents",
+    "OpenTelemetry plus retrieval ablation reports"
+  ],
+  "real_world_data_sources": [
+    {
+      "name": "SEC EDGAR company facts API",
+      "url": "https://data.sec.gov/api/xbrl/companyfacts/CIK0000320193.json",
+      "purpose": "Real tabular filing facts and document metadata"
+    },
+    {
+      "name": "arXiv API",
+      "url": "https://export.arxiv.org/api/query?search_query=all:document%20understanding&start=0&max_results=5",
+      "purpose": "Public papers and document-layout retrieval scenarios"
+    }
+  ],
+  "job_description_skills": [
+    "Multimodal embeddings and document understanding",
+    "OCR, layout, image, and text feature fusion",
+    "Vector retrieval and modality-aware ranking",
+    "Ablation testing and retrieval evaluation",
+    "Scalable document ingestion and object storage"
+  ],
+  "impact_targets": [
+    "Improve Recall@5 by >= 20% over text-only retrieval",
+    "Report modality ablations for OCR, layout, and image signals",
+    "Index 100,000 documents with resumable ingestion",
+    "Keep p95 retrieval latency below 500 ms"
+  ],
+  "baseline_evaluation": {
+    "test_examples": 4,
+    "accuracy": 1,
+    "synthetic_evaluation": true
+  },
+  "estimated_delivery": "8-12 weeks for one engineer",
+  "generated_baseline_is_production_ready": false
+}