extractors.py

"""Keyword and structured-field extraction for OCR'd document text.

Domain keyword sets flag "notification" events (things a reviewer would want
alerted on), and field patterns pull structured values (IDs, dates, amounts)
out of the surrounding text so results can be dropped straight into a CSV.
"""
import re

KEYWORD_SETS = {
    "construction": [
        "change order", "delay notice", "punch list", "non-conformance",
        "rfi", "submittal rejected", "back charge", "backcharge",
        "stop work", "safety violation", "material shortage",
        "schedule slip", "cost overrun", "field order", "notice to proceed",
    ],
    "compliance": [
        "non-compliant", "noncompliance", "violation", "deficiency",
        "corrective action", "audit finding", "expired", "expiration",
        "recall", "citation", "penalty", "out of compliance",
        "failed inspection", "not approved",
    ],
    "finance": [
        "past due", "overdue", "payment rejected", "invoice disputed",
        "credit hold", "chargeback", "insufficient funds", "write-off",
        "collections", "declined", "late fee", "short paid",
    ],
    "general": [
        "urgent", "immediate action required", "cancelled", "rejected",
        "approved", "pending approval", "revised", "on hold",
    ],
}

FIELD_PATTERNS = {
    "dates": re.compile(
        r"\b(?:\d{1,2}[/-]\d{1,2}[/-]\d{2,4}|\d{4}-\d{2}-\d{2}|"
        r"(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)[a-z]*\.?\s+"
        r"\d{1,2},?\s+\d{4})\b",
        re.IGNORECASE,
    ),
    "amounts": re.compile(
        r"\$\s?\d[\d,]*\.?\d{0,2}\b|\bUSD\s?\d[\d,]*\.?\d{0,2}\b",
        re.IGNORECASE,
    ),
    "percentages": re.compile(r"\b\d{1,3}(?:\.\d+)?\s?%"),
    "po_numbers": re.compile(
        r"\bP\.?\s?O\.?[ \t]*#?[ \t]*[:\-]?[ \t]*((?=[A-Z0-9-]*\d)[A-Z0-9\-]{3,15})\b",
        re.IGNORECASE,
    ),
    "sap_doc_numbers": re.compile(
        r"\bSAP[ \t]*(?:Doc(?:ument)?\.?|#)?[ \t]*[:\-]?[ \t]*(\d{8,10})\b|\b(4[5-8]\d{8})\b",
        re.IGNORECASE,
    ),
    "invoice_numbers": re.compile(
        r"\bINV(?:OICE)?\.?[ \t]*#?[ \t]*[:\-]?[ \t]*((?=[A-Z0-9-]*\d)[A-Z0-9\-]{3,15})\b",
        re.IGNORECASE,
    ),
    "cert_codes": re.compile(
        r"\b(ISO\s?\d{3,5}(?:-\d+)?|OSHA\s?\d{3,4}(?:\.\d+)?|"
        r"ASTM\s?[A-Z]?\d{2,4}(?:-\d{2})?|ANSI\s?[A-Z]?\d+(?:\.\d+)?)\b",
        re.IGNORECASE,
    ),
}

CONTEXT_WINDOW = 60


def _dedupe(values):
    seen = []
    for v in values:
        v = v.strip()
        if v and v not in seen:
            seen.append(v)
    return seen


def extract_fields(text):
    """Pull structured fields out of a page/block of text."""
    fields = {}
    for name, pattern in FIELD_PATTERNS.items():
        matches = pattern.findall(text)
        flat = []
        for m in matches:
            if isinstance(m, tuple):
                flat.extend(g for g in m if g)
            else:
                flat.append(m)
        fields[name] = _dedupe(flat)
    return fields


def find_notifications(text, categories, custom_keywords=None):
    """Scan text for keyword hits across the given categories.

    Returns a list of dicts: {category, keyword, context}.
    """
    hits = []
    keywords_by_category = {}
    for cat in categories:
        keywords_by_category[cat] = list(KEYWORD_SETS.get(cat, []))
    if custom_keywords:
        keywords_by_category["custom"] = list(custom_keywords)

    lower_text = text.lower()
    for category, keywords in keywords_by_category.items():
        for kw in keywords:
            kw_clean = kw.strip().lower()
            if not kw_clean:
                continue
            start = 0
            while True:
                idx = lower_text.find(kw_clean, start)
                if idx == -1:
                    break
                ctx_start = max(0, idx - CONTEXT_WINDOW)
                ctx_end = min(len(text), idx + len(kw_clean) + CONTEXT_WINDOW)
                context = text[ctx_start:ctx_end].replace("\n", " ").strip()
                hits.append({
                    "category": category,
                    "keyword": kw.strip(),
                    "context": context,
                })
                start = idx + len(kw_clean)
    return hits