Spaces:
Sleeping
Sleeping
File size: 4,239 Bytes
2a9e166 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 | """Keyword and structured-field extraction for OCR'd document text.
Domain keyword sets flag "notification" events (things a reviewer would want
alerted on), and field patterns pull structured values (IDs, dates, amounts)
out of the surrounding text so results can be dropped straight into a CSV.
"""
import re
KEYWORD_SETS = {
"construction": [
"change order", "delay notice", "punch list", "non-conformance",
"rfi", "submittal rejected", "back charge", "backcharge",
"stop work", "safety violation", "material shortage",
"schedule slip", "cost overrun", "field order", "notice to proceed",
],
"compliance": [
"non-compliant", "noncompliance", "violation", "deficiency",
"corrective action", "audit finding", "expired", "expiration",
"recall", "citation", "penalty", "out of compliance",
"failed inspection", "not approved",
],
"finance": [
"past due", "overdue", "payment rejected", "invoice disputed",
"credit hold", "chargeback", "insufficient funds", "write-off",
"collections", "declined", "late fee", "short paid",
],
"general": [
"urgent", "immediate action required", "cancelled", "rejected",
"approved", "pending approval", "revised", "on hold",
],
}
FIELD_PATTERNS = {
"dates": re.compile(
r"\b(?:\d{1,2}[/-]\d{1,2}[/-]\d{2,4}|\d{4}-\d{2}-\d{2}|"
r"(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)[a-z]*\.?\s+"
r"\d{1,2},?\s+\d{4})\b",
re.IGNORECASE,
),
"amounts": re.compile(
r"\$\s?\d[\d,]*\.?\d{0,2}\b|\bUSD\s?\d[\d,]*\.?\d{0,2}\b",
re.IGNORECASE,
),
"percentages": re.compile(r"\b\d{1,3}(?:\.\d+)?\s?%"),
"po_numbers": re.compile(
r"\bP\.?\s?O\.?[ \t]*#?[ \t]*[:\-]?[ \t]*((?=[A-Z0-9-]*\d)[A-Z0-9\-]{3,15})\b",
re.IGNORECASE,
),
"sap_doc_numbers": re.compile(
r"\bSAP[ \t]*(?:Doc(?:ument)?\.?|#)?[ \t]*[:\-]?[ \t]*(\d{8,10})\b|\b(4[5-8]\d{8})\b",
re.IGNORECASE,
),
"invoice_numbers": re.compile(
r"\bINV(?:OICE)?\.?[ \t]*#?[ \t]*[:\-]?[ \t]*((?=[A-Z0-9-]*\d)[A-Z0-9\-]{3,15})\b",
re.IGNORECASE,
),
"cert_codes": re.compile(
r"\b(ISO\s?\d{3,5}(?:-\d+)?|OSHA\s?\d{3,4}(?:\.\d+)?|"
r"ASTM\s?[A-Z]?\d{2,4}(?:-\d{2})?|ANSI\s?[A-Z]?\d+(?:\.\d+)?)\b",
re.IGNORECASE,
),
}
CONTEXT_WINDOW = 60
def _dedupe(values):
seen = []
for v in values:
v = v.strip()
if v and v not in seen:
seen.append(v)
return seen
def extract_fields(text):
"""Pull structured fields out of a page/block of text."""
fields = {}
for name, pattern in FIELD_PATTERNS.items():
matches = pattern.findall(text)
flat = []
for m in matches:
if isinstance(m, tuple):
flat.extend(g for g in m if g)
else:
flat.append(m)
fields[name] = _dedupe(flat)
return fields
def find_notifications(text, categories, custom_keywords=None):
"""Scan text for keyword hits across the given categories.
Returns a list of dicts: {category, keyword, context}.
"""
hits = []
keywords_by_category = {}
for cat in categories:
keywords_by_category[cat] = list(KEYWORD_SETS.get(cat, []))
if custom_keywords:
keywords_by_category["custom"] = list(custom_keywords)
lower_text = text.lower()
for category, keywords in keywords_by_category.items():
for kw in keywords:
kw_clean = kw.strip().lower()
if not kw_clean:
continue
start = 0
while True:
idx = lower_text.find(kw_clean, start)
if idx == -1:
break
ctx_start = max(0, idx - CONTEXT_WINDOW)
ctx_end = min(len(text), idx + len(kw_clean) + CONTEXT_WINDOW)
context = text[ctx_start:ctx_end].replace("\n", " ").strip()
hits.append({
"category": category,
"keyword": kw.strip(),
"context": context,
})
start = idx + len(kw_clean)
return hits
|