# -*- coding: utf-8 -*-
"""يبني ورقة docx منسّقة بالكامل (A4، 2.5cm، Times New Roman، أحجام خطوط، جداول قابلة للتحرير،
أشكال، معادلة مرقمة، RTL للعربية) من محتوى الورقة المحدّثة."""
from docx import Document
from docx.shared import Pt, Cm, RGBColor
from docx.enum.text import WD_ALIGN_PARAGRAPH
from docx.enum.table import WD_TABLE_ALIGNMENT
from docx.oxml.ns import qn
from docx.oxml import OxmlElement

doc = Document()
# ── إعداد الصفحة A4 + هوامش 2.5سم ──
s = doc.sections[0]
s.page_height, s.page_width = Cm(29.7), Cm(21.0)
for m in ('top_margin', 'bottom_margin', 'left_margin', 'right_margin'):
    setattr(s, m, Cm(2.5))
# ── الخط الافتراضي Times New Roman (لاتيني + عربي/complex) ──
normal = doc.styles['Normal']
normal.font.name = 'Times New Roman'; normal.font.size = Pt(12)
rpr = normal.element.get_or_add_rPr(); rf = rpr.get_or_add_rFonts()
rf.set(qn('w:ascii'), 'Times New Roman'); rf.set(qn('w:hAnsi'), 'Times New Roman')
rf.set(qn('w:cs'), 'Times New Roman'); rf.set(qn('w:eastAsia'), 'Times New Roman')
normal.paragraph_format.space_after = Pt(4); normal.paragraph_format.line_spacing = 1.06

def _rtl(p):
    ppr = p._p.get_or_add_pPr(); b = OxmlElement('w:bidi'); ppr.append(b)

def para(text='', size=12, bold=False, italic=False, align=None, rtl=False, after=4, color=None):
    p = doc.add_paragraph()
    if align: p.alignment = align
    p.paragraph_format.space_after = Pt(after)
    if rtl: _rtl(p)
    if text:
        r = p.add_run(text); r.font.size = Pt(size); r.bold = bold; r.italic = italic
        r.font.name = 'Times New Roman'
        rpr = r._r.get_or_add_rPr(); rf = rpr.get_or_add_rFonts(); rf.set(qn('w:cs'), 'Times New Roman')
        if rtl:
            rtlEl = OxmlElement('w:rtl'); rpr.append(rtlEl)
        if color: r.font.color.rgb = color
    return p

def heading(num, en):
    para(f'{num}. {en}', size=12, bold=True, after=3)

INK = RGBColor(0x1c, 0x21, 0x33); GOLD = RGBColor(0xC4, 0x96, 0x28)

def table(caption_num, caption, headers, rows, rtl_cells=False):
    para(f'Table {caption_num}. {caption}', size=10, bold=True, after=2)
    t = doc.add_table(rows=1, cols=len(headers)); t.style = 'Light Grid Accent 1'
    t.alignment = WD_TABLE_ALIGNMENT.CENTER
    hdr = t.rows[0].cells
    for i, h in enumerate(headers):
        hdr[i].text = ''
        rn = hdr[i].paragraphs[0].add_run(h); rn.bold = True; rn.font.size = Pt(9.5)
        rn.font.name = 'Times New Roman'
        hdr[i].paragraphs[0].alignment = WD_ALIGN_PARAGRAPH.CENTER
    for row in rows:
        cells = t.add_row().cells
        for i, v in enumerate(row):
            cells[i].text = ''
            p = cells[i].paragraphs[0]
            rn = p.add_run(str(v)); rn.font.size = Pt(9.5); rn.font.name = 'Times New Roman'
            p.alignment = WD_ALIGN_PARAGRAPH.CENTER if i > 0 else WD_ALIGN_PARAGRAPH.LEFT
    doc.add_paragraph().paragraph_format.space_after = Pt(2)

def figure(path, num, caption, width=Cm(15.5)):
    p = doc.add_paragraph(); p.alignment = WD_ALIGN_PARAGRAPH.CENTER
    p.add_run().add_picture(path, width=width)
    cap = para(f'Figure {num}. {caption}', size=10, italic=True, align=WD_ALIGN_PARAGRAPH.CENTER, after=6)

# ══════════════ العنوان وبيانات المؤلفين ══════════════
para('أثر — An Assistive Hybrid Pipeline for Arabic Manuscript HTR and Scholarly Review',
     size=14, bold=True, align=WD_ALIGN_PARAGRAPH.CENTER, after=6)
para('أثر', size=12, bold=True, align=WD_ALIGN_PARAGRAPH.CENTER, after=8)

# ══════════════ English Abstract ══════════════
para('Abstract', size=12, bold=True, after=2)
para('Historical Arabic manuscripts remain largely locked in page images: calligraphic variation, dense '
     'marginalia, page degradation, and domain-specific terminology defeat generic OCR. We present '
     'اثر, an assistive hybrid pipeline coupling layout segmentation and Kraken-based '
     'Handwritten Text Recognition (HTR) with six transparent assistance layers: a confidence-gated '
     'character n-gram language model, a layout router separating framed logic/philosophy pages from '
     'general pages, a human-in-the-loop manual/PAGE-XML segmentation mode, a contextual LLM reading '
     'assistant, a retrieval-augmented corrector aligning lines with published digital texts (Qur’an, '
     'ḥadīth), and a relevance-ranked manuscript catalog. Building on open resources (Muharaf, Kraken, '
     'eScriptorium), our strongest verified general model with an order-8 language model reaches 6.76% CER '
     'on RASAM-test and 8.68% CER on TariMa-test. The gated language model improves CER on three '
     'separated datasets without harming clean pages, and a closed-loop self-training experiment reduces '
     'logic-domain language-model perplexity by 62% (53.14→20.25). The contribution is a reproducible, '
     'review-oriented pipeline that preserves the original transcription and surfaces uncertainty.', size=10, after=3)
para('Keywords: Arabic manuscript HTR; confidence-gated language model; layout routing; '
     'retrieval-augmented correction; human-in-the-loop.', size=10, bold=False, italic=True, after=8)

# ══════════════ الخلاصة العربية ══════════════
para('الملخّص', size=12, bold=True, rtl=True, align=WD_ALIGN_PARAGRAPH.RIGHT, after=2)
para('تبقى المخطوطات العربية التاريخية حبيسة الصور؛ فتنوع الخطوط، وكثافة الحواشي، وتلف الصفحات، والمصطلحات المتخصصة تجعل أنظمة OCR العامة غير كافية. نقدم اثر، وهو خط معالجة هجين مساعد يجمع بين تجزئة التخطيط والتعرف على الخط اليدوي (HTR) بمحرك Kraken، مع ست طبقات مساعدة شفافة: نموذج لغوي حرفي موجه بالثقة، وراوتر تخطيط يميز صفحات المنطق والفلسفة المؤطرة عن الصفحات العامة، ووضع قص يدوي واستيراد PAGE-XML بمشاركة الإنسان، ومساعد قراءة سياقي بنموذج لغوي كبير، ومصحح بالاسترجاع يطابق الأسطر مع نصوص رقمية منشورة، وفهرس مخطوطات مرتب بحسب الصلة. يحقق أقوى نموذج عام متحقق منه معدل خطأ حرفي 6.76% على RASAM-test و 8.68% على TariMa-test عند استخدام نموذج لغوي من الرتبة الثامنة. كما تحسن البوابة اللغوية النتائج على ثلاث مجموعات منفصلة دون الإضرار بالصفحات النظيفة، وتخفض تجربة التدريب الذاتي حيرة النموذج اللغوي في مجال المنطق بنسبة 62%. يهدف النظام إلى تسريع مراجعة المخطوطات مع حفظ النص الأصلي وإظهار مواضع الشك بدلا من استبدالها بصمت.', size=10, rtl=True, align=WD_ALIGN_PARAGRAPH.RIGHT, after=3)
para('الكلمات المفتاحية: تعرّف الخط اليدوي العربي؛ نموذج لغوي مُبوَّب؛ راوتر تخطيط؛ التصحيح بالاسترجاع؛ '
     'مشاركة الإنسان.', size=10, italic=True, rtl=True, align=WD_ALIGN_PARAGRAPH.RIGHT, after=8)

# ══════════════ 1. Introduction ══════════════
heading(1, 'Introduction')
para('Digitizing Arabic manuscripts as images does not make them readable, searchable, or analyzable. '
     'Scholarly review needs more than a transcript: uncertainty flagging, alternative readings, term '
     'explanation, and cross-referencing with known texts. Generic OCR fails on handwritten Arabic because '
     'one base shape maps to several letters depending on dots and scribal convention, and dense commentary '
     'and frames break naive reading order. اثر builds on open foundations — Muharaf [1], '
     'eScriptorium [2], Kraken [3] — and extends them into an assistive hybrid pipeline for recognition '
     'and review together. Contributions: (i) domain fine-tuning cutting CER from 37.91% to 6.76%; (ii) a '
     'confidence-gated LM decoder helping hard lines without harming clean ones; (iii) six transparent '
     'assistance layers around — never replacing — the recognizer; and (iv) two research probes '
     '(retrieval-augmented correction and closed-loop self-training) validated on real pages.', size=12)

# ══════════════ 2. Problem ══════════════
heading(2, 'Project Problem and Objectives')
para('Historical Arabic manuscripts exhibit calligraphic variation (Naskh, Maghrebi, Eastern), page '
     'degradation, dense marginalia around framed main text, non-standard orthography, and domain-specific '
     'vocabulary where one uncertain word can change an argument. Off-the-shelf recognizers both mis-read '
     'these pages and hide their uncertainty — unsafe for scholarship. Our objectives: reduce CER through '
     'data preparation and fine-tuning; add a language layer that helps difficult lines without degrading '
     'easy ones; handle framed layouts via routing and human-in-the-loop tools; surface uncertainty with '
     'transparent, non-destructive assistance; and keep the pipeline reproducible and honest.', size=12)

# ══════════════ 3. Proposed Solution / Innovation ══════════════
heading(3, 'Proposed Solution and Originality')
para('The system is a staged pipeline (Figure 1): conservative preprocessing → layout routing → '
     'segmentation → HTR → confidence-gated LM decoding → a transparent post-OCR assistance layer, '
     'plus a human-in-the-loop manual/PAGE-XML path for the hardest pages. Unlike a plain OCR tool, the '
     'assistance layers are additive and reversible: the recognizer output is preserved and every suggestion '
     'is a labeled annotation the reviewer accepts by a click. Novel elements: confidence-gated shallow '
     'fusion (LM only on low-confidence lines, with a large-change guard flagging aggressive rewrites as '
     'uncertain); a frame-detection router (vertical-rule fill cf>0.92) that fixed a false-positive which had '
     'destroyed a normal page (58.3%→4.6% CER); retrieval-augmented correction against canonical texts; '
     'and closed-loop self-training that improves the domain LM without new hand labels.', size=12)
figure('figures/figure1_architecture.png', 1,
       'Hybrid pipeline: preprocessing, routing, segmentation, HTR, and gated-LM decoding, with a '
       'human-in-the-loop crop path and a transparent post-OCR assistance layer.')

# ══════════════ 4. Methodology ══════════════
heading(4, 'Methodology and Tools')
para('Preprocessing (safe path): recognition models operate on grayscale (L) images; binarization (nlbin), '
     'enhancement, resizing, and deskew are disabled by default, matching eScriptorium, because binarizing an '
     'L-model input distorts strokes. Segmentation and routing: a fine-tuned model (seg_finetuned_v4, trained '
     'on 319 RASAM pages) produces HTR-compatible baselines; only pages with a confirmed rectangular frame use '
     'the specialized logic models. Recognition: Kraken CTC HTR models, fine-tuned with a warmup/freeze recipe '
     '(--warmup 2, --freeze-backbone 2) protecting the visual backbone while sequence layers adapt. Gated LM '
     'decoding combines an offline character-level 8-gram LM with acoustic confidence during beam search:', size=12)
# ── المعادلة المرقمة ──
eq = para('Score(y | x) = log P_HTR(y | x) + α · log P_LM(y) + β · |y|            (1)',
          size=12, italic=True, align=WD_ALIGN_PARAGRAPH.CENTER, after=4)
para('with α=0.5, β=0.3, beam=10. A per-line confidence gate (mean character probability): if '
     'confidence ≥ 0.95 keep greedy; else apply the LM; if the LM changes >50% of the line, reject and flag '
     'uncertain. Tools: Kraken; eScriptorium; PyTorch on NVIDIA RTX 4050 (CUDA + FP16); FastAPI + PWA; an '
     'offline n-gram LM; the OpenAI GPT-5 family for the contextual assistant; OpenITI/Tanzil digital texts '
     'for retrieval. Evaluation uses CER (primary) and WER on frozen splits; three page sets (GT-segmented, '
     'manual/GT-demo, automatic-segmentation) are kept separate and never mixed.', size=12)

# ══════════════ 5. Results ══════════════
heading(5, 'Results and Practical Demonstration')
para('Recognition is a progression: a weak generic baseline improved by domain training, mixed-domain data, '
     'and LM rescoring (Table 1, Figure 2). Domain fine-tuning cuts CER 37.91%→7.48%; the order-8 LM adds '
     '7.48→6.76% on RASAM and 10.27→8.68% on TariMa (zero-shot domain gap).', size=12)
table(1, 'Model performance (source: metrics_summary.csv).',
      ['Model', 'Dataset', 'LM', 'CER%', 'WER%', 'CharAcc%', 'WordAcc%'],
      [['muharaf_rec_best (baseline)', 'RASAM-test', 'No', '37.91', '88.12', '62.09', '11.88'],
       ['exp4A (RASAM-trained)', 'RASAM-test', 'No', '7.64', '27.77', '92.36', '72.23'],
       ['exp6 (general)', 'RASAM-test', 'No', '7.48', '27.17', '92.52', '72.83'],
       ['exp6 + LM (order-8)', 'RASAM-test', 'Yes', '6.76', '27.17', '93.24', '72.83'],
       ['exp6 + LM (order-8)', 'TariMa-test', 'Yes', '8.68', '37.84', '91.32', '62.16'],
       ['exp7_easy (specialized)', 'Easy-Family', 'No', '5.94', '22.38', '94.06', '77.62'],
       ['exp7_hard (specialized)', 'Hard-Family', 'No', '13.70', '45.61', '86.30', '54.39']])
figure('figures/figure2_cer.png', 2, 'CER across models (lower is better). Source: project metric outputs.', width=Cm(13.5))
para('Confidence-gated LM (Table 2): always-LM helps hard sets but harms the clean set (0.39→0.56); the '
     'gate captures most of the gain while improving even the clean set (0.39→0.30) and lowering WER '
     '(D2 15.30→10.48). This validates gating as the safe operating point.', size=12)
table(2, 'Gated LM (threshold <0.95) vs. greedy vs. always-LM, CER% (source: LM_CONFIDENCE_GATE_REPORT.md).',
      ['Dataset', 'Greedy', 'Always-LM', 'Gated <0.95'],
      [['D1 clean (GT-seg)', '0.39', '0.56', '0.30'],
       ['D2 demo (GT/manual)', '4.34', '2.56', '3.29'],
       ['D3 auto-seg (v4)', '3.61', '2.28', '3.41'],
       ['Mean (3 sets)', '2.78', '1.80', '2.33']])
para('Full-pipeline benchmark on 13 held-out pages (auto-seg, GPU): normal pages average 7.63% CER; the two '
     '1926-family pages (a hard auto-seg layout) average 27.30% and raise the overall mean to 10.65%. On the '
     'same run: gated-LM interventions 12/18 improved (67% useful, 1 harmful); contextual candidate '
     'suggestions 6/10 improved and 0 harmful on the hardest lines; catalog search returned the correct work '
     'first on 5/5 queries after relevance re-ranking (Table 3).', size=12)
table(3, 'Assistance-layer measurements (source: FEATURES_BENCHMARK.md).',
      ['Component', 'Metric', 'Result'],
      [['Gated LM (auto-seg run)', 'useful interventions', '12 / 18 (67%)'],
       ['Contextual candidates', 'hard lines improved', '6 / 10 (0 harmful)'],
       ['Catalog search', 'correct work first', '5 / 5'],
       ['Retrieval correction', 'Qur’an al-ʿAlaq match', '98.3% (with reference)']])
para('Self-training the logic LM: transcribing 60 pages (GT excluded) yielded 418 high-confidence pseudo-'
     'labeled lines in the decoder’s own encoding. A mixed LM (general + pseudo) reduces perplexity on '
     'held-out logic GT by 62% (Figure 3, Table 4), confirming the closed loop models the domain far better; '
     'the human-in-the-loop PAGE-XML path recovers the hardest framed pages (logic 32.9%→3.2–3.6% CER '
     'with GT crops).', size=12)
table(4, 'Logic-domain LM perplexity, lower is better (source: RESEARCH_SELF_TRAINING.md).',
      ['Language model', 'Perplexity', 'vs. general'],
      [['General (char8_trainval)', '53.14', '—'],
       ['Self-trained (logic, 418 lines)', '40.70', '−23.4%'],
       ['Mixed (general + self-trained)', '20.25', '−61.9%']])
figure('figures/figure3_selftrain.png', 3, 'Logic-domain LM perplexity across self-training configurations '
       '(lower is better).', width=Cm(12.5))

# ══════════════ 6. Impact + Limitations ══════════════
heading(6, 'Applications, Limitations, and Conclusion')
para('The system turns page images into searchable, reviewable text with visible uncertainty: it accelerates '
     'scholarly transcription of logic/philosophy/legal manuscripts, supports catalog-level identification of '
     'works and authors, and flags/corrects canonical quotations via retrieval. Because assistance is additive '
     'and reversible, it is safe for critical-edition workflows. Limitations: automatic segmentation still '
     'fails on the densest framed layouts (1926 family ≈27% CER), recovered by the human-in-the-loop path; '
     'character-level LM rescoring leaves some spacing errors (≈16% of character errors); the custom logic '
     'subset is 32 pages (inventory 1,000 lines vs. ≈1,200 author audit — documented, not reconciled); '
     'and results are validated only for the included splits. Not measured: isolated post-processing effect, '
     'order-10 LM, end-to-end latency. Conclusion: اثر is a disciplined, reproducible assistive HTR '
     'pipeline reaching 6.76% CER (RASAM) / 8.68% (TariMa); its gated LM, router, human-in-the-loop '
     'segmentation, contextual assistant, retrieval correction, and catalog form a transparent review system, '
     'and two research probes show the architecture can improve itself (−62% perplexity) without new '
     'hand-labeled data.', size=12)

# ══════════════ References ══════════════
para('References', size=12, bold=True, after=2)
for ref in [
 '[1] M. Saeed, A. Chan, A. Mijar, J. Moukarzel, G. Habchi, C. Younes, A. Elias, C.-W. Wong, A. Khater. '
 'Muharaf: Manuscripts of Handwritten Arabic Dataset for Cursive Text Recognition. NeurIPS 2024 Datasets '
 'and Benchmarks Track, 2024.',
 '[2] The eScriptorium Project. eScriptorium: A Digital Text Production Pipeline for Print and Handwritten '
 'Texts using Machine Learning Techniques. Project documentation.',
 '[3] B. Kiessling. Kraken: a universal text recognizer for the humanities. Journal of Open Source Software, '
 '4(42):1764, 2019. DOI: 10.21105/joss.01764.',
 '[4] A. Graves, S. Fernandez, F. Gomez, and J. Schmidhuber. Connectionist Temporal Classification: Labelling '
 'Unsegmented Sequence Data with Recurrent Neural Networks. ICML, 2006.',
 '[5] OpenITI and Tanzil Project. Open Islamicate Texts Initiative and verified Qur\'anic text resources. '
 'Project documentation.']:
    para(ref, size=11, after=2)
para('Data ethics: the corpus combines public manuscript resources and project-curated material '
     '(≈1,200 author-transcribed lines; automated inventory 1,000). Public-source provenance remains '
     'visible; cleaning, organization, manual transcription, evaluation, and system integration are the '
     'project contributions. Every number is drawn from a committed result file or marked Not measured.',
     size=10, italic=True, after=2)

doc.save('PAPER_v2.docx')
print('DOCX_SAVED')