"""
Quick eval runner. Hits the deployed /api/chat/claude endpoint with each
question in sheet_questions.jsonl and reports:
  - whether expected keywords appear in the answer
  - whether the response carried a citation (URL in sources)
  - refusal behaviour for out-of-scope questions

Usage:
    BASE_URL=https://your-space.hf.space python eval/run_eval.py
"""
from __future__ import annotations

import json
import os
import sys
import time
import urllib.request
from pathlib import Path

BASE_URL = os.environ.get("BASE_URL", "http://localhost:7860").rstrip("/")
QUESTIONS_PATH = Path(__file__).with_name("sheet_questions.jsonl")
TIMEOUT = 45


def ask(question: str) -> dict:
    body = json.dumps({"message": question, "history": []}).encode("utf-8")
    req = urllib.request.Request(
        f"{BASE_URL}/api/chat/claude",
        data=body,
        headers={"Content-Type": "application/json"},
    )
    with urllib.request.urlopen(req, timeout=TIMEOUT) as resp:
        return json.loads(resp.read().decode("utf-8"))


def grade(case: dict, reply: dict) -> dict:
    text = (reply.get("response") or "").lower()
    sources = reply.get("sources") or []
    has_url = any((s.get("url") or "").startswith("http") for s in sources)

    if case.get("expects_refusal"):
        # Coarse refusal heuristic: chatbot should disclaim, not invent.
        refused = any(
            phrase in text
            for phrase in [
                "i don't have", "i do not have", "not in my", "outside",
                "can't help", "cannot help", "don't know",
            ]
        )
        return {"ok": refused, "kind": "refusal", "refused": refused}

    expected = [e.lower() for e in (case.get("expects") or [])]
    hits = [e for e in expected if e in text]
    keyword_ok = len(hits) >= max(1, len(expected) // 2) if expected else True
    citation_ok = (not case.get("expects_citation")) or has_url

    return {
        "ok": keyword_ok and citation_ok,
        "kind": "grounded",
        "keyword_hits": hits,
        "keyword_expected": expected,
        "has_citation_url": has_url,
        "sources": [
            {"title": s.get("title"), "url": s.get("url")} for s in sources
        ],
    }


def main() -> int:
    if not QUESTIONS_PATH.exists():
        print(f"Missing {QUESTIONS_PATH}", file=sys.stderr)
        return 2

    total = 0
    passed = 0
    rows: list[dict] = []
    with QUESTIONS_PATH.open() as f:
        for line in f:
            line = line.strip()
            if not line:
                continue
            case = json.loads(line)
            total += 1
            print(f"[{case['id']}] {case['question']}")
            t0 = time.time()
            try:
                reply = ask(case["question"])
            except Exception as e:
                print(f"  ERROR: {e}")
                rows.append({"id": case["id"], "error": str(e)})
                continue
            dt = time.time() - t0
            grade_row = grade(case, reply)
            grade_row["id"] = case["id"]
            grade_row["latency_s"] = round(dt, 2)
            rows.append(grade_row)
            if grade_row["ok"]:
                passed += 1
            mark = "PASS" if grade_row["ok"] else "FAIL"
            print(f"  {mark} ({dt:.1f}s) {grade_row}")

    print()
    print(f"Result: {passed}/{total} passed ({100 * passed // max(1, total)}%)")
    out_path = Path(__file__).with_name("last_run.json")
    out_path.write_text(json.dumps(rows, indent=2))
    print(f"Per-case results written to {out_path}")
    return 0 if passed == total else 1


if __name__ == "__main__":
    sys.exit(main())