chat_services.py

from __future__ import annotations

import math
import os
import pickle
import random
import re
import unicodedata
from collections import Counter
from pathlib import Path

import numpy as np

try:
    import pandas as pd
    from rank_bm25 import BM25Okapi
    from sklearn.metrics.pairwise import cosine_similarity
    _DEPS_OK = True
except ImportError:
    _DEPS_OK = False

try:
    from huggingface_hub import hf_hub_download
    _HF_OK = True
except ImportError:
    _HF_OK = False

NOMS_CAT = {
    'areti-maso':           'areti-maso',
    'aretim-po':            'aretim-po',
    'aretin-doha':          'aretin-doha',
    'aretin-kibo':          'aretin-kibo',
    'aretin-pivalanana':    'aretin-pivalanana',
    'aretin-tanana':        'aretin-tanana',
    'aretin-tenda':         'aretin-tenda',
    'aretin-tongotra':      'aretin-tongotra',
    'aretin-tratra':        'aretin-tratra',
    'aretin-tsaina':        'aretin-tsaina',
    'aretin-urinaire':      'aretin-urinaire',
    'diabeta':              'diabeta',
    'fiakaran-ny-tosi-dra': 'fiakaran-ny-tosi-dra',
    'kohaka':               'kohaka',
    'tazo':                 'tazo',
    'tazomoka':             'tazomoka',
}

ICONES = {
    'areti-maso':           '👁',
    'aretim-po':            '❤️',
    'aretin-doha':          '🧠',
    'aretin-kibo':          '🤢',
    'aretin-pivalanana':    '🚽',
    'aretin-tanana':        '🤲',
    'aretin-tenda':         '🗯',
    'aretin-tongotra':      '🦶',
    'aretin-tratra':        '🫀',
    'aretin-tsaina':        '🧘',
    'aretin-urinaire':      '💧',
    'diabeta':              '💉',
    'fiakaran-ny-tosi-dra': '🩸',
    'kohaka':               '😮',
    'tazo':                 '🌡',
    'tazomoka':             '🦟',
}

SALUTATIONS = {
    'salama': (
        " Salama! Tongasoa eto amin'ny Chatbot Médical Malagasy!\n"
        " Lazao amiko ny soritr'aretinao amin'ny teny malagasy.\n"
        " Ohatra: 'Marary ny lohako' na 'Voan'ny tazo aho'."
    ),
    'manao ahoana':     " Manao ahoana! Lazao amiko ny soritr'aretinao mba ahafahako manampy anao.",
    'manahoana':        " Manahoana! Inona no azoko anampiana anao?",
    'mbola tsara':      " Mbola tsara eh! Inona no azoko anampiana anao momban'ny fahasalamanao?",
    'veloma':           " Veloma! Mirary fahasalamana ho anao aho. Mitandrema amin'ny lafiny rehetra!",
    'misaotra':         " Misaotra betsaka! Faly hatrany aho manampy anao.",
    'misaotra betsaka': " Misaotra betsaka! Veloma ary tandremo ny fahasalamanao!",
    'eny':              " Tsara! Azonao atao ny milaza fanazavana fanampiny.",
    'tsia':             " Azo atao. Inona no mba hanampiako anao?",
    '/ampio': (
        " Torohevitra fampiasana:\n"
        " • Soraty ny soritr'aretinao amin'ny teny malagasy\n"
        " • Ohatra: 'Marary ny lohako', 'Voan'ny tazo aho'\n"
    ),
}

MSG_LANGUE = (
    " Azafady, resaho amin'ny malagasy ny momba ny fahasalamanao "
    "sy ny aretina mahazo anao."
)

MSGS_GIB = [
    "Tsy azonko izany teny izany. Mba soraty ny soritr'aretinao amin'ny malagasy.",
    "Miala tsiny, fa tsy mahalala izany voambolana izany aho. Lazao amin'ny malagasy azafady.",
    "Tsy mahafantatra izany aho. Inona ny soritr'aretina mahazo anao? Soraty amin'ny malagasy.",
]

MSGS_HORS = [
    "Miala tsiny fa tsy ao anatin'ny sehatry ny fitsaboana izany. Inona no soritr-aretina misy anao?",
    "Tsy misy fandikana ara-pahasalamana amin'izany. Lazao ny soritr'aretinao mba hanampiako.",
]

SYMPTOMES_GRAVES = [
    'mangorakoraka tampoka', 'very hevitra', 'lavo tampoka',
    'tsy afaka miaina', 'tazo 40', 'ra mivoaka be',
    'te hamono tena', 'tsy te-ho velona', 'tsy mahatsiaro tena', 'convulsion',
    'aita ny aina', 'infarctus', 'very ny hevitra', 'paralysie',
]

MSG_URGENCE = (
    " SORITR-ARETINA MAFY! MANDEHANA ANY AMIN-NY HOPITALY HAINGANA! \n"
    "Ity soritr-aretina ity dia mahafaty haingana aza miandry ela intsony, "
    "mandehana hopitaly izao!"
)

VOYELLES = set('aeiouàâäéèêëîïôùûü')

CONF_MED = 0.35
CONF_LOW = 0.18
SIM_LOW  = 0.12

MOTS_MG_CORE = {
    'aho', 'izaho', 'ianao', 'izy', 'ny', 'misy', 'tsy', 'efa', 'mbola',
    'marary', 'manaintaina', 'mafana', 'tazo', 'kohaka', 'mandrevo',
    'loha', 'kibo', 'maso', 'tenda', 'tratra', 'tongotra', 'tanana',
    'vatana', 'fo', 'nify', 'dokotera', 'fanafody', 'aretina',
    'tazomoka', 'diabeta', 'salama', 'veloma', 'misaotra',
}

MOTS_FR_CORE = {
    'je', 'tu', 'il', 'elle', 'nous', 'vous', 'ils', 'elles',
    'bonjour', 'bonsoir', 'merci', 'oui', 'non', 'est', 'suis',
    'avoir', 'faire', 'aller', 'mal', 'douleur', 'médecin', 'maladie',
}

MOTS_EN_CORE = {
    'i', 'you', 'he', 'she', 'we', 'they', 'is', 'are', 'have', 'has',
    'hello', 'hi', 'yes', 'no', 'pain', 'sick', 'doctor', 'fever',
    'headache', 'medicine', 'help', 'feel', 'need',
}

MOTS_HORS_CORE = {
    'fiara', 'moto', 'bus', 'taxi', 'vola', 'ariary', 'lalao',
    'football', 'cinema', 'facebook', 'internet', 'wifi', 'solosaina',
    'sekoly', 'oniversite', 'politika', 'fifidianana',
}

_ARTEFACTS = [
    "pipeline.pkl",
    "encoder.pkl",
    "retriever_tfidf.pkl",
    "dataset_clean.csv",
]

def _download_models_if_needed(model_dir: str, repo_id: str) -> None:

    if not _HF_OK:
        print("huggingface_hub non installé — pip install huggingface_hub")
        return

    os.makedirs(model_dir, exist_ok=True)

    for fichier in _ARTEFACTS:
        dest = os.path.join(model_dir, fichier)
        if not os.path.exists(dest):
            print(f"⬇Téléchargement {fichier} depuis HuggingFace...")
            try:
                hf_hub_download(
                    repo_id   = repo_id,
                    filename  = fichier,
                    local_dir = model_dir,
                )
                print(f"{fichier} prêt")
            except Exception as e:
                print(f"Erreur téléchargement {fichier} : {e}")
        else:
            print(f"{fichier} déjà présent")

class MalagasyTokenizer:
    SYNONYMES = {
        'marary':   ['manaintaina', 'mangirifiry', 'fanaintainana'],
        'loha':     ['lohako', 'lohany'],
        'kibo':     ['kibony', 'kiboko'],
        'maso':     ['masoko', 'masony'],
        'tratra':   ['tratrako'],
        'tenda':    ['tendako', 'tendany', 'marary tenda'],
        'tongotra': ['tongotro', 'lohalika', 'ratsan-tongotra'],
        'tanana':   ['tananako', 'ankihibe', 'ratsan-tanana'],
        'tazo':     ['manavy', 'mafana', 'mamay'],
        'tazomoka': ['tazo avy amin ny kaikitry ny moka'],
        'kohaka':   ['mikohaka'],
        'pipi':     ['mamany', 'mipipi'],
        'foko':     ['fo'],
        'diabeta':  ['fiakaran ny siramamy', 'fidinan ny siramamy'],
        'tsindry':  ['tosi-dra', 'hypertension'],
        'saiko':    ['saina', 'kivy'],
        'mandrevo': ['mivalanana', 'mivalan-tay'],
        'nify':     ['nifiko', 'nifinao'],
    }

    def clean(self, text: str) -> str:
        if not isinstance(text, str):
            return ''
        t = text.lower()
        t = unicodedata.normalize('NFKD', t)
        t = ''.join(c for c in t if not unicodedata.combining(c))
        t = t.replace("'", "'").replace('`', "'")
        t = re.sub(r"[^\w\s'\-]", ' ', t)
        return re.sub(r'\s+', ' ', t).strip()

    def tokenize(self, text: str) -> list[str]:
        tokens = []
        for t in self.clean(text).split():
            if len(t) > 1:
                tokens.append(t)
                if '-' in t:
                    tokens.extend([p for p in t.split('-') if len(p) > 1])
        return tokens

    def expand(self, text: str) -> str:
        tokens = self.tokenize(text)
        extra  = []
        for tok in tokens:
            for key, syns in self.SYNONYMES.items():
                if tok == key or tok in syns:
                    extra.extend([s for s in [key] + syns if s != tok])
        return ' '.join(tokens + extra)

    def for_tfidf(self, text: str) -> str:
        return self.clean(self.expand(text))

class HybridRetriever:
    def __init__(self, df, tok: MalagasyTokenizer, tfidf_vec):
        self.df    = df.reset_index(drop=True)
        self.tok   = tok
        corpus_tok = [tok.tokenize(str(t)) for t in df['texte']]
        self.bm25  = BM25Okapi(corpus_tok)
        self.vec   = tfidf_vec
        self.mat   = self.vec.transform(
            [tok.clean(str(t)) for t in df['texte']]
        )

    def get(self, query: str, category: str | None = None):
        qc     = self.tok.clean(self.tok.expand(query))
        bm25_s = np.array(self.bm25.get_scores(qc.split()))
        mx     = bm25_s.max()
        if mx > 0:
            bm25_s /= mx
        cos_s = cosine_similarity(self.vec.transform([qc]), self.mat).flatten()
        score = 0.45 * bm25_s + 0.55 * cos_s

        if category:
            mask = (self.df['cat_base'] == category).values
            if mask.sum() > 0:
                s2        = score.copy()
                s2[~mask] *= 0.25
                idx       = int(s2.argmax())
            else:
                idx = int(score.argmax())
        else:
            idx = int(score.argmax())

        return self.df.iloc[idx], float(score[idx])

def _vowel_ratio(word: str) -> float:
    if not word:
        return 0.0
    return sum(1 for c in word if c in VOYELLES) / len(word)


def _max_cons_run(word: str) -> int:
    run = mx = 0
    for c in word:
        if c.isalpha() and c not in VOYELLES:
            run += 1
            mx   = max(mx, run)
        else:
            run  = 0
    return mx


def _entropy(word: str) -> float:
    if not word:
        return 0.0
    c = Counter(word)
    l = len(word)
    return -sum((v / l) * math.log2(v / l) for v in c.values())


def _is_gib_word(w: str) -> bool:
    w = w.lower()
    if len(w) < 4:
        return False
    vr = _vowel_ratio(w)
    cr = _max_cons_run(w)
    if vr == 0.0 and len(w) >= 2:
        return True
    if vr < 0.30 and cr >= 2:
        return True
    if _entropy(w) > 3.2 and len(w) > 2:
        return True
    if cr >= 2:
        return True
    return False


def is_gibberish(text: str) -> bool:
    words = re.findall(r'[a-zA-Z]{4,}', text.lower())
    if not words:
        return False
    n_gib = sum(1 for w in words if _is_gib_word(w))
    return n_gib / len(words) > 0.50


def detect_language(text: str) -> str | None:
    if re.search(r'[\u0400-\u04FF\u0600-\u06FF\u4E00-\u9FFF\u3040-\u30FF]', text):
        return 'autre'
    words = set(re.findall(r'[a-zA-Z]+', text.lower()))
    if not words:
        return None
    if words & MOTS_MG_CORE:
        return None
    total   = max(len(words), 1)
    hits_fr = len(words & MOTS_FR_CORE)
    hits_en = len(words & MOTS_EN_CORE)
    if hits_fr / total > 0.25:
        return 'francais'
    if hits_en / total > 0.25:
        return 'anglais'
    if total >= 3 and not (words & MOTS_MG_CORE):
        return 'autre'
    return None


def is_hors_domaine(text: str) -> bool:
    words = set(text.lower().split())
    if words & MOTS_HORS_CORE:
        return True
    if len(words) > 2 and not (words & MOTS_MG_CORE):
        return True
    return False


def detect_salutation(text: str) -> str | None:
    t = text.lower().strip()
    if t in SALUTATIONS:
        return SALUTATIONS[t]
    for key in sorted(SALUTATIONS, key=len, reverse=True):
        if len(key) > 3 and (t == key or t.startswith(key) or key in t):
            return SALUTATIONS[key]
    return None


def check_grave(text: str) -> str | None:
    t = text.lower()
    return MSG_URGENCE if any(s in t for s in SYMPTOMES_GRAVES) else None

def build_response(category: str, med1: str, med2: str, astuce: str) -> str:
    def ok(v):
        v = str(v).strip()
        return v if v and v.lower() not in ('nan', 'none', '') else ''

    m1  = ok(med1)
    m2  = ok(med2)
    a   = ok(astuce)
    nom = NOMS_CAT.get(category, category)

    parties = []
    if m1:
        parties.append(f"Raha {nom} no mahazo anao dia mila mihinana ny fanafody {m1}.")
    else:
        parties.append(f"Raha {nom} no mahazo anao, tsara raha mandehana dokotera.")
    if m2:
        parties.append(f"Raha tsy sitrana, dia andramo indray ity fanafody ity {m2}.")
    if a:
        parties.append(f"Ary ity ny torohevitra kely omeko anao: {a}.")
    parties.append("Ka raha tena tsy mijanona ny aretina, mitsangana mandehana dokotera.")
    return ' '.join(parties)


def _indicator(conf: float) -> str:
    if conf >= CONF_MED:
        return "green"
    if conf >= CONF_LOW:
        return "yellow"
    return "red"

class ChatService:

    _instance: "ChatService | None" = None

    def __init__(self):
        self._ready     = False
        self._pipeline  = None
        self._le        = None
        self._retriever: HybridRetriever | None = None
        self._tok       = MalagasyTokenizer()
        self._idx2cat:  dict[int, str] = {}

        model_dir = os.getenv("CHATBOT_MODEL_DIR", "utils")
        repo_id   = os.getenv("HF_REPO_ID", "")

        if repo_id:
            _download_models_if_needed(model_dir, repo_id)

        path    = Path(model_dir)
        missing = [f for f in _ARTEFACTS if not (path / f).exists()]

        if missing:
            print(f"Artefacts manquants {missing} — mode fallback")
            return

        if not _DEPS_OK:
            print("Dépendances ML manquantes (pandas/rank_bm25) — mode fallback")
            return

        try:
            with open(path / "pipeline.pkl",        "rb") as f:
                self._pipeline = pickle.load(f)
            with open(path / "encoder.pkl",         "rb") as f:
                self._le       = pickle.load(f)
            with open(path / "retriever_tfidf.pkl", "rb") as f:
                tfidf_vec      = pickle.load(f)

            df = pd.read_csv(path / "dataset_clean.csv", encoding="utf-8")
            for col in ["texte", "medicament1", "medicament2", "astuce", "cat_base"]:
                if col not in df.columns:
                    df[col] = ""
                df[col] = df[col].fillna("")
            df = df.drop_duplicates(subset=["texte"]).reset_index(drop=True)

            self._retriever = HybridRetriever(df, self._tok, tfidf_vec)
            self._idx2cat   = {i: c for i, c in enumerate(self._le.classes_)}
            self._ready     = True
            print(f"ChatService prêt — {len(df)} docs, {len(self._le.classes_)} classes")

        except Exception as exc:
            print(f"Erreur chargement modèle : {exc} — mode fallback")

    @classmethod
    def get(cls) -> "ChatService":
        if cls._instance is None:
            cls._instance = cls()
        return cls._instance

    def chat(self, user_input: str, session_last_cat: str | None = None) -> dict:

        ui = (user_input or "").strip()

        if not ui:
            return self._special("empty", "Mba soraty mazava tompoko oh ...")
        if len(ui) < 5:
            return self._special("too_short", "Mba omeo fanazavana fanampiny.")

        sal = detect_salutation(ui)
        if sal:
            return {"type": "salutation", "response": sal}

        if is_gibberish(ui):
            return self._special("gibberish", random.choice(MSGS_GIB), ood=True)

        lang = detect_language(ui)
        if lang:
            return self._special("langue", MSG_LANGUE, ood=True)

        if is_hors_domaine(ui):
            return self._special("hors_domaine", random.choice(MSGS_HORS), ood=True)

        alerte = check_grave(ui)

        if not self._ready:
            fb = (alerte + "\n\n" if alerte else "") + random.choice(MSGS_HORS)
            return self._special("fallback", fb, alerte=alerte, ood=True)

        proc  = self._tok.for_tfidf(ui)
        probs = self._pipeline.predict_proba([proc])[0]
        idx   = int(probs.argmax())
        cat   = self._idx2cat[idx]
        conf  = float(probs[idx])

        if (conf < CONF_MED
                and session_last_cat
                and session_last_cat not in ("gibberish", "hors_domaine", "langue")):
            cat  = session_last_cat
            conf = max(conf, CONF_MED)

        top_idx = probs.argsort()[-3:][::-1]
        top3    = [
            {
                "categorie": self._idx2cat[i],
                "icon":      ICONES.get(self._idx2cat[i], "🏥"),
                "score":     round(float(probs[i]), 3),
            }
            for i in top_idx
        ]

        row, sim = self._retriever.get(ui, category=cat)

        if conf < CONF_LOW and sim < SIM_LOW and not alerte:
            return self._special("hors_domaine", random.choice(MSGS_HORS), ood=True)

        m1     = str(row.get("medicament1", "")).strip()
        m2     = str(row.get("medicament2", "")).strip()
        astuce = str(row.get("astuce", "")).strip()

        m1     = "" if m1.lower()     in ("nan", "none") else m1
        m2     = "" if m2.lower()     in ("nan", "none") else m2
        astuce = "" if astuce.lower() in ("nan", "none") else astuce

        generated = build_response(cat, m1, m2, astuce)
        icon      = ICONES.get(cat, "🏥")
        label_fr  = NOMS_CAT.get(cat, cat)
        indicator = _indicator(conf)

        return {
            "type":        "medical",
            "categorie":   cat,
            "label_fr":    label_fr,
            "icon":        icon,
            "confidence":  round(conf * 100, 1),
            "indicator":   indicator,
            "tfidf_sim":   round(sim, 4),
            "medicament1": m1     or None,
            "medicament2": m2     or None,
            "astuce":      astuce or None,
            "generated":   (alerte + "\n\n" + generated) if alerte else generated,
            "top3":        top3,
            "alerte":      alerte,
            "fallback":    None,
            "ood":         False,
        }

    @staticmethod
    def _special(
        type_: str,
        generated: str,
        *,
        alerte: str | None   = None,
        fallback: str | None = None,
        ood: bool            = False,
    ) -> dict:
        return {
            "type":        type_,
            "categorie":   None,
            "label_fr":    None,
            "icon":        None,
            "confidence":  None,
            "indicator":   None,
            "tfidf_sim":   None,
            "medicament1": None,
            "medicament2": None,
            "astuce":      None,
            "generated":   generated,
            "top3":        [],
            "alerte":      alerte,
            "fallback":    fallback,
            "ood":         ood,
        }