"""
Deduplicate articles that describe the same story across different sources.

The same headline often arrives from several feeds — e.g. an RSS outlet and a
Google News aggregator (which also appends " - Publisher" to titles). Counting
it once keeps the feed clean AND prevents one story from being double-counted
in the sentiment aggregate.

Strategy:
  1. Drop exact duplicate URLs.
  2. Normalize titles (strip " - Publisher" suffix, punctuation, casing) and
     keep only the first occurrence — preferring the most credible source.
"""

import re

from config import SOURCE_WEIGHTS

_PUBLISHER_SUFFIX = re.compile(r"\s[-–—|]\s[^-–—|]{2,40}$")   # " - The Block"
_NON_ALNUM = re.compile(r"[^a-z0-9 ]+")
_WS = re.compile(r"\s+")


def _source_rank(source: str) -> float:
    s = (source or "").lower()
    for key, weight in SOURCE_WEIGHTS.items():
        if key.lower() in s:
            return weight
    return SOURCE_WEIGHTS.get("_default", 1.0)


def _normalize_title(title: str) -> str:
    t = (title or "").strip()
    # strip a trailing " - Publisher" that aggregators add
    prev = None
    while prev != t:
        prev = t
        t = _PUBLISHER_SUFFIX.sub("", t).strip()
    t = t.lower()
    t = _NON_ALNUM.sub(" ", t)
    t = _WS.sub(" ", t).strip()
    return t


def cluster_consensus(items: list[dict]) -> dict[str, dict]:
    """
    #3 Contradiction / disagreement detection.

    Groups items by normalized title (same story, different sources) and, for
    each cluster, measures how much the sources AGREE on sentiment:
      - sentiment_variance  : variance of per-source compound scores
      - consensus_strength  : 1 - (stdev / max_stdev), 1.0 = unanimous

    Returns {normalized_title: {cluster_size, sentiment_variance,
    consensus_strength, urls:[...]}}. Call AFTER sentiment scoring.
    """
    clusters: dict[str, list[dict]] = {}
    for it in items:
        key = _normalize_title(it.get("title", ""))
        if not key:
            continue
        clusters.setdefault(key, []).append(it)

    out: dict[str, dict] = {}
    for key, members in clusters.items():
        scores = [m.get("sentiment", {}).get("compound", 0.0) for m in members]
        n = len(scores)
        if n == 0:
            continue
        mean = sum(scores) / n
        variance = sum((x - mean) ** 2 for x in scores) / n
        stdev = variance ** 0.5
        # compound is in [-1,1] so the max possible stdev is ~1.0
        strength = round(max(0.0, 1.0 - min(stdev, 1.0)), 4)
        out[key] = {
            "cluster_size":       n,
            "sentiment_variance": round(variance, 4),
            "consensus_strength": strength,
            "urls":               [m.get("url", "") for m in members],
        }
    return out


def dedupe_articles(items: list[dict]) -> list[dict]:
    """Returns a deduplicated list, preserving the highest-credibility copy."""
    seen_urls: set[str] = set()
    by_title: dict[str, int] = {}   # normalized title -> index in result
    result: list[dict] = []

    for item in items:
        url = (item.get("url") or "").strip()
        if url and url in seen_urls:
            continue

        key = _normalize_title(item.get("title", ""))
        if not key:
            # no usable title — keep it, can't compare
            if url:
                seen_urls.add(url)
            result.append(item)
            continue

        if key in by_title:
            # duplicate story — keep whichever has the stronger source
            existing = result[by_title[key]]
            if _source_rank(item.get("source", "")) > _source_rank(existing.get("source", "")):
                result[by_title[key]] = item
            continue

        by_title[key] = len(result)
        if url:
            seen_urls.add(url)
        result.append(item)

    return result