""" Google Trends "search interest" — a free retail-attention signal. Uses Google's undocumented Trends JSON endpoints directly (no pytrends/pandas dependency). Google frequently rate-limits/blocks datacenter IPs, so this is best-effort: on ANY failure it returns the last cached value (or {}), and the pipeline continues unaffected. Results are cached for ~1h to avoid rate limits. """ import json import time import logging import requests from urllib.parse import quote logger = logging.getLogger(__name__) _EXPLORE = "https://trends.google.com/trends/api/explore" _MULTILINE = "https://trends.google.com/trends/api/widgetdata/multiline" _HEADERS = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120 Safari/537.36", "Accept-Language": "en-US,en;q=0.9"} _cache: dict = {} _last_fetch = 0.0 _MIN_INTERVAL = 3600 # 1h def _strip(txt: str): # Google prefixes JSON with )]}' — drop the first line. i = txt.find("{") return json.loads(txt[i:]) if i >= 0 else None def _interest(keyword: str, timeframe="now 7-d", geo="") -> int | None: """Latest search-interest value (0-100) for one keyword, or None.""" s = requests.Session(); s.headers.update(_HEADERS) req = {"comparisonItem": [{"keyword": keyword, "geo": geo, "time": timeframe}], "category": 0, "property": ""} r = s.get(_EXPLORE, params={"hl": "en-US", "tz": "0", "req": json.dumps(req)}, timeout=12) if r.status_code != 200: raise RuntimeError(f"explore {r.status_code}") widgets = _strip(r.text).get("widgets", []) w = next((w for w in widgets if w.get("id") == "TIMESERIES"), None) if not w: raise RuntimeError("no timeseries widget") r2 = s.get(_MULTILINE, params={"hl": "en-US", "tz": "0", "req": json.dumps(w["request"]), "token": w["token"]}, timeout=12) if r2.status_code != 200: raise RuntimeError(f"multiline {r2.status_code}") vals = [pt["value"][0] for pt in _strip(r2.text).get("default", {}).get("timelineData", []) if pt.get("value")] return vals[-1] if vals else None def fetch_trends(keywords: list[str]) -> dict: """{keyword: interest 0-100}. Cached ~1h; best-effort (never raises).""" global _last_fetch now = time.time() if _cache and now - _last_fetch < _MIN_INTERVAL: return _cache out = {} try: for kw in keywords[:5]: # keep it light try: v = _interest(kw) if v is not None: out[kw] = v time.sleep(1.0) # be gentle except Exception as exc: logger.info(f"[Trends] '{kw}' failed: {exc}") if out: _cache.clear(); _cache.update(out); _last_fetch = now except Exception as exc: logger.info(f"[Trends] fetch failed: {exc}") return _cache or out