""" Fetches articles from RSS/Atom feeds. No API key required. Falls back gracefully if a feed is unreachable. """ import feedparser import time import logging from datetime import datetime, timezone from typing import Any from config import RSS_FEEDS, MAX_ARTICLE_AGE_HOURS logger = logging.getLogger(__name__) def _parse_published(entry: Any) -> datetime: """Return a timezone-aware UTC datetime from a feedparser entry.""" if hasattr(entry, "published_parsed") and entry.published_parsed: return datetime(*entry.published_parsed[:6], tzinfo=timezone.utc) if hasattr(entry, "updated_parsed") and entry.updated_parsed: return datetime(*entry.updated_parsed[:6], tzinfo=timezone.utc) return datetime.now(timezone.utc) class RSSCollector: def __init__(self, feeds: dict[str, str] | None = None): # Crypto feeds + macro feeds (macro tagged is_macro=True) from config import MACRO_FEEDS self.feeds = feeds or RSS_FEEDS self.macro_feeds = MACRO_FEEDS def fetch_all(self) -> list[dict]: """Fetch articles from all configured RSS feeds (crypto + macro).""" articles: list[dict] = [] from monitoring import monitor for source, url in {**self.feeds, **self.macro_feeds}.items(): is_macro = source.startswith("Macro") try: fetched = self._fetch_feed(source, url, is_macro=is_macro) articles.extend(fetched) if fetched: monitor.source_ok(source, len(fetched)) else: monitor.source_failure(source, "empty response") logger.debug(f"[RSS] {source}: {len(fetched)} articles") except Exception as exc: monitor.source_failure(source, str(exc)[:120]) logger.warning(f"[RSS] Failed to fetch {source}: {exc}") return articles def _fetch_feed(self, source: str, url: str, is_macro: bool = False) -> list[dict]: feed = feedparser.parse(url) cutoff = datetime.now(timezone.utc) results: list[dict] = [] for entry in feed.entries: published = _parse_published(entry) age_hours = (cutoff - published).total_seconds() / 3600 if age_hours > MAX_ARTICLE_AGE_HOURS: continue title = entry.get("title", "").strip() summary = entry.get("summary", entry.get("description", "")).strip() # strip HTML tags from summary cheaply import re summary = re.sub(r"<[^>]+>", " ", summary).strip() results.append({ "source": source, "title": title, "summary": summary, "url": entry.get("link", ""), "published": published.isoformat(), "type": "macro" if is_macro else "news", "is_macro": is_macro, "score": 0, # raw engagement score (N/A for RSS) "text": f"{title}. {summary}", }) return results