""" Fetches hot/new posts from crypto subreddits via PRAW (free Reddit API). Falls back to a keyless Reddit JSON API if credentials are not set. """ import os import logging import time import json import requests from datetime import datetime, timezone from typing import Any from config import SUBREDDITS, REDDIT_POST_LIMIT, REDDIT_COMMENT_LIMIT logger = logging.getLogger(__name__) _REDDIT_JSON_HEADERS = { "User-Agent": "CryptoNarrativeBot/1.0" } class RedditCollector: def __init__(self): self._praw = self._init_praw() def _init_praw(self) -> Any | None: client_id = os.getenv("REDDIT_CLIENT_ID", "") client_secret = os.getenv("REDDIT_CLIENT_SECRET", "") user_agent = os.getenv("REDDIT_USER_AGENT", "CryptoNarrativeBot/1.0") if not (client_id and client_secret): logger.info("[Reddit] No PRAW credentials found — using keyless JSON API fallback") return None try: import praw reddit = praw.Reddit( client_id=client_id, client_secret=client_secret, user_agent=user_agent, read_only=True, ) # test connection _ = reddit.subreddit("CryptoCurrency").id logger.info("[Reddit] PRAW authenticated successfully") return reddit except Exception as exc: logger.warning(f"[Reddit] PRAW init failed ({exc}) — falling back to JSON API") return None def fetch_all(self) -> list[dict]: posts: list[dict] = [] for sub in SUBREDDITS: try: fetched = ( self._fetch_praw(sub) if self._praw else self._fetch_json(sub) ) posts.extend(fetched) logger.debug(f"[Reddit] r/{sub}: {len(fetched)} posts") time.sleep(0.5) # be polite except Exception as exc: logger.warning(f"[Reddit] Failed r/{sub}: {exc}") return posts def _fetch_praw(self, subreddit: str) -> list[dict]: sub = self._praw.subreddit(subreddit) results: list[dict] = [] for post in sub.hot(limit=REDDIT_POST_LIMIT): top_comments = self._get_praw_comments(post) results.append(self._normalize_praw(post, subreddit, top_comments)) return results def _get_praw_comments(self, post) -> list[str]: try: post.comments.replace_more(limit=0) return [ c.body for c in post.comments.list()[:REDDIT_COMMENT_LIMIT] if hasattr(c, "body") ] except Exception: return [] def _normalize_praw(self, post, subreddit: str, comments: list[str]) -> dict: published = datetime.fromtimestamp(post.created_utc, tz=timezone.utc) text = f"{post.title}. {post.selftext or ''}. Comments: {' '.join(comments)}" return { "source": f"Reddit/r/{subreddit}", "title": post.title, "summary": (post.selftext or "")[:500], "url": f"https://reddit.com{post.permalink}", "published": published.isoformat(), "type": "social", "score": post.score, "upvote_ratio": post.upvote_ratio, "num_comments": post.num_comments, "text": text.strip(), } def _fetch_json(self, subreddit: str) -> list[dict]: """Keyless Reddit JSON API fallback — 60 req/min allowed.""" url = f"https://www.reddit.com/r/{subreddit}/hot.json?limit={REDDIT_POST_LIMIT}" resp = requests.get(url, headers=_REDDIT_JSON_HEADERS, timeout=10) resp.raise_for_status() data = resp.json() posts = data.get("data", {}).get("children", []) results: list[dict] = [] for item in posts: p = item.get("data", {}) published = datetime.fromtimestamp( p.get("created_utc", time.time()), tz=timezone.utc ) title = p.get("title", "") selftext = p.get("selftext", "") results.append({ "source": f"Reddit/r/{subreddit}", "title": title, "summary": selftext[:500], "url": f"https://reddit.com{p.get('permalink', '')}", "published": published.isoformat(), "type": "social", "score": p.get("score", 0), "upvote_ratio": p.get("upvote_ratio", 0), "num_comments": p.get("num_comments", 0), "text": f"{title}. {selftext}".strip(), }) return results