feat: complete OpenAlex integration as 4th discovery source (search + download + discipline mapping)

app/api/routes/analysis.py

@@ -17,6 +17,7 @@ from app.services.k2_think_engine import K2ThinkEngine
 from app.models.schemas import ScientificDocument, AnalysisRequest as K2AnalysisRequest, DocumentType, ChatRequest, ChatResponse
 from app.services.export_service import ExportService
 from app.services.arxiv_service import ArXivService
+from app.services.openalex_service import OpenAlexService
 from fastapi.responses import Response, FileResponse
 
 router = APIRouter()
@@ -183,6 +184,27 @@ async def get_specific_analysis(
                         except Exception as pubmed_err:
                             logger.error(f"DEMO_REAL: PubMed download failed for {pid}: {pubmed_err}")
                             continue
+                    elif pid.startswith("openalex_"):
+                        logger.info(f"DEMO_REAL: {pid} is OpenAlex. Attempting fetch and download...")
+                        try:
+                            openalex_service = OpenAlexService(download_dir=UPLOAD_DIR)
+                            # Fetch paper metadata to get the OA PDF URL
+                            # Extract the raw OpenAlex Work ID (e.g. W2987984220)
+                            clean_id = pid.replace("openalex_", "")
+                            search_results = openalex_service.fetch_papers(clean_id, max_results=1)
+                            if search_results and search_results[0].get("url"):
+                                file_to_process = openalex_service.download_paper(
+                                    search_results[0]["url"], pid
+                                )
+                                if not file_to_process:
+                                    logger.error(f"DEMO_REAL: OpenAlex download returned empty path for {pid}")
+                                    continue
+                            else:
+                                logger.error(f"DEMO_REAL: No OA PDF URL found for OpenAlex {pid}")
+                                continue
+                        except Exception as oa_err:
+                            logger.error(f"DEMO_REAL: OpenAlex download failed for {pid}: {oa_err}")
+                            continue
                     else:
                         # ArXiv ID (Standard)
                         logger.info(f"DEMO_REAL: {pid} not found locally. Attempting on-demand download...")

app/api/routes/discovery.py

@@ -1,5 +1,5 @@
 """
-Discovery API Routes
+Discovery API Routes — unified search across ArXiv, DOAJ, PubMed, and OpenAlex.
 """
 from fastapi import APIRouter, Depends, HTTPException
 from sqlalchemy.orm import Session
@@ -9,6 +9,7 @@ from app.dependencies import get_db
 from app.services.arxiv_service import ArXivService
 from app.services.doaj_service import DOAJService
 from app.services.pubmed_service import PubMedService
+from app.services.openalex_service import OpenAlexService
 from app.services.analysis_service import AnalysisService
 from app.core.logging import logger
 import os
@@ -28,135 +29,166 @@ class DiscoveryResponse(BaseModel):
 def map_to_discipline(paper: dict) -> str:
     """
     Maps a paper to one of the 5 target disciplines based on its metadata.
+    Handles: ArXiv (category codes), PubMed (source), DOAJ (subjects), OpenAlex (topics).
     """
     source = paper.get("source", "")
     sources = paper.get("sources", [source])
-    categories = paper.get("categories", [])
+    categories = paper.get("categories", [])   # ArXiv codes or topic names
     title = paper.get("title", "").lower()
     summary = paper.get("summary", "").lower()
 
-    # Priority 1: Life Sciences & Medicine (PubMed always maps here)
+    # Lowercase join of all category/topic labels for keyword matching
+    cats_lower = " ".join(c.lower() for c in categories)
+
+    # Priority 1: Life Sciences & Medicine
     if "PubMed" in sources:
         return "Life Sciences & Medicine"
+    life_keywords = ["medicine", "biology", "clinical", "health", "disease", "genomics",
+                     "pharmaceutical", "neuroscience", "biomedical", "cancer", "patient"]
+    if any(k in cats_lower or k in title or k in summary for k in life_keywords):
+        return "Life Sciences & Medicine"
 
     # Priority 2: Computer Science & AI
-    cs_keywords = ["computer science", "neural network", "deep learning", "artificial intelligence", "algorithms"]
-    if any(c.startswith("cs.") for c in categories) or any(k in title or k in summary for k in cs_keywords):
+    cs_arxiv = any(c.startswith("cs.") for c in categories)
+    cs_keywords = ["computer science", "machine learning", "deep learning", "neural network",
+                   "artificial intelligence", "algorithm", "natural language", "computer vision",
+                   "reinforcement learning", "large language model"]
+    if cs_arxiv or any(k in cats_lower or k in title or k in summary for k in cs_keywords):
         return "Computer Science & AI"
 
     # Priority 3: Physics & Mathematics
-    math_phys_categories = ["math.", "phys.", "stat.", "astro-ph.", "quant-ph.", "nlin."]
-    if any(any(c.startswith(mp) for mp in math_phys_categories) for c in categories):
+    math_phys_arxiv = ["math.", "phys.", "stat.", "astro-ph.", "quant-ph.", "nlin.", "gr-qc.", "hep-"]
+    math_phys_keywords = ["mathematics", "physics", "quantum", "statistics", "astrophysics",
+                           "thermodynamics", "algebra", "topology", "calculus"]
+    if (any(any(c.startswith(mp) for mp in math_phys_arxiv) for c in categories)
+            or any(k in cats_lower or k in title or k in summary for k in math_phys_keywords)):
         return "Physics & Mathematics"
 
     # Priority 4: Engineering
-    eng_categories = ["eess."]
-    eng_keywords = ["engineering", "electronics", "circuits", "hardware", "material science"]
-    if any(any(c.startswith(ec) for ec in eng_categories) for c in categories) or any(k in title or k in summary for k in eng_keywords):
+    eng_keywords = ["engineering", "electronics", "circuits", "hardware", "material science",
+                    "robotics", "signal processing", "nanotechnology", "semiconductor"]
+    if (any(c.startswith("eess.") for c in categories)
+            or any(k in cats_lower or k in title or k in summary for k in eng_keywords)):
         return "Engineering"
 
     # Priority 5: Social Sciences
-    soc_categories = ["econ.", "q-fin."]
-    soc_keywords = ["sociology", "psychology", "economics", "philosophy", "social", "humanities"]
-    if any(any(c.startswith(sc) for sc in soc_categories) for c in categories) or any(k in title or k in summary for k in soc_keywords):
+    soc_keywords = ["sociology", "psychology", "economics", "philosophy", "social science",
+                    "humanities", "political science", "anthropology", "education", "law"]
+    if (any(c.startswith(("econ.", "q-fin.")) for c in categories)
+            or any(k in cats_lower or k in title or k in summary for k in soc_keywords)):
         return "Social Sciences"
 
-    # Fallback based on DOAJ subjects if still missing
-    doaj_subjects = [s.lower() for s in categories] if source == "DOAJ" else []
-    if any("medicine" in s or "biology" in s for s in doaj_subjects): return "Life Sciences & Medicine"
-    if any("social" in s or "law" in s or "education" in s for s in doaj_subjects): return "Social Sciences"
-    if any("mathematics" in s or "physics" in s for s in doaj_subjects): return "Physics & Mathematics"
-    if any("technology" in s or "engineering" in s for s in doaj_subjects): return "Engineering"
-    if any("computer" in s for s in doaj_subjects): return "Computer Science & AI"
-
     return "General Science"
 
 @router.post("/search", response_model=DiscoveryResponse)
 async def discovery_search(request: SearchRequest, db: Session = Depends(get_db)):
     """
-    Search ArXiv, DOAJ, and PubMed and return merged metadata for browsing.
+    Search ArXiv, DOAJ, PubMed, and OpenAlex — returns merged, deduplicated metadata.
     """
+    import re
     try:
         # 1. Fetch from ArXiv
         arxiv_service = ArXivService(download_dir="./uploaded_files")
         arxiv_data = arxiv_service.fetch_papers(request.query, request.max_results)
-        
+        for p in arxiv_data:
+            p["source"] = "ArXiv"
+            p["has_pdf"] = True  # ArXiv always has PDF
+
         # 2. Fetch from DOAJ
         doaj_service = DOAJService(download_dir="./uploaded_files")
         doaj_data = doaj_service.fetch_papers(request.query, request.max_results)
-        
+
         # 3. Fetch from PubMed
         pubmed_service = PubMedService(download_dir="./uploaded_files")
         pubmed_data = pubmed_service.fetch_papers(request.query, request.max_results)
-        
-        # 4. Merge and deduplicate by title
-        seen_titles = {}
-        merged_results = []
-        
-        all_papers = []
-        for p in arxiv_data:
-            p["source"] = "ArXiv"
-            all_papers.append(p)
-        for p in doaj_data:
-            all_papers.append(p)
-        for p in pubmed_data:
-            all_papers.append(p)
-            
+
+        # 4. Fetch from OpenAlex (only open-access works)
+        openalex_service = OpenAlexService(download_dir="./uploaded_files")
+        openalex_data = openalex_service.fetch_papers(request.query, request.max_results)
+
+        # 5. Merge and deduplicate by normalized title
+        seen_titles: dict = {}
+        merged_results: list = []
+
+        all_papers = arxiv_data + doaj_data + pubmed_data + openalex_data
+
         for paper in all_papers:
-            import re
-            norm_title = re.sub(r'\W+', '', paper["title"].lower())
-            
+            source = paper.get("source", "Unknown")
+            norm_title = re.sub(r'\W+', '', paper.get("title", "").lower())
+            if not norm_title:
+                continue
+
             if norm_title in seen_titles:
+                # Merge into existing entry
                 existing_index = seen_titles[norm_title]
-                if paper["source"] not in merged_results[existing_index]["sources"]:
-                    merged_results[existing_index]["sources"].append(paper["source"])
-                if paper.get("has_pdf") and not merged_results[existing_index].get("has_pdf"):
-                    merged_results[existing_index]["url"] = paper["url"]
-                    merged_results[existing_index]["has_pdf"] = True
-                # Merge categories
-                if "categories" in paper:
-                    current_cats = merged_results[existing_index].get("raw_categories", [])
-                    merged_results[existing_index]["raw_categories"] = list(set(current_cats + paper["categories"]))
+                existing = merged_results[existing_index]
+
+                if source not in existing["sources"]:
+                    existing["sources"].append(source)
+
+                # Upgrade to direct PDF URL if current entry doesn't have one
+                if paper.get("has_pdf") and not existing.get("has_pdf"):
+                    existing["url"] = paper["url"]
+                    existing["has_pdf"] = True
+
+                # Merge categories/topics
+                current_cats = existing.get("raw_categories", [])
+                new_cats = paper.get("categories", [])
+                existing["raw_categories"] = list(dict.fromkeys(current_cats + new_cats))
+
+                # Prefer longer/richer abstract
+                if len(paper.get("summary", "")) > len(existing.get("summary", "")):
+                    existing["summary"] = paper["summary"]
+
+                # Carry cited_by_count if OpenAlex provides it
+                if paper.get("cited_by_count") and not existing.get("cited_by_count"):
+                    existing["cited_by_count"] = paper["cited_by_count"]
             else:
-                paper["sources"] = [paper["source"]]
-                if paper["source"] == "ArXiv":
-                    paper["has_pdf"] = True
-                
                 seen_titles[norm_title] = len(merged_results)
                 merged_results.append({
-                    "title": paper["title"],
                     "id": paper["id"],
-                    "authors": paper["authors"],
-                    "summary": paper["summary"],
-                    "publication_date": paper["publication_date"],
+                    "title": paper["title"],
+                    "authors": paper.get("authors", []),
+                    "summary": paper.get("summary", "No abstract available."),
+                    "publication_date": paper.get("publication_date", "n.d."),
                     "url": paper.get("url"),
-                    "sources": paper["sources"],
                     "has_pdf": paper.get("has_pdf", False),
-                    "raw_categories": paper.get("categories", [])
+                    "sources": [source],
+                    "raw_categories": paper.get("categories", []),
+                    "cited_by_count": paper.get("cited_by_count", 0),
                 })
-        
-        # 5. Assign disciplines
-        for i in range(len(merged_results)):
-            paper_for_mapping = {
-                "sources": merged_results[i]["sources"],
-                "categories": merged_results[i].get("raw_categories", []),
-                "title": merged_results[i]["title"],
-                "summary": merged_results[i]["summary"]
-            }
-            merged_results[i]["discipline"] = map_to_discipline(paper_for_mapping)
-            # Remove raw_categories to keep response clean
-            merged_results[i].pop("raw_categories", None)
+
+        # 6. Assign disciplines and clean up temp fields
+        for entry in merged_results:
+            entry["discipline"] = map_to_discipline({
+                "sources": entry["sources"],
+                "categories": entry.get("raw_categories", []),
+                "title": entry["title"],
+                "summary": entry["summary"],
+            })
+            entry.pop("raw_categories", None)  # Remove internal field from response
 
         if not merged_results:
             raise HTTPException(status_code=404, detail="No papers found for this query")
-        
+
+        logger.info(
+            f"Discovery: {len(merged_results)} unique papers "
+            f"(ArXiv:{len(arxiv_data)} DOAJ:{len(doaj_data)} "
+            f"PubMed:{len(pubmed_data)} OpenAlex:{len(openalex_data)})"
+        )
+
         return DiscoveryResponse(
-            message=f"Found {len(merged_results)} unique papers matching your query.",
+            message=(
+                f"Found {len(merged_results)} unique papers from "
+                f"ArXiv, DOAJ, PubMed & OpenAlex."
+            ),
             papers=merged_results,
-            analysis_id="", 
+            analysis_id="",
             status="idle"
         )
-        
+
+    except HTTPException:
+        raise
     except Exception as e:
         logger.error(f"Discovery error: {e}")
         raise HTTPException(status_code=500, detail=str(e))

app/services/openalex_service.py

@@ -1,85 +1,205 @@
 """
-Service for searching papers via OpenAlex.
+Service for searching and downloading papers from OpenAlex.
+OpenAlex is a fully open catalog of the global research system (200M+ works).
+API docs: https://developers.openalex.org/
 """
 import httpx
-from typing import List, Dict
+import os
+from typing import List, Dict, Optional
 from app.core.logging import logger
 
+
 class OpenAlexService:
-    def __init__(self, email: str = "soumanadama93@gmail.com"):
-        self.base_url = "https://api.openalex.org/works"
-        self.email = email
+    """
+    Service complet pour OpenAlex : recherche + téléchargement PDF open-access.
+    Suit exactement le même pattern que ArXivService, DOAJService, PubMedService.
+    """
+
+    BASE_URL = "https://api.openalex.org/works"
+
+    def __init__(
+        self,
+        download_dir: str = "./uploaded_files",
+        email: str = "soumanadama93@gmail.com"
+    ):
+        self.download_dir = download_dir
+        self.email = email  # Required by OpenAlex polite pool (higher rate limits)
+        if not os.path.exists(self.download_dir):
+            os.makedirs(self.download_dir)
 
+    # ------------------------------------------------------------------
+    # PUBLIC: fetch_papers — identique aux autres services
+    # ------------------------------------------------------------------
     def fetch_papers(self, query: str, max_results: int = 10) -> List[Dict]:
         """
-        Search OpenAlex for papers (Metadata).
+        Search OpenAlex for papers and return normalized metadata.
+        Uses 'polite pool' via mailto param for better rate limits.
+        Filters: only open-access works with a usable PDF or landing page URL.
         """
-        logger.info(f"Searching OpenAlex for: {query} (max: {max_results})")
+        logger.info(f"Searching OpenAlex for: '{query}' (max: {max_results})")
+
         params = {
             "search": query,
             "mailto": self.email,
-            "per-page": max_results
+            "per-page": min(max_results, 50),   # OpenAlex max per-page = 200
+            "sort": "relevance_score:desc",
+            # Only return open-access works to maximize PDF availability
+            "filter": "is_oa:true",
+            # Select only the fields we need (faster, lighter response)
+            "select": (
+                "id,title,authorships,abstract_inverted_index,"
+                "open_access,topics,publication_date,doi,"
+                "cited_by_count,primary_location"
+            ),
         }
+
         results = []
         try:
-            with httpx.Client() as client:
-                response = client.get(self.base_url, params=params, timeout=15.0)
+            with httpx.Client(timeout=20.0) as client:
+                response = client.get(self.BASE_URL, params=params)
                 response.raise_for_status()
                 data = response.json()
-                items = data.get("results", [])
-                
-                for item in items:
-                    # Ignore results with no title
-                    if not item.get("title"):
-                        continue
-                        
-                    # Reconstruct inverted abstract
-                    abstract = ""
-                    abs_dict = item.get("abstract_inverted_index")
-                    if abs_dict:
-                        # Find max index to pre-allocate array
-                        max_idx = -1
-                        for indices in abs_dict.values():
-                            for idx in indices:
-                                if idx > max_idx:
-                                    max_idx = idx
-                                    
-                        if max_idx >= 0:
-                            words = [""] * (max_idx + 1)
-                            for word, indices in abs_dict.items():
-                                for idx in indices:
-                                    words[idx] = word
-                            abstract = " ".join([w for w in words if w]).strip()
-                    else:
-                        abstract = "No abstract available."
-
-                    # Extract PDF url if open access
-                    oa = item.get("open_access", {})
-                    pdf_url = oa.get("oa_url")
-                    
-                    authors = []
-                    for authorship in item.get("authorships", []):
-                        author = authorship.get("author", {})
-                        if author.get("display_name"):
-                            authors.append(author.get("display_name"))
-                            
-                    concepts = [c.get("display_name") for c in item.get("concepts", [])[:5]]
-                    
-                    # Ensure ID is clean (OpenAlex ids look like https://openalex.org/W298...)
-                    raw_id = item.get("id", "")
-                    clean_id = raw_id.split("/")[-1] if raw_id else "unknown_id"
-
-                    results.append({
-                        "id": clean_id,
-                        "title": item.get("title"),
-                        "authors": authors,
-                        "summary": abstract,
-                        "url": pdf_url or item.get("doi"),
-                        "has_pdf": bool(pdf_url),
-                        "categories": concepts,
-                        "publication_date": item.get("publication_date")
-                    })
+
+            for item in data.get("results", []):
+                parsed = self._parse_item(item)
+                if parsed:
+                    results.append(parsed)
+
+            logger.info(f"OpenAlex returned {len(results)} results for '{query}'")
+
+        except httpx.HTTPStatusError as e:
+            logger.error(f"OpenAlex HTTP error {e.response.status_code}: {e}")
         except Exception as e:
-            logger.error(f"Error fetching from OpenAlex: {e}")
-            
+            logger.error(f"OpenAlex search error: {e}")
+
         return results
+
+    # ------------------------------------------------------------------
+    # PUBLIC: download_paper — même signature que DOAJService/PubMedService
+    # ------------------------------------------------------------------
+    def download_paper(self, paper_url: str, paper_id: str) -> str:
+        """
+        Downloads an open-access PDF from OpenAlex oa_url.
+        Returns local file path, or empty string on failure.
+        """
+        if not paper_url:
+            logger.warning(f"OpenAlex: no URL provided for {paper_id}")
+            return ""
+
+        # Sanitize filename (OpenAlex IDs look like "W2987984220")
+        safe_id = paper_id.replace("/", "_").replace(":", "_")
+        file_name = f"openalex_{safe_id}.pdf"
+        file_path = os.path.join(self.download_dir, file_name)
+
+        if os.path.exists(file_path):
+            logger.info(f"OpenAlex: cache hit for {paper_id}")
+            return file_path
+
+        logger.info(f"OpenAlex: downloading {paper_id} from {paper_url}")
+        try:
+            with httpx.Client(timeout=60.0, follow_redirects=True) as client:
+                response = client.get(paper_url)
+                response.raise_for_status()
+
+                content_type = response.headers.get("content-type", "").lower()
+                if "application/pdf" not in content_type and len(response.content) < 1000:
+                    logger.warning(
+                        f"OpenAlex: response for {paper_id} is not a PDF "
+                        f"(Content-Type: {content_type})"
+                    )
+                    return ""
+
+                with open(file_path, "wb") as f:
+                    f.write(response.content)
+
+            logger.info(f"OpenAlex: saved {paper_id} → {file_path}")
+            return file_path
+
+        except Exception as e:
+            logger.error(f"OpenAlex: failed to download {paper_id}: {e}")
+            return ""
+
+    # ------------------------------------------------------------------
+    # PRIVATE: helpers
+    # ------------------------------------------------------------------
+    def _parse_item(self, item: dict) -> Optional[Dict]:
+        """
+        Normalizes a single OpenAlex work into the common paper dict format
+        shared by all services in this project.
+        """
+        title = item.get("title")
+        if not title:
+            return None  # Skip works without title
+
+        # --- ID ---
+        raw_id = item.get("id", "")
+        # Looks like "https://openalex.org/W2987984220" → clean to "W2987984220"
+        clean_id = f"openalex_{raw_id.split('/')[-1]}" if raw_id else "openalex_unknown"
+
+        # --- Abstract (inverted index → plain text) ---
+        abstract = self._reconstruct_abstract(item.get("abstract_inverted_index"))
+
+        # --- Authors ---
+        authors = []
+        for authorship in item.get("authorships", []):
+            name = authorship.get("author", {}).get("display_name")
+            if name:
+                authors.append(name)
+
+        # --- Open Access PDF URL ---
+        oa = item.get("open_access", {})
+        pdf_url = oa.get("oa_url")  # Direct PDF or landing page
+        has_pdf = bool(pdf_url)
+
+        # Fallback: primary_location landing page
+        if not pdf_url:
+            primary = item.get("primary_location") or {}
+            pdf_url = primary.get("landing_page_url") or primary.get("pdf_url")
+            has_pdf = False  # It's a landing page, not a direct PDF
+
+        # Fallback: DOI
+        if not pdf_url and item.get("doi"):
+            pdf_url = item["doi"]
+
+        # --- Topics/Categories (replacing deprecated concepts) ---
+        topics = []
+        for topic in item.get("topics", [])[:5]:
+            name = topic.get("display_name")
+            if name:
+                topics.append(name)
+
+        return {
+            "id": clean_id,
+            "title": title,
+            "authors": authors if authors else ["Unknown"],
+            "summary": abstract,
+            "url": pdf_url,
+            "has_pdf": has_pdf,
+            "categories": topics,
+            "publication_date": item.get("publication_date", "n.d."),
+            "cited_by_count": item.get("cited_by_count", 0),
+            "source": "OpenAlex",
+        }
+
+    @staticmethod
+    def _reconstruct_abstract(abstract_inverted_index: Optional[dict]) -> str:
+        """
+        OpenAlex stores abstracts as an inverted index: {word: [positions]}.
+        This reconstructs the plain text.
+        """
+        if not abstract_inverted_index:
+            return "No abstract available."
+
+        try:
+            max_idx = max(
+                idx
+                for positions in abstract_inverted_index.values()
+                for idx in positions
+            )
+            words = [""] * (max_idx + 1)
+            for word, positions in abstract_inverted_index.items():
+                for idx in positions:
+                    words[idx] = word
+            return " ".join(w for w in words if w).strip()
+        except Exception:
+            return "Abstract reconstruction failed."