fix: Correct ArXiv/OpenAlex paper fetching in analysis_service

- ArXiv: strip 'arxiv_' prefix correctly, attempt full PDF download
with PDF parser for richer content, fallback to abstract
- OpenAlex: use direct Work ID REST endpoint (/works/W...) instead of
search-by-ID (which returned irrelevant results)
- All sources: prefer full content over abstract for K2 analysis
- Fix missing remote_id DB lookup before triggering remote fetch
- Add proper logging for each step

app/services/analysis_service.py

@@ -20,7 +20,7 @@ class AnalysisService:
         project_id: UUID,
         model_used: str
     ):
-        """Crée une nouvelle run d' d'analyse"""
+        """Crée une nouvelle run d'analyse"""
         analysis = AnalysisRun(
             project_id=project_id,
             model_used=model_used,
@@ -43,7 +43,7 @@ class AnalysisService:
             analysis.status = status
             analysis.completed_at = datetime.utcnow()
             if result:
-                analysis.result_data = result # SQLAlchemy handles JSON serialization
+                analysis.result_data = result  # SQLAlchemy handles JSON serialization
             self.analysis_repo.db.add(analysis)
             self.analysis_repo.db.commit()
             self.analysis_repo.db.refresh(analysis)
@@ -61,82 +61,150 @@ class AnalysisService:
         from app.services.doaj_service import DOAJService
         from app.services.pubmed_service import PubMedService
         from app.services.openalex_service import OpenAlexService
-        
+
+        UPLOAD_DIR = "./uploaded_files"
         db = self.analysis_repo.db
         try:
-            # 1. Fetch papers from DB or Remote
+            # 1. Fetch papers from DB or fetch on-demand from remote sources
             paper_repo = PaperRepository(db)
             analysis_run = self.analysis_repo.get_by_id(UUID(analysis_id))
             project_id = analysis_run.project_id
-            
+
             docs = []
             for pid in request.paper_ids:
-                # Try UUID search first
                 paper = None
+
+                # 1a. Try UUID lookup in DB (for uploaded PDFs)
                 try:
                     paper_uuid = UUID(pid)
                     paper = paper_repo.get_by_id(paper_uuid)
                 except (ValueError, AttributeError):
-                    # Try Remote ID search in DB
+                    pass
+
+                # 1b. Try remote_id lookup in DB (for previously fetched discovery papers)
+                if paper is None:
                     paper = db.query(ResearchPaper).filter(
                         ResearchPaper.remote_id == pid,
                         ResearchPaper.project_id == project_id
                     ).first()
-                
+
+                # 1c. Not in DB → fetch from remote source on-demand
                 if not paper:
-                    # FETCH FROM REMOTE AND SAVE
-                    logger.info(f"Paper {pid} not in DB. Fetching metadata on-demand...")
+                    logger.info(f"Paper {pid} not in DB. Fetching from remote source...")
                     remote_data = None
                     try:
                         if "doaj_" in pid:
-                            svc = DOAJService(); remote_data = svc.fetch_papers(pid.replace("doaj_", ""), 1)
+                            svc = DOAJService(download_dir=UPLOAD_DIR)
+                            remote_data = svc.fetch_papers(pid.replace("doaj_", ""), 1)
+
                         elif "pubmed_" in pid:
-                            svc = PubMedService(); remote_data = svc.fetch_papers(pid.replace("pubmed_", ""), 1)
+                            svc = PubMedService(download_dir=UPLOAD_DIR)
+                            remote_data = svc.fetch_papers(pid.replace("pubmed_", ""), 1)
+
                         elif "openalex_" in pid:
-                            svc = OpenAlexService(); remote_data = svc.fetch_papers(pid.replace("openalex_", ""), 1)
+                            # Use OpenAlex direct Work ID endpoint (e.g. W2987984220)
+                            svc = OpenAlexService(download_dir=UPLOAD_DIR)
+                            openalex_bare_id = pid.replace("openalex_", "")
+                            import httpx
+                            try:
+                                with httpx.Client(timeout=15.0) as client:
+                                    r = client.get(
+                                        f"https://api.openalex.org/works/{openalex_bare_id}",
+                                        params={
+                                            "mailto": svc.email,
+                                            "select": (
+                                                "id,title,authorships,abstract_inverted_index,"
+                                                "open_access,topics,publication_date,doi,"
+                                                "cited_by_count,primary_location"
+                                            )
+                                        }
+                                    )
+                                    r.raise_for_status()
+                                    parsed = svc._parse_item(r.json())
+                                    remote_data = [parsed] if parsed else []
+                                    logger.info(f"OpenAlex direct ID lookup succeeded for {openalex_bare_id}")
+                            except Exception as oa_err:
+                                logger.warning(f"OpenAlex direct ID lookup failed for {openalex_bare_id}: {oa_err}. Falling back to search.")
+                                remote_data = svc.fetch_papers(openalex_bare_id, 1)
+
                         elif "arxiv_" in pid:
-                            svc = ArXivService(); remote_data = svc.fetch_papers(pid.replace("arxiv_", ""), 1)
+                            # ArXiv: strip prefix to get proper ArXiv ID, try to download full PDF
+                            svc = ArXivService(download_dir=UPLOAD_DIR)
+                            arxiv_id = pid.replace("arxiv_", "")
+                            meta_list = svc.fetch_papers(arxiv_id, 1)
+                            if meta_list:
+                                try:
+                                    from app.rag.pdf_parser import PDFParser
+                                    pdf_path = svc.download_paper(arxiv_id)
+                                    if pdf_path:
+                                        full_text = PDFParser.extract_text(pdf_path)
+                                        if full_text:
+                                            meta_list[0]["content"] = full_text
+                                            logger.info(f"ArXiv PDF downloaded & parsed for {arxiv_id}: {len(full_text)} chars")
+                                except Exception as pdf_err:
+                                    logger.warning(f"ArXiv PDF download failed for {arxiv_id}, using abstract only: {pdf_err}")
+                            remote_data = meta_list
+
                         else:
-                            # Fallback to ArXiv if no prefix (for old saved papers without prefix)
-                            svc = ArXivService(); remote_data = svc.fetch_papers(pid, 1)
-                        
+                            # Fallback: treat as ArXiv search query
+                            svc = ArXivService(download_dir=UPLOAD_DIR)
+                            meta_list = svc.fetch_papers(pid, 1)
+                            if meta_list:
+                                try:
+                                    from app.rag.pdf_parser import PDFParser
+                                    pdf_path = svc.download_paper(pid)
+                                    if pdf_path:
+                                        full_text = PDFParser.extract_text(pdf_path)
+                                        if full_text:
+                                            meta_list[0]["content"] = full_text
+                                except Exception:
+                                    pass
+                            remote_data = meta_list
+
                         if remote_data:
                             p_info = remote_data[0]
-                            snippet = p_info.get("content", p_info.get("summary", ""))[:10000]
+                            # Prefer full PDF content over abstract for richer K2 analysis
+                            snippet = (p_info.get("content") or p_info.get("summary") or "")[:10000]
                             paper = ResearchPaper(
                                 project_id=project_id,
                                 remote_id=pid,
                                 title=p_info.get("title", "Unknown"),
                                 authors=", ".join(p_info.get("authors", [])),
                                 summary=snippet,
-                                publication_year=datetime.now().year # Fallback
+                                publication_year=datetime.now().year  # Fallback
                             )
                             db.add(paper)
                             db.commit()
                             db.refresh(paper)
-                            logger.info(f"Saved remote paper {pid} to project {project_id}")
+                            logger.info(f"Saved remote paper {pid} to project {project_id} ({len(snippet)} chars of content)")
                         else:
-                            logger.warning(f"No remote data found for {pid}")
+                            logger.warning(f"No remote data found for {pid} — skipping.")
+
                     except Exception as fetch_err:
                         logger.error(f"Failed to fetch/save remote paper {pid}: {fetch_err}")
 
+                # 1d. Build ScientificDocument from DB record
                 if paper:
-                    # Convert authors string to list
                     author_list = [a.strip() for a in paper.authors.split(",")] if paper.authors else ["Unknown"]
-                    
-                    # Determine document type based on remote_id prefix
+
+                    # Determine document type
                     dtype = "custom"
                     if paper.remote_id:
-                        if "pubmed_" in paper.remote_id: dtype = "pubmed"
-                        elif "arxiv_" in paper.remote_id or "." in paper.remote_id: dtype = "arxiv"
-                        else: dtype = "arxiv" # default for most research platforms
+                        if "pubmed_" in paper.remote_id:
+                            dtype = "pubmed"
+                        elif "arxiv_" in paper.remote_id or "." in paper.remote_id:
+                            dtype = "arxiv"
+                        elif "openalex_" in paper.remote_id:
+                            dtype = "openalex"
+                        else:
+                            dtype = "arxiv"  # sensible default
 
                     docs.append(ScientificDocument(
                         id=str(paper.id),
                         title=paper.title,
                         authors=author_list,
                         abstract=paper.summary or "",
-                        content=(paper.summary or "")[:10000], 
+                        content=(paper.summary or "")[:10000],
                         document_type=dtype,
                         url=paper.pdf_path or ""
                     ))
@@ -145,6 +213,8 @@ class AnalysisService:
                 logger.error(f"No documents found for analysis {analysis_id}")
                 raise ValueError("Zero documents identified for analysis. Aborting.")
 
+            logger.info(f"Starting K2 analysis {analysis_id} with {len(docs)} document(s)")
+
             # 2. Prepare K2 Request
             k2_req = K2AnalysisRequest(
                 documents=docs,
@@ -164,35 +234,31 @@ class AnalysisService:
                 analysis_id=UUID(analysis_id),
                 step_number=1,
                 reasoning=result.reasoning_summary or "Analysis complete",
-                source_chunks=result.reasoning_trace # JSON field
+                source_chunks=result.reasoning_trace  # JSON field
             )
             db.add(trace)
-            
-            # 5. Complete Analysis
-            # result is a Pydantic model, convert to dict for storage
+
+            # 5. Complete Analysis (convert Pydantic model to dict for storage)
             self.complete_analysis(UUID(analysis_id), result=result.model_dump())
             db.commit()
+            logger.info(f"Analysis {analysis_id} completed successfully")
 
         except Exception as e:
-            db.rollback() # CLEAN TRANSACTION
+            db.rollback()  # CLEAN TRANSACTION
             logger.error(f"FATAL ERROR in process_analysis for {analysis_id}: {str(e)}")
             import traceback
             logger.error(traceback.format_exc())
-            
+
             try:
                 error_data = {
                     "status": "FAILED",
                     "reasoning_summary": f"Erreur technique lors de l'analyse : {str(e)}\n\nTrace: {traceback.format_exc()[:500]}...",
                     "confidence_overall": 0
                 }
-                # Use a safe UUID conversion
-                try:
-                    target_uuid = UUID(analysis_id) if isinstance(analysis_id, str) else analysis_id
-                    self.complete_analysis(target_uuid, status="FAILED", result=error_data)
-                    db.commit()
-                    logger.info(f"Marked analysis {analysis_id} as FAILED in DB")
-                except Exception as uuid_err:
-                    logger.error(f"Could not convert {analysis_id} to UUID or save failure: {uuid_err}")
+                target_uuid = UUID(analysis_id) if isinstance(analysis_id, str) else analysis_id
+                self.complete_analysis(target_uuid, status="FAILED", result=error_data)
+                db.commit()
+                logger.info(f"Marked analysis {analysis_id} as FAILED in DB")
             except Exception as final_err:
                 logger.error(f"Failed to even mark analysis as FAILED: {final_err}")
                 db.rollback()