# ——— Patch 1: Stop Streamlit watcher hitting torch._classes.__path__ ———
import torch
class _DummyPath:
def __init__(self):
self._path = []
def __getattr__(self, name):
return []
torch._classes.__path__ = _DummyPath()
# ——— Patch 2: Make SentenceTransformer.to() fall back to to_empty() on meta modules ———
import sentence_transformers as _st
_BaseST = _st.SentenceTransformer
class SentenceTransformer(_BaseST):
def to(self, *args, **kwargs):
try:
return super().to(*args, **kwargs)
except NotImplementedError:
return super().to_empty(*args, **kwargs)
# ——— Standard imports ———
import streamlit as st
import streamlit.components.v1 as components
import PyPDF2
import numpy as np
from typing import List, Dict
from langdetect import detect, detect_langs
from sklearn.metrics.pairwise import cosine_similarity
import google.generativeai as genai
from gtts import gTTS
import speech_recognition as sr
import tempfile, base64, os
import requests, time
import sqlite3
from datetime import datetime
import pandas as pd
import faiss # Import FAISS
# ——— Configuration ———
GENAI_API_KEY = "AIzaSyA5xtoT9HAjH-wsa7OHFXlBjRRcXwCFBMg"
DID_API_KEY = "a3Jpc2huYW12aXB1bEBnbWFpbC4Y29t:5DSNuJuWUBZQ0G44TfJlJ" # Replace with your actual D-ID API key
AVATAR_IMAGE_URL = "https://raw.githubusercontent.com/de-id/live-streaming-demo/main/alex_v2_idle_image.png"
# Ensure data directories exist
if not os.path.exists("data"):
os.makedirs("data")
if not os.path.exists("data/pdfs"):
os.makedirs("data/pdfs")
if not os.path.exists("data/faiss_indexes"):
os.makedirs("data/faiss_indexes")
# ——— SQLite DB Setup ———
def init_db():
conn = sqlite3.connect("interactions.db")
cursor = conn.cursor()
cursor.execute("""
CREATE TABLE IF NOT EXISTS users (
id INTEGER PRIMARY KEY AUTOINCREMENT,
username TEXT UNIQUE NOT NULL,
password TEXT NOT NULL
)
""")
cursor.execute("""
CREATE TABLE IF NOT EXISTS interactions (
id INTEGER PRIMARY KEY AUTOINCREMENT,
user_id INTEGER,
timestamp TEXT,
language TEXT,
question TEXT,
answer TEXT,
FOREIGN KEY (user_id) REFERENCES users (id)
)
""")
cursor.execute("""
CREATE TABLE IF NOT EXISTS documents (
id INTEGER PRIMARY KEY AUTOINCREMENT,
user_id INTEGER,
filename TEXT NOT NULL,
filepath TEXT NOT NULL,
faiss_index_path TEXT NOT NULL,
language TEXT, -- Store the detected primary language of the document
FOREIGN KEY (user_id) REFERENCES users (id)
)
""")
conn.commit()
conn.close()
def add_user(username, password):
conn = sqlite3.connect("interactions.db")
cursor = conn.cursor()
try:
# NOTE: For production, use a strong hashing library like 'bcrypt' or 'passlib'
# For this example, a simple hash() is used, which is NOT SECURE for real applications.
cursor.execute("INSERT INTO users (username, password) VALUES (?, ?)", (username, hash(password)))
conn.commit()
return True
except sqlite3.IntegrityError:
return False # Username already exists
finally:
conn.close()
def verify_user(username, password):
conn = sqlite3.connect("interactions.db")
cursor = conn.cursor()
# NOTE: For production, use a strong hashing library like 'bcrypt' or 'passlib'
cursor.execute("SELECT id FROM users WHERE username = ? AND password = ?", (username, hash(password)))
user = cursor.fetchone()
conn.close()
return user[0] if user else None
def save_interaction(user_id: int, language: str, question: str, answer: str):
conn = sqlite3.connect("interactions.db")
cursor = conn.cursor()
cursor.execute("""
INSERT INTO interactions (user_id, timestamp, language, question, answer)
VALUES (?, ?, ?, ?, ?)
""", (user_id, datetime.now().isoformat(), language, question, answer))
conn.commit()
conn.close()
def save_document_metadata(user_id: int, filename: str, filepath: str, faiss_index_path: str, language: str):
conn = sqlite3.connect("interactions.db")
cursor = conn.cursor()
cursor.execute("""
INSERT INTO documents (user_id, filename, filepath, faiss_index_path, language)
VALUES (?, ?, ?, ?, ?)
""", (user_id, filename, filepath, faiss_index_path, language))
conn.commit()
conn.close()
def get_user_documents(user_id: int) -> List[Dict]:
conn = sqlite3.connect("interactions.db")
cursor = conn.cursor()
cursor.execute("SELECT id, filename, filepath, faiss_index_path, language FROM documents WHERE user_id = ?", (user_id,))
docs = [{"id": row[0], "filename": row[1], "filepath": row[2], "faiss_index_path": row[3], "language": row[4]} for row in cursor.fetchall()]
conn.close()
return docs
# ——— RAGSingleLanguage class ———
class RAGSingleLanguage:
def __init__(self, api_key: str):
genai.configure(api_key=api_key)
self.model = genai.GenerativeModel('gemini-1.5-flash')
self.embedder = SentenceTransformer('paraphrase-multilingual-MiniLM-L12-v2')
self.chunks: List[str] = []
self.faiss_index = None
self.language: str = 'en' # Default language for translation if not explicitly set
def detect_languages(self, text: str) -> List[str]:
seg_size = 1000
probs = {}
for i in range(0, len(text), seg_size):
seg = text[i:i+seg_size]
try:
for lang in detect_langs(seg):
probs[lang.lang] = max(probs.get(lang.lang, 0.0), lang.prob)
except:
continue
# Only return languages with a probability >= 0.2
langs = [l for l,p in probs.items() if p >= 0.2]
# Fallback to English if no strong detection
return langs or ['en']
def translate(self, text: str, tgt: str) -> str:
try:
src = detect(text)
except:
src = 'en' # Assume English if detection fails
if src.lower() == tgt.lower():
return text
prompt = f"Translate to {tgt.upper()}:\n\n{text}"
try:
return self.model.generate_content(prompt).text.strip()
except Exception as e:
st.warning(f"Translation failed: {e}. Returning original text.")
return text
def process_document(self, pdf_file_path: str, chunk_size: int = 500) -> str:
reader = PyPDF2.PdfReader(pdf_file_path)
pages = [p.extract_text() or "" for p in reader.pages]
full_text = " ".join(pages)
# Detect dominant language of the document
detected_langs = self.detect_languages(full_text)
# We'll store the first detected language as the document's primary language
doc_language = detected_langs[0] if detected_langs else 'en'
full = full_text.split()
self.chunks = [
" ".join(full[i:i+chunk_size])
for i in range(0, len(full), chunk_size)
]
# Generate embeddings
embeddings = self.embedder.encode(
self.chunks,
convert_to_numpy=True,
normalize_embeddings=True
)
# Create FAISS index
dimension = embeddings.shape[1]
self.faiss_index = faiss.IndexFlatL2(dimension)
self.faiss_index.add(embeddings)
return doc_language # Return the detected language for saving
def load_faiss_index(self, faiss_index_path: str, document_chunks: List[str]):
try:
self.faiss_index = faiss.read_index(faiss_index_path)
self.chunks = document_chunks # Load associated chunks
return True
except Exception as e:
st.error(f"Error loading FAISS index: {e}")
return False
def set_language(self, lang: str):
self.language = lang
def answer_question(self, question: str, top_k: int = 5) -> str:
if self.faiss_index is None or not self.chunks:
return "Please select a document to query from."
q_en = self.translate(question, 'en')
q_emb = self.embedder.encode([q_en], convert_to_numpy=True, normalize_embeddings=True)
# Search FAISS index
# D, I are distances and indices respectively.
# For normalized embeddings, L2 distance (d) is related to cosine similarity (s) by d^2 = 2(1-s)
distances, indices = self.faiss_index.search(q_emb, top_k)
contexts = []
for i, dist in zip(indices[0], distances[0]):
if i >= 0 and i < len(self.chunks): # Ensure index is valid
sim_score = 1 - (dist / 2) # Convert L2 distance to cosine similarity for display
contexts.append(f"[Score: {sim_score:.2f}]\n{self.chunks[i]}")
ctx = "\n\n".join(contexts)
prompt = (
"Answer the following question using only the provided context. "
"Be accurate and detailed. If the answer is not present, say: "
"'I apologize, but I cannot find this information in the documentation. "
"Please contact customer support for accurate assistance on this matter.'\n\n"
f"Context:\n{ctx}\n\nQuestion: {q_en}"
)
try:
out = self.model.generate_content(prompt).text.strip()
except Exception as e:
return f"Error generating answer: {e}"
return self.translate(out, self.language)
# ——— Voice Input ———
def recognize_voice(lang_code='en-IN') -> str:
r = sr.Recognizer()
with sr.Microphone() as src:
st.info("🎤 Adjusting for ambient noise…")
r.adjust_for_ambient_noise(src, duration=1)
st.info("Listening…")
try:
audio = r.listen(src, timeout=10, phrase_time_limit=10)
except sr.WaitTimeoutError:
st.warning("⏰ No speech detected.")
return ""
try:
return r.recognize_google(audio, language=lang_code)
except sr.UnknownValueError:
st.error("❗ Could not understand audio.")
except sr.RequestError as e:
st.error(f"🚫 Speech API error: {e}")
return ""
# ——— D-ID Avatar Generator ———
def generate_did_avatar_video(answer_text: str, image_url: str) -> str:
url = "https://api.d-id.com/talks"
headers = {
"Authorization": f"Basic {base64.b64encode(DID_API_KEY.encode()).decode()}",
"Content-Type": "application/json"
}
payload = {
"source_url": image_url,
"script": {
"type": "text",
"input": answer_text,
"provider": {
"type": "microsoft",
"voice_id": "en-US-GuyNeural", # Default English voice
"voice_config": {"style": "Cheerful"}
}
},
"config": {"stitch": True}
}
response = requests.post(url, json=payload, headers=headers)
if response.status_code not in [200, 201]:
st.error(f"❌ Avatar video request failed: {response.text}")
return ""
talk_id = response.json().get("id")
if not talk_id:
st.error("❌ Talk ID not found in response.")
return ""
# Poll for video status
for _ in range(30): # Try for up to 60 seconds (30 * 2 seconds)
time.sleep(2)
check = requests.get(f"https://api.d-id.com/talks/{talk_id}", headers=headers)
if check.status_code == 200:
data = check.json()
if data.get("status") == "done":
return data.get("result_url")
elif data.get("status") == "error":
st.error(f"❌ D-ID video generation error: {data.get('error')}")
return ""
st.warning("⚠️ Avatar video is still processing or timed out.")
return ""
# ——— Main App ———
def main():
init_db()
st.set_page_config(page_title="Voice‑Viz RAG", page_icon="🔊")
st.title("🔊 AI Helpdesk")
# Initialize all session state variables at the top
if 'rag' not in st.session_state:
st.session_state.rag = RAGSingleLanguage(GENAI_API_KEY)
if 'logged_in' not in st.session_state:
st.session_state.logged_in = False
st.session_state.user_id = None
st.session_state.username = None
if 'selected_doc_id' not in st.session_state:
st.session_state.selected_doc_id = None
st.session_state.selected_doc_chunks = []
if 'current_doc_language' not in st.session_state: # Stores the language of the currently loaded document
st.session_state.current_doc_language = 'en'
if 'interaction_language' not in st.session_state: # Stores the language chosen for interaction (can differ from doc lang)
st.session_state.interaction_language = 'en'
if 'voice_q' not in st.session_state: # THIS IS THE FIX FOR THE ATTRIBUTEERROR
st.session_state.voice_q = ""
st.sidebar.header("How to use")
st.sidebar.markdown("""
1. Login or Sign Up.
2. Upload PDF(s) to your account.
3. Select a document from your uploads.
4. Confirm or change the interaction language.
5. Type or speak your question.
6. Read or listen to the AI's answer.
""")
if not st.session_state.logged_in:
st.subheader("User Authentication")
auth_option = st.radio("Choose an option:", ("Login", "Sign Up"))
with st.form("auth_form"):
username = st.text_input("Username")
password = st.text_input("Password", type="password")
submitted = st.form_submit_button("Submit")
if submitted:
if auth_option == "Login":
user_id = verify_user(username, password)
if user_id:
st.session_state.logged_in = True
st.session_state.user_id = user_id
st.session_state.username = username
st.success(f"Welcome, {username}!")
st.rerun() # Rerun to switch to the main app view
else:
st.error("Invalid username or password.")
elif auth_option == "Sign Up":
if add_user(username, password):
st.success("Account created successfully! Please log in.")
else:
st.error("Username already exists. Please choose a different one.")
else:
st.sidebar.write(f"Logged in as: **{st.session_state.username}**")
if st.sidebar.button("Logout"):
st.session_state.logged_in = False
st.session_state.user_id = None
st.session_state.username = None
st.session_state.selected_doc_id = None
st.session_state.selected_doc_chunks = []
st.session_state.current_doc_language = 'en'
st.session_state.interaction_language = 'en'
st.session_state.voice_q = "" # Reset voice input
st.session_state.rag = RAGSingleLanguage(GENAI_API_KEY) # Reset RAG instance
st.rerun()
st.subheader("Document Management")
uploaded_file = st.file_uploader("Upload your PDF manual(s)", type="pdf", accept_multiple_files=True)
if uploaded_file:
for file in uploaded_file:
# Check if the file (by name) is already uploaded by this user
existing_docs = get_user_documents(st.session_state.user_id)
if file.name in [doc['filename'] for doc in existing_docs]:
st.info(f"Document '{file.name}' already uploaded by you.")
continue # Skip to the next file if already exists
with st.spinner(f"Processing {file.name}…"):
# Save PDF to disk
pdf_path = os.path.join("data", "pdfs", file.name)
with open(pdf_path, "wb") as f:
f.write(file.getbuffer())
# Process document and get its primary language
doc_language = st.session_state.rag.process_document(pdf_path)
# Save FAISS index
faiss_index_filename = f"{os.path.splitext(file.name)[0]}_{st.session_state.user_id}.faiss"
faiss_index_path = os.path.join("data", "faiss_indexes", faiss_index_filename)
faiss.write_index(st.session_state.rag.faiss_index, faiss_index_path)
# Save chunks separately (FAISS only stores embeddings, not the text chunks)
chunks_filename = f"{os.path.splitext(file.name)[0]}_{st.session_state.user_id}.chunks"
chunks_path = os.path.join("data", "faiss_indexes", chunks_filename)
with open(chunks_path, "w", encoding="utf-8") as f:
# Use a unique delimiter that is unlikely to appear in the text
f.write("\n--CHUNK_DELIMITER--\n".join(st.session_state.rag.chunks))
# Save document metadata to DB
save_document_metadata(st.session_state.user_id, file.name, pdf_path, faiss_index_path, doc_language)
st.success(f"✅ Document '{file.name}' processed and saved!")
st.rerun() # Rerun to refresh the document list
# Display and allow selection of user's uploaded documents
user_docs = get_user_documents(st.session_state.user_id)
if user_docs:
doc_options_display = {doc['filename']: doc for doc in user_docs}
# Add an empty option for "No document selected"
selected_filename = st.selectbox(
"Select a document to query:",
[""] + list(doc_options_display.keys()),
key="doc_selector" # Add a key to avoid potential widget errors
)
# Logic to load selected document's FAISS index and chunks
if selected_filename and selected_filename != "":
selected_doc_info = doc_options_display[selected_filename]
# Check if this document is already loaded
if st.session_state.selected_doc_id != selected_doc_info['id']:
st.session_state.selected_doc_id = selected_doc_info['id']
chunks_filename = f"{os.path.splitext(selected_doc_info['filename'])[0]}_{st.session_state.user_id}.chunks"
chunks_path = os.path.join("data", "faiss_indexes", chunks_filename)
if os.path.exists(chunks_path):
with open(chunks_path, "r", encoding="utf-8") as f:
st.session_state.selected_doc_chunks = f.read().split("\n--CHUNK_DELIMITER--\n")
else:
st.error("Error: Chunks file not found for this document.")
st.session_state.selected_doc_chunks = []
st.session_state.selected_doc_id = None # Invalidate selection
if st.session_state.selected_doc_id and \
st.session_state.rag.load_faiss_index(selected_doc_info['faiss_index_path'], st.session_state.selected_doc_chunks):
st.success(f"Selected document: '{selected_filename}'")
# Set the detected language of the document
st.session_state.current_doc_language = selected_doc_info['language']
st.session_state.interaction_language = selected_doc_info['language'] # Default interaction language to doc's
st.session_state.rag.set_language(st.session_state.interaction_language)
st.rerun() # Rerun to update language selector and clear old inputs
else:
st.error(f"Could not load FAISS index for '{selected_filename}'.")
st.session_state.selected_doc_id = None
st.session_state.rag.faiss_index = None
st.session_state.rag.chunks = []
st.session_state.current_doc_language = 'en'
st.session_state.interaction_language = 'en'
# If a document is selected and loaded, allow language choice for interaction
if st.session_state.selected_doc_id:
st.markdown("---") # Separator for clarity
st.markdown("**Choose Interaction Language**")
# You could fetch all detected languages from the processed document if desired
# For simplicity, we'll offer a few common ones, plus the detected document language
available_langs = sorted(list(set(['en', 'hi', 'fr', 'es', 'de', st.session_state.current_doc_language])))
# Remove duplicates and ensure the current_doc_language is an option
lang_selection = st.selectbox(
"Select the language for your question and the AI's answer:",
[lang.upper() for lang in available_langs],
index=available_langs.index(st.session_state.interaction_language) if st.session_state.interaction_language in available_langs else 0,
key="interaction_lang_selector"
)
if lang_selection:
new_lang = lang_selection.lower()
if new_lang != st.session_state.interaction_language:
st.session_state.interaction_language = new_lang
st.session_state.rag.set_language(new_lang)
st.rerun() # Rerun to update the question input field's language
st.markdown(f"**Asking in:** `{st.session_state.interaction_language.upper()}`")
st.markdown("---") # Separator
st.markdown("**Type your question**")
typed_question = st.text_input(f"Ask in {st.session_state.interaction_language.upper()}:", value=st.session_state.voice_q, key="typed_question_input")
st.markdown("**Or use voice input**")
if st.button("🎙️ Speak Your Question", key="speak_button"):
# Adjust language code for speech recognition based on interaction language
recognizer_lang_code = st.session_state.interaction_language
if recognizer_lang_code == "en":
recognizer_lang_code = "en-IN" # Default to Indian English for better recognition in some cases
elif recognizer_lang_code == "hi":
recognizer_lang_code = "hi-IN" # Hindi
# Add more specific regional codes if necessary for other languages
recd_speech = recognize_voice(recognizer_lang_code)
if recd_speech:
st.session_state.voice_q = recd_speech
st.success(f"🎤 You said: {recd_speech}")
st.rerun() # Rerun to populate the text input with spoken text
else:
st.warning("No speech recognized.")
# Use the typed input or the voice input if available
question_to_process = typed_question or st.session_state.voice_q
if st.button("Get Answer", key="get_answer_button") and question_to_process:
st.markdown(f"🔍 Question: `{question_to_process}`")
with st.spinner("Thinking…"):
answer = st.session_state.rag.answer_question(question_to_process)
st.markdown(f"**Answer ({st.session_state.interaction_language.upper()}):** {answer}")
# Save to DB
save_interaction(st.session_state.user_id, st.session_state.interaction_language, question_to_process, answer)
# Text-to-Speech (gTTS)
with tempfile.NamedTemporaryFile(delete=False, suffix=".mp3") as fp:
try:
gTTS(text=answer, lang=st.session_state.interaction_language).save(fp.name)
mp3_bytes = open(fp.name, "rb").read()
b64 = base64.b64encode(mp3_bytes).decode()
html = f"""
"""
components.html(html, height=150)
except Exception as e:
st.error(f"Error generating audio: {e}. Please ensure gTTS supports '{st.session_state.interaction_language}'.")
# Clear voice_q after processing the answer
st.session_state.voice_q = ""
st.markdown("### 🧑💼 Speaking AI Avatar")
with st.spinner("Generating avatar video…"):
video_url = generate_did_avatar_video(answer, AVATAR_IMAGE_URL)
if video_url:
st.video(video_url)
else:
st.error("Failed to load avatar video.")
elif st.button("Get Answer") and not question_to_process:
st.warning("Please enter or speak a question.")
else:
st.info("Please select a document from your uploaded files to start querying.")
# Reset RAG if no document is selected
st.session_state.selected_doc_id = None
st.session_state.rag.faiss_index = None
st.session_state.rag.chunks = []
st.session_state.current_doc_language = 'en'
st.session_state.interaction_language = 'en'
st.session_state.rag.set_language('en') # Reset RAG's internal language
else:
st.info("No documents uploaded yet. Please upload a PDF to begin.")
# --- Optional: Admin View ---
st.sidebar.markdown("---")
st.sidebar.header("Admin Views")
if st.sidebar.checkbox("📜 Show Past Interactions"):
if st.session_state.logged_in:
conn = sqlite3.connect("interactions.db")
df = pd.read_sql_query(f"SELECT timestamp, language, question, answer FROM interactions WHERE user_id = {st.session_state.user_id} ORDER BY timestamp DESC", conn)
if not df.empty:
st.sidebar.dataframe(df)
else:
st.sidebar.info("No past interactions for this user.")
conn.close()
else:
st.sidebar.warning("Please log in to view past interactions.")
if st.sidebar.checkbox("📂 Show My Uploaded Documents"):
if st.session_state.logged_in:
user_docs = get_user_documents(st.session_state.user_id)
if user_docs:
df_docs = pd.DataFrame(user_docs)
st.sidebar.dataframe(df_docs[['filename', 'language']])
else:
st.sidebar.info("No documents uploaded yet.")
else:
st.sidebar.warning("Please log in to view your uploaded documents.")
if __name__ == "__main__":
main()