Update app.py

app.py

@@ -5,30 +5,33 @@ from langchain.chains.question_answering import load_qa_chain
 from langchain.llms import OpenAI
 import os
 import streamlit as st
+import torch
+from peft import PeftModel, PeftConfig
+from transformers import AutoModelForCausalLM, AutoTokenizer
 
-with open("guide1.txt") as f:
-    hitchhikersguide = f.read()
+peft_model_id = "fiona/to_onion_news_converter"
+config = PeftConfig.from_pretrained(peft_model_id)
+model = AutoModelForCausalLM.from_pretrained(config.base_model_name_or_path, return_dict=True, load_in_8bit=False)
+tokenizer = AutoTokenizer.from_pretrained(config.base_model_name_or_path)
 
-text_splitter = CharacterTextSplitter(chunk_size=1000, chunk_overlap=0, separator = "\n")
-texts = text_splitter.split_text(hitchhikersguide)
+# Load the Lora model
+model = PeftModel.from_pretrained(model, peft_model_id)
 
-embeddings = OpenAIEmbeddings()
+def make_inference(news_headline):
+  batch = tokenizer(f"### INSTRUCTION\nBelow is a standard news headline, please rewrite it in a satirical style .\n\n### Standard:\n{news_headline}\n\n### new news:\n", return_tensors='pt')
 
-docsearch = Chroma.from_texts(texts, embeddings, metadatas=[{"source": str(i)} for i in range(len(texts))]).as_retriever()
-
-chain = load_qa_chain(OpenAI(temperature=0), chain_type="stuff")
-
-def make_inference(query):
-    docs = docsearch.get_relevant_documents(query)
-    return(chain.run(input_documents=docs, question=query))
+  with torch.cuda.amp.autocast():
+    output_tokens = model.generate(**batch, max_new_tokens=200)
+      
+  return tokenizer.decode(output_tokens[0], skip_special_tokens=True)
 
 if __name__ == "__main__":
     # Title of the web application
-    st.title('🗣️TalkToMyDoc📄')
+    st.title('Onion news converter')
     
     # Text input widget
-    user_input = st.text_input('Enter a question about Hitchhiker\'s Galaxy Guide book:', '', help='🗣️TalkToMyDoc📄 is a tool that allows you to ask questions about a document. In this case - Hitch Hitchhiker\'s Guide to the Galaxy..')
+    user_input = st.text_input('Enter a news headline', '')
     
     # Displaying output directly below the input field
     if user_input:
-        st.write('Answer:', make_inference(user_input))
+        st.write('The onion style:', make_inference(user_input))