Update app.py

app.py

@@ -1,5 +1,5 @@
 import torch
-from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig
+from transformers import AutoModelForCausalLM, AutoTokenizer
 from peft import PeftModel
 import gradio as gr
 import os
@@ -20,29 +20,27 @@ FINETUNED_MODEL_DIR = "./finetuned_model"  # Path to your adapter weights
 # Load tokenizer
 tokenizer = AutoTokenizer.from_pretrained(BASE_MODEL)
 
-# Load base model with quantization
-quantization_config = BitsAndBytesConfig(
-    load_in_4bit=True,  
-    bnb_4bit_compute_dtype=torch.float16,
-    bnb_4bit_use_double_quant=True
-)
-
+# Load base model (WITHOUT bitsandbytes)
 base_model = AutoModelForCausalLM.from_pretrained(
     BASE_MODEL,  
-    quantization_config=quantization_config,
-    torch_dtype=torch.float16,
-    device_map="auto"
+    torch_dtype=torch.float32  # Ensure CPU compatibility
 )
 
+# Move base model to CPU
+base_model.to("cpu")
+
 # Load LoRA adapter
 model = PeftModel.from_pretrained(base_model, FINETUNED_MODEL_DIR)
 
 # Merge adapter with base model
 model = model.merge_and_unload()
 
+# Move model to CPU (again, just to be sure)
+model.to("cpu")
+
 # Inference function
 def chat(message):
-    inputs = tokenizer(message, return_tensors="pt").to("cuda" if torch.cuda.is_available() else "cpu")
+    inputs = tokenizer(message, return_tensors="pt").to("cpu")  # Ensure inputs are on CPU
     output = model.generate(**inputs, max_new_tokens=100)
     response = tokenizer.decode(output[0], skip_special_tokens=True)
     return response