File size: 2,373 Bytes
8564450 f3144c4 98b82e8 f3144c4 98b82e8 c776287 f3144c4 c776287 f3144c4 c776287 f3144c4 98b82e8 f3144c4 98b82e8 8564450 f3144c4 98b82e8 f3144c4 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 | import re
import gradio as gr
from huggingface_hub import hf_hub_download
from llama_cpp import Llama
print("Downloading model...")
model_path = hf_hub_download(
repo_id="Jackrong/Qwen3.5-9B-Claude-4.6-Opus-Reasoning-Distilled-v2-GGUF",
filename="Qwen3.5-9B.Q4_K_M.gguf",
)
print(f"Model: {model_path}")
print("Loading model...")
llm = Llama(
model_path=model_path,
n_ctx=2048,
n_threads=2,
n_batch=256,
verbose=False,
)
print("Model loaded!")
def strip_think_tags(text):
return re.sub(r"<think>.*?</think>\s*", "", text, flags=re.DOTALL).strip()
def chat(message, history, system_prompt, max_tokens, temperature, show_thinking):
messages = []
if system_prompt.strip():
messages.append({"role": "system", "content": system_prompt.strip()})
for h in history[-4:]:
if h["role"] == "user":
messages.append({"role": "user", "content": h["content"]})
elif h["role"] == "assistant":
messages.append({"role": "assistant", "content": h["content"]})
messages.append({"role": "user", "content": message})
try:
response = llm.create_chat_completion(
messages=messages,
max_tokens=int(max_tokens),
temperature=float(temperature),
stream=True,
)
partial = ""
for chunk in response:
delta = chunk["choices"][0]["delta"].get("content", "")
partial += delta
if show_thinking:
yield partial
else:
yield strip_think_tags(partial)
except Exception as e:
yield f"Error: {str(e)}"
with gr.Blocks(title="Qwen3.5-9B Claude Reasoning") as demo:
gr.Markdown("# Qwen3.5-9B Claude 4.6 Opus Reasoning v2\nCPU inference - response ကြာနိုင်ပါတယ်")
system_prompt = gr.Textbox(label="System Prompt", value="You are a helpful assistant. Respond in the same language the user uses.", lines=2)
with gr.Row():
max_tokens = gr.Slider(64, 2048, 512, step=64, label="Max Tokens")
temperature = gr.Slider(0.1, 1.5, 0.7, step=0.1, label="Temperature")
show_thinking = gr.Checkbox(label="Show thinking", value=False)
gr.ChatInterface(fn=chat, additional_inputs=[system_prompt, max_tokens, temperature, show_thinking])
demo.launch(server_name="0.0.0.0", server_port=7860) |