Update app.py

app.py

@@ -1,69 +1,178 @@
-import gradio as gr
-from huggingface_hub import InferenceClient
-
-
-def respond(
-    message,
-    history: list[dict[str, str]],
-    system_message,
-    max_tokens,
-    temperature,
-    top_p,
-    hf_token: gr.OAuthToken,
-):
-    """
-    For more information on `huggingface_hub` Inference API support, please check the docs: https://huggingface.co/docs/huggingface_hub/v0.22.2/en/guides/inference
-    """
-    client = InferenceClient(token=hf_token.token, model="openai/gpt-oss-20b")
-
-    messages = [{"role": "system", "content": system_message}]
-
-    messages.extend(history)
-
-    messages.append({"role": "user", "content": message})
-
-    response = ""
-
-    for message in client.chat_completion(
-        messages,
-        max_tokens=max_tokens,
-        stream=True,
-        temperature=temperature,
-        top_p=top_p,
-    ):
-        choices = message.choices
-        token = ""
-        if len(choices) and choices[0].delta.content:
-            token = choices[0].delta.content
-
-        response += token
-        yield response
-
-
-"""
-For information on how to customize the ChatInterface, peruse the gradio docs: https://www.gradio.app/docs/chatinterface
-"""
-chatbot = gr.ChatInterface(
-    respond,
-    additional_inputs=[
-        gr.Textbox(value="You are a friendly Chatbot.", label="System message"),
-        gr.Slider(minimum=1, maximum=2048, value=512, step=1, label="Max new tokens"),
-        gr.Slider(minimum=0.1, maximum=4.0, value=0.7, step=0.1, label="Temperature"),
-        gr.Slider(
-            minimum=0.1,
-            maximum=1.0,
-            value=0.95,
-            step=0.05,
-            label="Top-p (nucleus sampling)",
-        ),
-    ],
-)
-
-with gr.Blocks() as demo:
-    with gr.Sidebar():
-        gr.LoginButton()
-    chatbot.render()
+from __future__ import annotations
+import json
+import os
+import subprocess
+import tarfile
+import time
+import urllib.request
+import zipfile
 
+from fastapi import Request
+from fastapi.responses import StreamingResponse
+from gradio import Server
+import httpx
+
+app = Server()
+
+try:
+    import spaces
+    HAS_SPACES = True
+except ImportError:
+    HAS_SPACES = False
+
+@spaces.GPU if HAS_SPACES else lambda f: f
+def whygpu():
+    pass
+
+def build_latest_llama_server():
+    binary_path = os.path.abspath("llama.cpp/build/bin/llama-server")
+    if os.path.exists(binary_path):
+        return binary_path
+
+    print("Building latest llama.cpp from master source...")
+
+    if not os.path.exists("llama.cpp"):
+        subprocess.run(["git", "clone", "https://github.com/unslothai/llama.cpp.git"], check=True)
+    else:
+        subprocess.run(["git", "pull"], cwd="llama.cpp", check=True)
+
+    subprocess.run(["git", "fetch", "origin", "pull/144/head:mtp"], cwd="llama.cpp", check=True)
+    subprocess.run(["git", "checkout", "mtp"], cwd="llama.cpp", check=True)
+
+    env = os.environ.copy()
+    if "/usr/local/cuda/bin" not in env.get("PATH", ""):
+        env["PATH"] = f"/usr/local/cuda/bin:{env.get('PATH', '')}"
+
+    cmake_cmd = [
+        "cmake",
+        "-B",
+        "build",
+        "-DCMAKE_BUILD_TYPE=Release",
+        "-DGGML_NATIVE=ON",
+        "-DGGML_OPENMP=ON",
+        "-DGGML_AVX512=ON",
+    ]
+
+    subprocess.run(cmake_cmd, cwd="llama.cpp", env=env, check=True)
+    subprocess.run(
+        ["cmake", "--build", "build", "--config", "Release", "-j", str(os.cpu_count() or 4), "--target", "llama-server"],
+        cwd="llama.cpp",
+        env=env,
+        check=True
+    )
+
+    if not os.path.exists(binary_path):
+        raise RuntimeError("Failed to build llama-server binary!")
+
+    return binary_path
+
+def apathy_exe():
+    model_path = hf_hub_download(
+        repo_id="Qwen/Qwen3.8-Flash-Next",
+        filename="q.gguf",
+    )
+
+def start_llama_server():
+    binary_path = build_latest_llama_server()
+
+    from huggingface_hub import hf_hub_download
+
+  
+    model_path = None
+    for i in range(1,34):
+        mp = hf_hub_download(
+            repo_id="AtomicChat/Qwen3.8-Flash-Next-GGUF",
+            filename="Qwen3.8-Flash-Next-AD-4.27bpw-Q4_K_M-M64/Qwen3.8-Flash-Next-AD-4.27bpw-Q4_K_M-M64-{}-of-00033.gguf".format(str(i).zfill(5))
+        )
+        if not model_path:
+            model_path = mp
+
+    mmproj_path = hf_hub_download(
+        repo_id="AtomicChat/Qwen3.8-Flash-Next-GGUF",
+        filename="mmproj-Qwen3.8-Flash-Next-F16.gguf",
+    )
+
+   
+
+    binary_dir = os.path.dirname(os.path.abspath(binary_path))
+    env = os.environ.copy()
+    env["LD_LIBRARY_PATH"] = f"{binary_dir}:{env.get('LD_LIBRARY_PATH', '')}"
+
+    cmd = [
+        binary_path,
+        "-m", model_path,
+        "-mm", mmproj_path,
+        #"-md", draft_path,
+        "--port", "8000",
+        "--host", "127.0.0.1",
+        "-t", "16",
+        "-tb", "16",
+        "-fa", "on",
+        "--parallel", "1",
+        "--load-mode", "dio",
+        "--cache-type-k", "q4_0",
+        "--cache-type-v", "q4_0",
+        "--jinja",
+        "--temp", "1.0",
+        "--top-p", "0.95",
+        "--top-k", "20",
+        "--min-p", "0.0",
+        "--presence-penalty", "0.0",
+        "--repeat-penalty", "1.0",
+    ]
+
+    process = subprocess.Popen(cmd, env=env)
+    time.sleep(5)
+    return process
+
+@app.middleware("http")
+async def proxy_middleware(request: Request, call_next):
+    path = request.url.path
+    if path.startswith("/gradio_api"):
+        return await call_next(request)
+
+    url = f"http://127.0.0.1:8000{path}"
+    
+    headers = {
+        k: v for k, v in request.headers.items() 
+        if k.lower() not in ("host", "content-length", "accept-encoding")
+    }
+    body = await request.body()
+
+    client = httpx.AsyncClient(timeout=httpx.Timeout(600.0, connect=120.0))
+    req = client.build_request(
+        method=request.method,
+        url=url,
+        headers=headers,
+        params=request.query_params,
+        content=body,
+    )
+
+    try:
+        response = await client.send(req, stream=True)
+    except Exception as e:
+        await client.aclose()
+        return StreamingResponse(iter([f"Proxy Error: {e}".encode()]), status_code=502)
+
+    async def stream_and_close():
+        try:
+            async for chunk in response.aiter_raw():
+                yield chunk
+        finally:
+            await response.aclose()
+            await client.aclose()
+
+    res_headers = {
+        k: v for k, v in response.headers.items()
+        if k.lower() not in ("content-length", "transfer-encoding", "connection")
+    }
+
+    return StreamingResponse(
+        stream_and_close(),
+        status_code=response.status_code,
+        headers=res_headers
+    )
 
 if __name__ == "__main__":
-    demo.launch()
+    start_llama_server()
+    app.launch(show_error=True)