Upload model.py with huggingface_hub

model.py

@@ -9,6 +9,7 @@ create_completion. Result: immediate response, no thinking.
 
 from __future__ import annotations
 
+import os
 import threading
 import time
 from pathlib import Path
@@ -80,7 +81,27 @@ def _load_model():
     try:
         from llama_cpp import Llama
 
-        _llm = Llama(model_path=str(gguf), n_ctx=2048, verbose=False)
+        # Thread count is the single biggest speed lever on CPU. Left to auto-
+        # detect, llama.cpp counts the *host's* cores inside the container and
+        # spawns far more threads than the Space actually has (HF cpu-basic =
+        # 2 vCPU), so they thrash and generation crawls. Pin it to the real
+        # quota on Spaces; use all cores locally. Override with LLAMA_THREADS.
+        env_threads = os.environ.get("LLAMA_THREADS")
+        if env_threads:
+            n_threads = int(env_threads)
+        elif os.environ.get("SPACE_ID"):
+            n_threads = 2  # HF cpu-basic
+        else:
+            n_threads = os.cpu_count() or 4
+
+        _llm = Llama(
+            model_path=str(gguf),
+            n_ctx=1536,
+            n_threads=n_threads,
+            n_batch=256,
+            verbose=False,
+        )
+        print(f"[model] llama.cpp loaded with n_threads={n_threads}")
         _using_mock = False
     except Exception as exc:
         print(f"[model] Could not load GGUF ({gguf}): {exc}")
@@ -151,7 +172,7 @@ def stream_broadcast(
     system_prompt: str,
     user_prompt: str = DEFAULT_USER_PROMPT,
     seed: int | None = None,
-    max_tokens: int = 220,
+    max_tokens: int = 160,
     temperature: float = 0.7,
 ):
     """Generator that yields the broadcast text token by token."""