Switch inference to transformers + bnb-4bit + LoRA for ZeroGPU

llama.cpp cannot use ZeroGPU's PyTorch-only CUDA emulation. Replace it
with the QLoRA training-time config: pre-quantized 4-bit base
(unsloth/qwen2.5-coder-14b-instruct-bnb-4bit) + LoRA adapter
(build-small-hackathon/lfed-qwen2.5-coder-14b-sql-lora).

- model_inference.py: TransformersLLM wrapper, llama.cpp-compatible
call/response schema, TextIteratorStreamer streaming
- app.py: spaces.GPU(duration=120)
- requirements.txt: drop llama-cpp-python; add torch/transformers/
peft/bitsandbytes/accelerate/hf_transfer

app.py

@@ -3,7 +3,7 @@ app.py — Kasualdad LFED: Local-First Education Data Analytics.
 
 Thin Gradio controller. All logic lives in:
   - prompts.py         (system prompt, schema docs, few-shot examples)
-  - model_inference.py (llama.cpp wrapper, SQL generation + streaming)
+  - model_inference.py (transformers + LoRA wrapper, SQL generation + streaming)
   - data_engine.py     (DuckDB lifecycle, schema seeding, execution guard)
 """
 
@@ -12,7 +12,7 @@ import gradio as gr
 # spaces.GPU is only available on HF Spaces — use a no-op locally
 try:
     import spaces
-    _gpu_decorator = spaces.GPU
+    _gpu_decorator = spaces.GPU(duration=120)
 except ImportError:
     _gpu_decorator = lambda fn: fn  # no-op for local dev
 
@@ -37,7 +37,10 @@ if not _pq_found:
     _pq_out = _parquet_dirs[0] if _parquet_dirs[0].exists() else _parquet_dirs[1]
     export_parquet(_pq_out)
 
-print("🦙 Loading model...")
+# Load the model at startup. On ZeroGPU, bitsandbytes/transformers go through
+# PyTorch, so the model loads onto the emulated CUDA device here and runs on
+# the real GPU inside @spaces.GPU. (llama.cpp could not do this.)
+print("🤖 Loading model (Qwen2.5-Coder-14B bnb-4bit + LoRA)...")
 llm = load_model()
 print("✅ Ready.")
 

model_inference.py

@@ -1,11 +1,25 @@
 """
-model_inference.py — llama.cpp wrapper for local SQL generation.
+model_inference.py — transformers + PEFT wrapper for local SQL generation.
+
+ZeroGPU-compatible: uses PyTorch (transformers + bitsandbytes 4-bit), which is
+the only CUDA path supported by HF Spaces ZeroGPU. The previous llama.cpp
+backend could not access ZeroGPU's PyTorch-only CUDA emulation.
+
+Model = pre-quantized 4-bit base (unsloth/qwen2.5-coder-14b-instruct-bnb-4bit)
+      + LoRA adapter (build-small-hackathon/lfed-qwen2.5-coder-14b-sql-lora)
+
+This is exactly the configuration the model was QLoRA fine-tuned in.
 
 Handles:
-  - Model loading (lazy, cached) from local GGUF
+  - Model loading (lazy, cached, thread-safe)
   - generate_sql(): prompt → raw text → cleaned SQL
-  - generate_sql_streaming(): yields tokens for Gradio stream=True
-  - Model download from Hugging Face Hub (fallback if not cached locally)
+  - generate_sql_streaming(): yields accumulated text for Gradio stream=True
+
+The loaded object (`TransformersLLM`) is callable with the same signature and
+response schema as llama_cpp.Llama, so downstream code is backend-agnostic:
+
+    out = llm(prompt, max_tokens=256, stop=[...], temperature=0.0)
+    text = out["choices"][0]["text"]
 """
 
 from __future__ import annotations
@@ -13,156 +27,146 @@ from __future__ import annotations
 import os
 import time
 import threading
-from pathlib import Path
 from typing import Generator, Optional
 
-os.environ["HF_HUB_ENABLE_HF_TRANSFER"] = "1"
-
-# ── CUDA preload ────────────────────────────────────────────────────────
-# llama-cpp-python's libllama.so (CUDA build) needs libcudart, libcublas,
-# libcublasLt, and possibly others at dlopen time.  Setting LD_LIBRARY_PATH
-# via os.environ does NOT help — the dynamic linker reads it once at process
-# start.  Instead we preload every lib*.so* we can find (system CUDA paths +
-# pip-installed nvidia-*-cu12 packages) with RTLD_GLOBAL so they're already
-# resident when libllama.so loads.
-
-import ctypes
-import sys
-
-_CUDA_PRELOADED = 0
-
-
-def _preload_dir(lib_dir: str) -> int:
-    """Preload every lib*.so* in *lib_dir* with RTLD_GLOBAL.  Returns count."""
-    n = 0
-    if not os.path.isdir(lib_dir):
-        return n
-    for _f in sorted(os.listdir(lib_dir)):
-        if not (_f.startswith("lib") and ".so" in _f):
-            continue
-        _full = os.path.join(lib_dir, _f)
-        if not os.path.isfile(_full):
-            continue
-        try:
-            ctypes.CDLL(_full, mode=ctypes.RTLD_GLOBAL)
-            n += 1
-        except OSError:
-            pass
-    return n
-
-
-# 1. System CUDA installs
-for _p in [
-    "/usr/local/cuda/lib64",
-    "/usr/local/cuda-12.1/lib64",
-    "/usr/local/cuda-12.4/lib64",
-]:
-    _CUDA_PRELOADED += _preload_dir(_p)
-
-# 2. Pip-installed nvidia-*-cu12 packages (common on HF Spaces)
-for _sp in sys.path:
-    _nvidia_root = os.path.join(_sp, "nvidia")
-    if not os.path.isdir(_nvidia_root):
-        continue
-    for _pkg in os.listdir(_nvidia_root):
-        _CUDA_PRELOADED += _preload_dir(os.path.join(_nvidia_root, _pkg, "lib"))
-
-if _CUDA_PRELOADED:
-    print(f"🔧 Preloaded {_CUDA_PRELOADED} CUDA shared librar{'y' if _CUDA_PRELOADED == 1 else 'ies'}")
-else:
-    print("ℹ️  No CUDA libraries found — if GPU is available install CUDA 12 or nvidia-*-cu12 pip packages.")
-
-# ── Imports ────────────────────────────────────────────────────────────
-
-from huggingface_hub import hf_hub_download
-
-try:
-    from llama_cpp import Llama
-except (OSError, RuntimeError) as e:
-    _msg = str(e)
-    if any(_kw in _msg.lower() for _kw in ("libcuda", "libcudart", "libcublas", "libcusparse", "libnv", "cudart", "cublas")):
-        raise RuntimeError(
-            "Failed to load llama-cpp-python — CUDA runtime not found.\n"
-            "If this is a CPU-only machine, install the CPU wheel:\n"
-            "  pip uninstall llama-cpp-python -y && pip install llama-cpp-python\n"
-            f"Original error: {e}"
-        ) from e
-    raise
+os.environ.setdefault("HF_HUB_ENABLE_HF_TRANSFER", "1")
 
 from prompts import build_prompt
 
 
 # ── Model configuration ────────────────────────────────────────────────
 
-# Cache: try 14B (production), then 7B (dev), then download fallback
-_LOCAL_14B_PATH = os.path.expanduser(
-    "~/.cache/huggingface/hub/models--build-small-hackathon--lfed-qwen2.5-coder-14b-sql-gguf"
-    "/snapshots/a46c7209055a9447b8e3a85a714d3c662d8c6149/lfed-qwen2.5-coder-14b-sql-Q4_K_M.gguf"
-)
-_LOCAL_7B_PATH = os.path.expanduser(
-    "~/.cache/huggingface/hub/models--build-small-hackathon--lfed-qwen2.5-coder-7b-sql-gguf"
-    "/snapshots/2a171f8745823d1b8c03e7467d046bcfedea7fa0/lfed-qwen2.5-coder-7b-sql-Q4_K_M.gguf"
-)
-
-# Cascade: 14B → 7B → old explicit path
-LOCAL_MODEL_PATH = (
-    _LOCAL_14B_PATH if os.path.exists(_LOCAL_14B_PATH)
-    else _LOCAL_7B_PATH if os.path.exists(_LOCAL_7B_PATH)
-    else "/tmp/lfed-models/qwen/Qwen2.5-Coder-7B-Instruct.Q4_K_M.gguf"
-)
-
-# HF Hub backup (download if nothing cached locally)
-HF_REPO_ID = "build-small-hackathon/lfed-qwen2.5-coder-14b-sql-gguf"
-HF_MODEL_FILE = "lfed-qwen2.5-coder-14b-sql-Q4_K_M.gguf"
-
-# Inference defaults
-_ZERO_GPU = os.environ.get("SPACES_ZERO_GPU", "").lower() == "true"
-
-DEFAULT_N_CTX = 4096
-DEFAULT_N_THREADS = 4 if _ZERO_GPU else 2  # Zero GPU: fewer threads needed with GPU offload
-DEFAULT_N_GPU_LAYERS = -1  # Always GPU offload; Zero GPU emulation handles module-level CUDA
+BASE_MODEL_4BIT = "unsloth/qwen2.5-coder-14b-instruct-bnb-4bit"
+ADAPTER_REPO = "build-small-hackathon/lfed-qwen2.5-coder-14b-sql-lora"
+
+# Override via env for local dev (e.g. a smaller model on a Mac)
+BASE_MODEL_4BIT = os.environ.get("LFED_BASE_MODEL", BASE_MODEL_4BIT)
+ADAPTER_REPO = os.environ.get("LFED_ADAPTER_REPO", ADAPTER_REPO)
+
 DEFAULT_MAX_TOKENS = 256
 DEFAULT_TEMPERATURE = 0.0
 STOP_SEQUENCES = ["\n\n", "Question:", "User:", "<|im_end|>", "<|im_start|>"]
 
 # Thread-safe model cache
 _lock = threading.Lock()
-_llm: Optional[Llama] = None
+_llm: Optional["TransformersLLM"] = None
 
 
-# ── Model loading ──────────────────────────────────────────────────────
+# ── llama.cpp-compatible wrapper ───────────────────────────────────────
 
-def _resolve_model_path() -> str:
-    """Find the GGUF model — local cache first, then download from Hub."""
-    local = Path(LOCAL_MODEL_PATH)
-    if local.exists():
-        print(f"📦 Using cached model: {local}")
-        return str(local)
-
-    print(f"⬇️  Local model not found, downloading from HF Hub: {HF_REPO_ID}/{HF_MODEL_FILE}")
-    path = hf_hub_download(repo_id=HF_REPO_ID, filename=HF_MODEL_FILE)
-    print(f"✅ Downloaded to: {path}")
-    return path
-
-
-def load_model(
-    model_path: str | None = None,
-    n_ctx: int = DEFAULT_N_CTX,
-    n_threads: int = DEFAULT_N_THREADS,
-    verbose: bool = False,
-) -> Llama:
-    """
-    Load the llama.cpp model. Thread-safe; caches the model globally.
+class TransformersLLM:
+    """Callable wrapper around transformers generate() that mimics the
+    llama_cpp.Llama response schema used by the rest of the app."""
 
-    Args:
-        model_path: Path to GGUF file. Auto-resolves if None.
-        n_ctx: Context window size.
-        n_threads: CPU threads for inference.
-        verbose: Print llama.cpp diagnostics.
+    def __init__(self, base_model: str = BASE_MODEL_4BIT, adapter: str = ADAPTER_REPO):
+        import torch
+        from transformers import AutoModelForCausalLM, AutoTokenizer
+        from peft import PeftModel
 
-    Returns:
-        Loaded Llama instance.
-    """
+        self.torch = torch
+        use_cuda = torch.cuda.is_available()
+        print(f"🤖 Loading base model: {base_model} (cuda={use_cuda})")
+
+        # Tokenizer comes from the adapter repo (carries the fine-tune's
+        # chat template); falls back to the base model.
+        try:
+            self.tokenizer = AutoTokenizer.from_pretrained(adapter)
+        except Exception:
+            self.tokenizer = AutoTokenizer.from_pretrained(base_model)
+
+        load_kwargs = {"low_cpu_mem_usage": True}
+        if use_cuda:
+            # Pre-quantized bnb-4bit checkpoint: no BitsAndBytesConfig needed.
+            load_kwargs["device_map"] = "auto"
+            load_kwargs["torch_dtype"] = torch.bfloat16
+        else:
+            # CPU/MPS dev fallback — bitsandbytes requires CUDA. Expect this
+            # only with LFED_BASE_MODEL pointing at a small fp16 model.
+            load_kwargs["torch_dtype"] = torch.float32
+
+        model = AutoModelForCausalLM.from_pretrained(base_model, **load_kwargs)
+
+        if adapter:
+            print(f"🔗 Applying LoRA adapter: {adapter}")
+            model = PeftModel.from_pretrained(model, adapter)
+
+        model.eval()
+        self.model = model
+
+    # -- helpers --------------------------------------------------------
+
+    def _truncate_on_stop(self, text: str, stop: list[str] | None) -> tuple[str, bool]:
+        if not stop:
+            return text, False
+        cut = len(text)
+        hit = False
+        for s in stop:
+            idx = text.find(s)
+            if idx != -1 and idx < cut:
+                cut = idx
+                hit = True
+        return text[:cut], hit
+
+    def _gen_kwargs(self, max_tokens: int, temperature: float) -> dict:
+        kwargs = {
+            "max_new_tokens": max_tokens,
+            "pad_token_id": self.tokenizer.pad_token_id or self.tokenizer.eos_token_id,
+        }
+        if temperature and temperature > 0:
+            kwargs.update(do_sample=True, temperature=temperature)
+        else:
+            kwargs.update(do_sample=False)
+        return kwargs
+
+    # -- llama.cpp-style call -------------------------------------------
+
+    def __call__(
+        self,
+        prompt: str,
+        max_tokens: int = DEFAULT_MAX_TOKENS,
+        stop: list[str] | None = None,
+        temperature: float = DEFAULT_TEMPERATURE,
+        echo: bool = False,
+        stream: bool = False,
+    ):
+        if stream:
+            return self._stream(prompt, max_tokens, stop, temperature)
+
+        inputs = self.tokenizer(prompt, return_tensors="pt").to(self.model.device)
+        with self.torch.inference_mode():
+            output_ids = self.model.generate(
+                **inputs, **self._gen_kwargs(max_tokens, temperature)
+            )
+        new_ids = output_ids[0][inputs["input_ids"].shape[1]:]
+        text = self.tokenizer.decode(new_ids, skip_special_tokens=True)
+        text, _ = self._truncate_on_stop(text, stop)
+        return {"choices": [{"text": text}]}
+
+    def _stream(
+        self,
+        prompt: str,
+        max_tokens: int,
+        stop: list[str] | None,
+        temperature: float,
+    ) -> Generator[dict, None, None]:
+        from transformers import TextIteratorStreamer
+
+        inputs = self.tokenizer(prompt, return_tensors="pt").to(self.model.device)
+        streamer = TextIteratorStreamer(
+            self.tokenizer, skip_prompt=True, skip_special_tokens=True
+        )
+        kwargs = dict(inputs, streamer=streamer, **self._gen_kwargs(max_tokens, temperature))
+        thread = threading.Thread(target=self.model.generate, kwargs=kwargs)
+        thread.start()
+        for piece in streamer:
+            yield {"choices": [{"text": piece}]}
+        thread.join()
+
+
+# ── Model loading ──────────────────────────────────────────────────────
+
+def load_model(verbose: bool = False) -> TransformersLLM:
+    """Load the model (base 4-bit + LoRA). Thread-safe global singleton."""
     global _llm
 
     if _llm is not None:
@@ -172,29 +176,13 @@ def load_model(
         if _llm is not None:  # Double-check after acquiring lock
             return _llm
 
-        if model_path is None:
-            model_path = _resolve_model_path()
-
         t0 = time.time()
-        print(f"🦙 Loading model: {model_path}")
-
-        # Zero GPU uses CUDA emulation at module level (startup) and real GPU
-        # inside @spaces.GPU.  Loading with GPU layers at startup is the
-        # recommended pattern — Zero GPU handles the context switch.
-        n_gpu = DEFAULT_N_GPU_LAYERS
-        _llm = Llama(
-            model_path=model_path,
-            n_ctx=n_ctx,
-            n_threads=n_threads,
-            n_gpu_layers=n_gpu,
-            verbose=verbose,
-        )
-
-        print(f"✅ Model loaded in {time.time() - t0:.1f}s (n_gpu_layers={n_gpu}, threads={n_threads})")
+        _llm = TransformersLLM()
+        print(f"✅ Model loaded in {time.time() - t0:.1f}s")
         return _llm
 
 
-def get_model() -> Llama | None:
+def get_model() -> TransformersLLM | None:
     """Return the cached model, or None if not loaded."""
     return _llm
 
@@ -203,7 +191,7 @@ def get_model() -> Llama | None:
 
 def generate_sql(
     user_question: str,
-    llm: Llama | None = None,
+    llm=None,
     max_tokens: int = DEFAULT_MAX_TOKENS,
     temperature: float = DEFAULT_TEMPERATURE,
     schema: dict | None = None,
@@ -211,15 +199,8 @@ def generate_sql(
     """
     Generate SQL from a natural-language question.
 
-    Args:
-        user_question: The admin's question in plain English.
-        llm: Llama instance (uses cached global if None).
-        max_tokens: Max tokens to generate.
-        temperature: Sampling temperature (0 = deterministic).
-        schema: Table schema dict for prompt context.
-
     Returns:
-        (raw_output, prompt) tuple — raw_output includes ```sql``` wrapping.
+        (raw_output, prompt) tuple — raw_output may include ```sql``` wrapping.
     """
     if llm is None:
         llm = get_model()
@@ -245,26 +226,14 @@ def generate_sql(
 
 def generate_sql_streaming(
     user_question: str,
-    llm: Llama | None = None,
+    llm=None,
     max_tokens: int = DEFAULT_MAX_TOKENS,
     temperature: float = DEFAULT_TEMPERATURE,
     schema: dict | None = None,
 ) -> Generator[str, None, None]:
     """
-    Stream SQL tokens for real-time Gradio display.
-
-    Accumulates tokens and yields the full text so far on each chunk.
-    Stops when a stop sequence appears in the accumulated text.
-
-    Args:
-        user_question: The admin's question in plain English.
-        llm: Llama instance (uses cached global if None).
-        max_tokens: Max tokens to generate.
-        temperature: Sampling temperature.
-        schema: Table schema dict for prompt context.
-
-    Yields:
-        The full accumulated SQL text so far (for display replacement).
+    Stream SQL for real-time Gradio display. Yields the full accumulated
+    text so far on each chunk; stops when a stop sequence appears.
     """
     if llm is None:
         llm = get_model()

requirements.txt

@@ -2,4 +2,9 @@ spaces
 gradio>=6.15.0
 duckdb==1.5.3
 huggingface_hub>=0.26.0
-llama-cpp-python
+hf_transfer
+torch
+transformers>=4.46.0
+peft>=0.13.0
+bitsandbytes>=0.44.0
+accelerate>=1.0.0

tests/test_model_inference.py

@@ -147,22 +147,20 @@ class TestModelSingleton:
         assert model_inference.get_model() is None
 
     def test_load_model_caches_instance(self):
-        """Mock Llama to avoid loading a real 4GB model."""
+        """Mock TransformersLLM to avoid loading a real 14B model."""
         import model_inference
 
         # Save original
         original = model_inference._llm
 
-        with patch("model_inference.Llama") as mock_llama_class:
+        with patch("model_inference.TransformersLLM") as mock_llama_class:
             mock_llama = MagicMock()
             mock_llama_class.return_value = mock_llama
 
             # Reset cache
             model_inference._llm = None
 
-            # Also mock _resolve_model_path to avoid file checks
-            with patch("model_inference._resolve_model_path", return_value="/fake/path.gguf"):
-                result = model_inference.load_model()
+            result = model_inference.load_model()
 
             assert result is mock_llama
             assert model_inference.get_model() is mock_llama
@@ -170,7 +168,7 @@ class TestModelSingleton:
             # Second call returns cached
             result2 = model_inference.load_model()
             assert result2 is mock_llama
-            # Llama constructor should only be called once
+            # Constructor should only be called once
             mock_llama_class.assert_called_once()
 
         # Restore original