Upload infer_gguf.py with huggingface_hub

infer_gguf.py

@@ -216,13 +216,6 @@ def apply_qlora_v2(model, target_modules=None, r=8, alpha=16, dropout=0.0):
     print(f" QLoRA applied: {qlora_params:,} LoRA params | trainable: {n:,}")
 
 
-# ─── V3 model (from model_tiny) ─────────────────────────────────────────
-
-def _load_v3_model():
-    from scripts.model_tiny import TinyModel as TinyModelV3
-    return TinyModelV3
-
-
 # ─── Tokenizer ──────────────────────────────────────────────────────────
 
 def load_tokenizer(ckpt_type=None):
@@ -252,17 +245,21 @@ def build_chat(system, user, tokenizer):
 
 def _detect_ckpt_type(state):
     keys = list(state.keys())
-    if any("rpw" in k or "vcr" in k for k in keys):
-        return "v3"
-    if any("qweight" in k for k in keys):
+    has_v3 = any("rpw" in k or "vcr" in k for k in keys)
+    has_v3_blocks = False
+    if any(k.startswith("blocks.") for k in keys):
+        blk_keys = [k.split(".")[1] for k in keys if k.startswith("blocks.")]
+        max_blk = max(int(b) for b in blk_keys) if blk_keys else 0
+        has_v3_blocks = max_blk >= 3
+    has_qlora = any("qweight" in k for k in keys)
+    if has_v3:
+        return "v3_qlora" if has_qlora else "v3"
+    if has_v3_blocks:
+        return "v3_qlora" if has_qlora else "v3"
+    if has_qlora:
         return "v2_qlora"
     if any("gate" in k for k in keys) or any("mlp.gate" in k for k in keys):
         return "v2_fp32"
-    if any(k.startswith("blocks.") for k in keys) and any("ln1.weight" in k for k in keys):
-        blk_keys = [k.split(".")[1] for k in keys if k.startswith("blocks.")]
-        max_blk = max(int(b) for b in blk_keys) if blk_keys else 0
-        if max_blk >= 3:
-            return "v3"
     return "v2_fp32"
 
 
@@ -333,9 +330,14 @@ def load_checkpoint(ckpt_path):
         model.eval()
         return model, ckpt_type
 
-    TV3 = _load_v3_model()
+    from scripts.model_tiny import TinyModel as TV3
+    from scripts.model_tiny import apply_qlora as apply_qlora_v3
+    is_qlora = "qlora" in ckpt_type
     model = TV3()
-    model.load_state_dict(state)
+    model.reset_weights()
+    if is_qlora:
+        model = apply_qlora_v3(model, r=8, alpha=16)
+    model.load_state_dict(state, strict=False)
     model.eval()
     return model, ckpt_type