#!/usr/bin/env python3 """ Sharona B² 7B Jbliterated — Interactive inference with file-based context. Place M1.txt (guardrails) and M3.txt (knowledge/identity) in this directory, then run: python inference.py # bf16 (~18 GB VRAM) python inference.py --quantized # int8 (~10 GB VRAM) M1.txt controls what the model will and won't do. M3.txt provides identity and knowledge context. M2 is the live conversation — no file needed. See QUICKSTART.md for full usage instructions. """ import sys import argparse from pathlib import Path import torch SCRIPT_DIR = Path(__file__).resolve().parent sys.path.insert(0, str(SCRIPT_DIR)) from config import SharonaConfig from model.sharona_model import SharonaModel from tokenizer_config import ( get_tokenizer, PAD_TOKEN_ID, EOS_TOKEN_ID, M1_TOKEN_ID, M2_TOKEN_ID, M3_TOKEN_ID, ) def pad_to(ids, length, pad_id=PAD_TOKEN_ID): return ids[:length] if len(ids) >= length else ids + [pad_id] * (length - len(ids)) def load_context_file(path, token_id, max_length, tokenizer, label, required=False): """Load a context stream from a plain-text file.""" if not path.exists(): if required: print(f"\n ERROR: {label} file not found: {path}") print(f" Create {path.name} with your {label.lower()} text and try again.\n") sys.exit(1) print(f" {label}: not found — running without") return pad_to([token_id], max_length) text = path.read_text(encoding="utf-8").strip() if not text: if required: print(f"\n ERROR: {label} file is empty: {path}") sys.exit(1) print(f" {label}: empty file — running without") return pad_to([token_id], max_length) tokens = tokenizer.encode(text) used = min(len(tokens), max_length - 1) ids = pad_to([token_id] + list(tokens[:used]), max_length) print(f" {label}: {used} tokens from {path.name}") return ids def generate(model, tokenizer, m1_t, m3_t, prompt, device, max_tokens=512, temperature=0.7, top_p=0.9): q_ids = list(tokenizer.encode(prompt)) m2_ids = [M2_TOKEN_ID] + q_ids + [EOS_TOKEN_ID] generated = [] with torch.no_grad(): for _ in range(max_tokens): m2_t = torch.tensor([m2_ids + generated], device=device) out = model(context1_ids=m2_t, m1_ids=m1_t, m3_ids=m3_t) logits = out["logits"][:, -1, :].float() if temperature > 0: logits = logits / temperature sorted_logits, sorted_idx = torch.sort(logits, descending=True) cumprob = torch.softmax(sorted_logits, dim=-1).cumsum(dim=-1) mask = (cumprob - torch.softmax(sorted_logits, dim=-1)) >= top_p sorted_logits[mask] = float("-inf") probs = torch.softmax(sorted_logits, dim=-1) idx = torch.multinomial(probs, 1) nt = sorted_idx[0, idx[0, 0]].item() else: nt = logits.argmax().item() if nt in (PAD_TOKEN_ID, EOS_TOKEN_ID): break generated.append(nt) return tokenizer.decode(generated).strip() def main(): parser = argparse.ArgumentParser( description="Sharona B² 7B — Interactive Chat", formatter_class=argparse.RawDescriptionHelpFormatter, epilog="See QUICKSTART.md for full documentation.", ) parser.add_argument("--quantized", action="store_true", help="Load int8 model (~10 GB VRAM instead of ~18 GB)") parser.add_argument("--m1", type=str, default=str(SCRIPT_DIR / "M1.txt"), help="Path to M1 guardrails file (default: ./M1.txt)") parser.add_argument("--m3", type=str, default=str(SCRIPT_DIR / "M3.txt"), help="Path to M3 knowledge file (default: ./M3.txt)") parser.add_argument("--temperature", type=float, default=0.7, help="Sampling temperature (0 = greedy, default: 0.7)") parser.add_argument("--top-p", type=float, default=0.9, help="Nucleus sampling threshold (default: 0.9)") parser.add_argument("--max-tokens", type=int, default=512, help="Maximum tokens to generate per response") args = parser.parse_args() device = "cuda" if torch.cuda.is_available() else "cpu" weight_file = "model_q8.pt" if args.quantized else "model_bf16.pt" weight_path = SCRIPT_DIR / weight_file if not weight_path.exists(): print(f"Model weights not found: {weight_path}") print("Download from: https://huggingface.co/ApolloRaines/Sharona-B2-7B-Jbliterated") sys.exit(1) fmt = "int8" if args.quantized else "bf16" print(f"Loading Sharona B² 7B ({fmt}) on {device}...") config = SharonaConfig.sharona_7b() tokenizer = get_tokenizer() model = SharonaModel(config).to(dtype=torch.bfloat16, device=device) state = torch.load(str(weight_path), map_location=device, weights_only=False) model.load_state_dict(state, strict=True) model.eval() del state if device == "cuda": torch.cuda.empty_cache() total = sum(p.numel() for p in model.parameters()) print(f" Model: {total / 1e9:.2f}B parameters") # --- Load context streams --- m1_ids = load_context_file( Path(args.m1), M1_TOKEN_ID, config.m1_length, tokenizer, "M1 (guardrails)", required=True, ) m1_t = torch.tensor([m1_ids], device=device) m3_ids = load_context_file( Path(args.m3), M3_TOKEN_ID, config.m3_length, tokenizer, "M3 (knowledge)", required=False, ) m3_t = torch.tensor([m3_ids], device=device) print(f"\nReady. Type a message, or one of these commands:\n" f" /reload_m1 [path] Reload guardrails (default: {Path(args.m1).name})\n" f" /reload_m3 [path] Reload knowledge (default: {Path(args.m3).name})\n" f" /temp Change temperature\n" f" quit Exit\n") while True: try: prompt = input("You: ").strip() except (EOFError, KeyboardInterrupt): print() break if not prompt: continue if prompt.lower() in ("quit", "exit", "q"): break if prompt.startswith("/reload_m1"): parts = prompt.split(maxsplit=1) path = Path(parts[1]) if len(parts) > 1 else Path(args.m1) m1_ids = load_context_file( path, M1_TOKEN_ID, config.m1_length, tokenizer, "M1 (guardrails)", required=True, ) m1_t = torch.tensor([m1_ids], device=device) continue if prompt.startswith("/reload_m3"): parts = prompt.split(maxsplit=1) path = Path(parts[1]) if len(parts) > 1 else Path(args.m3) m3_ids = load_context_file( path, M3_TOKEN_ID, config.m3_length, tokenizer, "M3 (knowledge)", required=False, ) m3_t = torch.tensor([m3_ids], device=device) continue if prompt.startswith("/temp"): try: args.temperature = float(prompt.split()[1]) print(f" Temperature set to {args.temperature}") except (IndexError, ValueError): print(" Usage: /temp 0.7") continue answer = generate(model, tokenizer, m1_t, m3_t, prompt, device, max_tokens=args.max_tokens, temperature=args.temperature, top_p=args.top_p) print(f"\nSharona: {answer}\n") print("Goodbye.") if __name__ == "__main__": main()