Add MNN Q4 conversion for TokForge mobile inference

.gitattributes

@@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+llm.mnn filter=lfs diff=lfs merge=lfs -text
+llm.mnn.weight filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,98 @@
+---
+license: apache-2.0
+language:
+  - en
+pipeline_tag: text-generation
+base_model: mistralai/Mistral-7B-Instruct-v0.3
+tags:
+  - mnn
+  - mistral
+  - mobile
+  - on-device
+  - tokforge
+  - uncensored
+  - abliterated
+---
+
+# Mistral-7B-Instruct-v0.3-MNN
+
+Pre-converted [Mistral 7B Instruct v0.3](https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.3) in MNN format for on-device inference with [TokForge](https://tokforge.ai).
+
+> **Original model by [Mistral AI](https://huggingface.co/Mistral AI)** — converted to MNN Q4 for mobile deployment.
+
+## Model Details
+
+| | |
+|---|---|
+| **Architecture** | Mistral (sliding window attention, 32 layers, GQA) |
+| **Parameters** | 7B (4-bit quantized) |
+| **Format** | MNN (Alibaba Mobile Neural Network) |
+| **Quantization** | W4A16 (4-bit weights, block size 128) |
+| **Vocab** | 32,768 tokens |
+| **Source** | [mistralai/Mistral-7B-Instruct-v0.3](https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.3) |
+
+## Description
+
+Mistral AI's 7B Instruct v0.3 — the model that started the open-source LLM revolution. Updated to v0.3 with extended vocabulary and function calling support. One of the most popular and battle-tested open models. Excellent balance of quality and speed.
+
+## Files
+
+| File | Description |
+|------|-------------|
+| `llm.mnn` | Model computation graph |
+| `llm.mnn.weight` | Quantized weight data (Q4, block=128) |
+| `llm_config.json` | Model config with Jinja chat template |
+| `tokenizer.txt` | Tokenizer vocabulary |
+| `config.json` | MNN runtime config |
+
+## Usage with TokForge
+
+This model is optimized for **[TokForge](https://tokforge.ai)** — a free Android app for private, on-device LLM inference.
+
+1. Download [TokForge from the Play Store](https://tokforge.ai)
+2. Open the app → Models → Download this model
+3. Start chatting — runs 100% locally, no internet required
+
+### Recommended Settings
+
+| Setting | Value |
+|---------|-------|
+| Backend | OpenCL (Qualcomm) / Vulkan (MediaTek) / CPU (fallback) |
+| Precision | Low |
+| Threads | 4 |
+| Thinking | Off (or On for thinking-capable models) |
+
+
+
+## Performance
+
+Actual speed varies by device, thermal state, and generation length. Typical ranges for this model size:
+
+| Device | SoC | Backend | tok/s |
+|---|---|---|---|
+| RedMagic 11 Pro | SM8850 | OpenCL | ~15-18 tok/s |
+| Lenovo TB520FU | SM8650 | OpenCL | ~10-12 tok/s |
+
+## Attribution
+
+This is an MNN conversion of **[Mistral 7B Instruct v0.3](https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.3)** by **[Mistral AI](https://huggingface.co/Mistral AI)**. All credit for the model architecture, training, and fine-tuning goes to the original author(s). This conversion only changes the runtime format for mobile deployment.
+
+## Limitations
+
+- Intended for TokForge / MNN on-device inference on Android
+- This is a runtime bundle, not a standard Transformers training checkpoint
+- Quantization (Q4) may slightly reduce quality compared to the full-precision original
+- Abliterated/uncensored models have had safety filters removed — **use responsibly**
+
+## Community
+
+- **Website:** [tokforge.ai](https://tokforge.ai)
+- **Discord:** [Join our Discord](https://discord.gg/Acv3CBtfVm)
+- **GitHub:** [TokForge on GitHub](https://github.com/darkmaniac7/Elysium)
+
+## Export Details
+
+Converted using MNN's `llmexport` pipeline:
+```bash
+python llmexport.py --path mistralai/Mistral-7B-Instruct-v0.3 --export mnn --quant_bit 4 --quant_block 128
+```

config.json

@@ -0,0 +1,10 @@
+{
+    "llm_model": "llm.mnn",
+    "llm_weight": "llm.mnn.weight",
+    "backend_type": "cpu",
+    "thread_num": 4,
+    "precision": "low",
+    "memory": "low",
+    "sampler_type": "penalty",
+    "penalty": 1.1
+}

embeddings_bf16.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f6844d1e657509198ee0e494d9f8e09efc4f55ffcca197a8f54465c6b6405d96
+size 268435456

export_args.json

@@ -0,0 +1,42 @@
+{
+    "path": "/root/models/hf_convert_queue/Mistral-7B-Instruct-v0.3",
+    "type": null,
+    "tokenizer_path": "/root/models/hf_convert_queue/Mistral-7B-Instruct-v0.3",
+    "eagle_path": null,
+    "lora_path": null,
+    "gptq_path": null,
+    "dst_path": "/root/models/hf_uploads/Mistral-7B-Instruct-v0.3-MNN",
+    "verbose": false,
+    "test": null,
+    "export": "mnn",
+    "onnx_slim": false,
+    "quant_bit": 4,
+    "quant_block": 128,
+    "visual_quant_bit": null,
+    "visual_quant_block": null,
+    "lm_quant_bit": 4,
+    "lm_quant_block": 128,
+    "mnnconvert": "../../../build/MNNConvert",
+    "ppl": false,
+    "awq": false,
+    "hqq": false,
+    "omni": false,
+    "transformer_fuse": false,
+    "group_conv_native": false,
+    "smooth": false,
+    "sym": false,
+    "visual_sym": false,
+    "seperate_embed": true,
+    "lora_split": false,
+    "calib_data": null,
+    "act_bit": 16,
+    "embed_bit": 16,
+    "act_sym": false,
+    "quant_config": null,
+    "generate_for_npu": false,
+    "skip_weight": false,
+    "omni_epochs": 20,
+    "omni_lr": 0.005,
+    "omni_wd": 0.0001,
+    "tie_word_embeddings": false
+}

llm.mnn

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:335a0c12d911731a5a97664b0b0c842ec112ae088b39c44902237f3efccd4734
+size 557656

llm.mnn.weight

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f144b54f32593843aa5e347654afbcff65e97a53be492825d8f80e5ece9c9ee6
+size 4003500886

llm_config.json

@@ -0,0 +1,12 @@
+{
+    "model_type": "mistral",
+    "hidden_size": 4096,
+    "attention_mask": "float",
+    "attention_type": "full",
+    "is_mrope": false,
+    "jinja": {
+        "chat_template": "{%- if messages[0][\"role\"] == \"system\" %}\n    {%- set system_message = messages[0][\"content\"] %}\n    {%- set loop_messages = messages[1:] %}\n{%- else %}\n    {%- set loop_messages = messages %}\n{%- endif %}\n{%- if not tools is defined %}\n    {%- set tools = none %}\n{%- endif %}\n{%- set user_messages = loop_messages | selectattr(\"role\", \"equalto\", \"user\") | list %}\n\n{#- This block checks for alternating user/assistant messages, skipping tool calling messages #}\n{%- set ns = namespace() %}\n{%- set ns.index = 0 %}\n{%- for message in loop_messages %}\n    {%- if not (message.role == \"tool\" or message.role == \"tool_results\" or (message.tool_calls is defined and message.tool_calls is not none)) %}\n        {%- if (message[\"role\"] == \"user\") != (ns.index % 2 == 0) %}\n            {{- raise_exception(\"After the optional system message, conversation roles must alternate user/assistant/user/assistant/...\") }}\n        {%- endif %}\n        {%- set ns.index = ns.index + 1 %}\n    {%- endif %}\n{%- endfor %}\n\n{{- bos_token }}\n{%- for message in loop_messages %}\n    {%- if message[\"role\"] == \"user\" %}\n        {%- if tools is not none and (message == user_messages[-1]) %}\n            {{- \"[AVAILABLE_TOOLS] [\" }}\n            {%- for tool in tools %}\n                {%- set tool = tool.function %}\n                {{- '{\"type\": \"function\", \"function\": {' }}\n                {%- for key, val in tool.items() if key != \"return\" %}\n                    {%- if val is string %}\n                        {{- '\"' + key + '\": \"' + val + '\"' }}\n                    {%- else %}\n                        {{- '\"' + key + '\": ' + val|tojson }}\n                    {%- endif %}\n                    {%- if not loop.last %}\n                        {{- \", \" }}\n                    {%- endif %}\n                {%- endfor %}\n                {{- \"}}\" }}\n                {%- if not loop.last %}\n                    {{- \", \" }}\n                {%- else %}\n                    {{- \"]\" }}\n                {%- endif %}\n            {%- endfor %}\n            {{- \"[/AVAILABLE_TOOLS]\" }}\n            {%- endif %}\n        {%- if loop.last and system_message is defined %}\n            {{- \"[INST] \" + system_message + \"\\n\\n\" + message[\"content\"] + \"[/INST]\" }}\n        {%- else %}\n            {{- \"[INST] \" + message[\"content\"] + \"[/INST]\" }}\n        {%- endif %}\n    {%- elif message.tool_calls is defined and message.tool_calls is not none %}\n        {{- \"[TOOL_CALLS] [\" }}\n        {%- for tool_call in message.tool_calls %}\n            {%- set out = tool_call.function|tojson %}\n            {{- out[:-1] }}\n            {%- if not tool_call.id is defined or tool_call.id|length != 9 %}\n                {{- raise_exception(\"Tool call IDs should be alphanumeric strings with length 9!\") }}\n            {%- endif %}\n            {{- ', \"id\": \"' + tool_call.id + '\"}' }}\n            {%- if not loop.last %}\n                {{- \", \" }}\n            {%- else %}\n                {{- \"]\" + eos_token }}\n            {%- endif %}\n        {%- endfor %}\n    {%- elif message[\"role\"] == \"assistant\" %}\n        {{- \" \" + message[\"content\"]|trim + eos_token}}\n    {%- elif message[\"role\"] == \"tool_results\" or message[\"role\"] == \"tool\" %}\n        {%- if message.content is defined and message.content.content is defined %}\n            {%- set content = message.content.content %}\n        {%- else %}\n            {%- set content = message.content %}\n        {%- endif %}\n        {{- '[TOOL_RESULTS] {\"content\": ' + content|string + \", \" }}\n        {%- if not message.tool_call_id is defined or message.tool_call_id|length != 9 %}\n            {{- raise_exception(\"Tool call IDs should be alphanumeric strings with length 9!\") }}\n        {%- endif %}\n        {{- '\"call_id\": \"' + message.tool_call_id + '\"}[/TOOL_RESULTS]' }}\n    {%- else %}\n        {{- raise_exception(\"Only user and assistant roles are supported, with the exception of an initial optional system message!\") }}\n    {%- endif %}\n{%- endfor %}\n",
+        "bos": "<s>",
+        "eos": "</s>"
+    }
+}