Add FP8 (dynamic) quantization for Mellum2.1-12B-A2.5B-Thinking

README.md

@@ -0,0 +1,68 @@
+---
+license: other
+base_model: JetBrains/Mellum2.1-12B-A2.5B-Thinking
+base_model_relation: quantized
+library_name: transformers
+pipeline_tag: text-generation
+tags:
+- fp8
+- compressed-tensors
+- vllm
+- quantized
+quantized_by: liodon-ai
+---
+
+# Mellum2.1-12B-A2.5B-Thinking — FP8 (dynamic)
+
+FP8 quantization of [JetBrains/Mellum2.1-12B-A2.5B-Thinking](https://huggingface.co/JetBrains/Mellum2.1-12B-A2.5B-Thinking), published by [Liodon AI](https://huggingface.co/liodon-ai).
+
+Quantized with [llm-compressor](https://github.com/vllm-project/llm-compressor) using the
+`FP8_DYNAMIC` scheme: weights are cast to FP8 (E4M3) per-channel ahead of time, activations are
+quantized to FP8 dynamically per-token at inference time. No calibration dataset is needed for this
+scheme, so the quantized weights are numerically just a direct cast of the original — no calibration-set
+bias to worry about. `lm_head` is left unquantized (standard practice — negligible size, disproportionate
+quality impact if quantized).
+
+Original size: 24.3 GB → Quantized: 12.6 GB.
+
+## Quick Start
+
+**vLLM**
+```bash
+vllm serve liodon-ai/Mellum2.1-12B-A2.5B-Thinking-FP8
+```
+
+**Text Generation Inference (TGI)**
+```bash
+docker run --gpus all -p 8080:80 ghcr.io/huggingface/text-generation-inference \
+    --model-id liodon-ai/Mellum2.1-12B-A2.5B-Thinking-FP8
+```
+
+**SGLang**
+```bash
+python -m sglang.launch_server --model-path liodon-ai/Mellum2.1-12B-A2.5B-Thinking-FP8
+```
+
+FP8 execution requires an NVIDIA GPU with compute capability ≥ 8.9 (Ada/Hopper/Blackwell — RTX 40-series,
+L4/L40S, H100/H200, B100/B200/GB10). On older GPUs, vLLM/TGI will dequantize to run, which loses the
+speed/memory benefit.
+
+## Source
+
+- **Model**: [JetBrains/Mellum2.1-12B-A2.5B-Thinking](https://huggingface.co/JetBrains/Mellum2.1-12B-A2.5B-Thinking)
+- **License**: other
+
+## Citation
+
+```bibtex
+@misc{liodonai_mellum2_1_12b_a2_5b_thinking_fp8,
+  title        = {Mellum2.1-12B-A2.5B-Thinking — FP8},
+  author       = {{Liodon AI}},
+  year         = {2026},
+  howpublished = {\url{https://huggingface.co/liodon-ai/Mellum2.1-12B-A2.5B-Thinking-FP8}},
+  note         = {FP8 (dynamic) quantization of JetBrains/Mellum2.1-12B-A2.5B-Thinking}
+}
+```
+
+---
+*Quantized by [Liodon AI](https://huggingface.co/liodon-ai)*

chat_template.jinja

@@ -0,0 +1,103 @@
+{%- macro normalize_content(content) -%}
+    {%- if content is string -%}
+        {{- content -}}
+    {%- elif content is iterable and content is not mapping -%}
+        {%- set ns_c = namespace(text='') -%}
+        {%- for part in content -%}
+            {%- if part is mapping -%}
+                {%- if part.type == 'text' -%}
+                    {%- set ns_c.text = ns_c.text + (part.text or '') -%}
+                {%- elif part.type == 'tool-result' and part.output is mapping and part.output.type == 'text' -%}
+                    {%- set ns_c.text = ns_c.text + (part.output.value or '') -%}
+                {%- endif -%}
+            {%- endif -%}
+        {%- endfor -%}
+        {{- ns_c.text -}}
+    {%- endif -%}
+{%- endmacro -%}
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0].role == 'system' %}
+        {{- normalize_content(messages[0].content) + '\n\n' }}
+    {%- endif %}
+    {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0].role == 'system' %}
+        {{- '<|im_start|>system\n' + normalize_content(messages[0].content) + '<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+    {%- set index = (messages|length - 1) - loop.index0 %}
+    {%- set msg_content = normalize_content(message.content) %}
+    {%- if ns.multi_step_tool and message.role == "user" and not(msg_content.startswith('<tool_response>') and msg_content.endswith('</tool_response>')) %}
+        {%- set ns.multi_step_tool = false %}
+        {%- set ns.last_query_index = index %}
+    {%- endif %}
+{%- endfor %}
+{%- for message in messages %}
+    {%- set content = normalize_content(message.content) %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+        {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {%- set reasoning_content = '' %}
+        {%- if message.reasoning_content is string %}
+            {%- set reasoning_content = message.reasoning_content %}
+        {%- else %}
+            {%- if '</think>' in content %}
+                {%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
+                {%- set content = content.split('</think>')[-1].lstrip('\n') %}
+            {%- endif %}
+        {%- endif %}
+        {%- if loop.index0 > ns.last_query_index %}
+            {%- if reasoning_content %}
+                {{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content.strip('\n') + '\n</think>\n\n' + content.lstrip('\n') }}
+            {%- else %}
+                {{- '<|im_start|>' + message.role + '\n' + content }}
+            {%- endif %}
+        {%- else %}
+            {{- '<|im_start|>' + message.role + '\n' + content }}
+        {%- endif %}
+        {%- if message.tool_calls %}
+            {%- for tool_call in message.tool_calls %}
+                {%- if (loop.first and content) or (not loop.first) %}
+                    {{- '\n' }}
+                {%- endif %}
+                {%- if tool_call.function %}
+                    {%- set tool_call = tool_call.function %}
+                {%- endif %}
+                {{- '<tool_call>\n{"name": "' }}
+                {{- tool_call.name }}
+                {{- '", "arguments": ' }}
+                {%- if tool_call.arguments is string %}
+                    {{- tool_call.arguments }}
+                {%- else %}
+                    {{- tool_call.arguments | tojson }}
+                {%- endif %}
+                {{- '}\n</tool_call>' }}
+            {%- endfor %}
+        {%- endif %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+    {%- if enable_thinking is defined and enable_thinking is false %}
+        {{- '<think>\n\n</think>\n\n' }}
+    {%- endif %}
+{%- endif %}

config.json

@@ -0,0 +1,188 @@
+{
+  "architectures": [
+    "MellumForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 0,
+  "dtype": "bfloat16",
+  "eos_token_id": 28,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 2304,
+  "initializer_range": 0.02,
+  "intermediate_size": 7168,
+  "layer_types": [
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention"
+  ],
+  "max_position_embeddings": 131072,
+  "max_window_layers": 0,
+  "mlp_layer_types": [
+    "sparse",
+    "sparse",
+    "sparse",
+    "sparse",
+    "sparse",
+    "sparse",
+    "sparse",
+    "sparse",
+    "sparse",
+    "sparse",
+    "sparse",
+    "sparse",
+    "sparse",
+    "sparse",
+    "sparse",
+    "sparse",
+    "sparse",
+    "sparse",
+    "sparse",
+    "sparse",
+    "sparse",
+    "sparse",
+    "sparse",
+    "sparse",
+    "sparse",
+    "sparse",
+    "sparse",
+    "sparse"
+  ],
+  "model_type": "mellum",
+  "moe_intermediate_size": 896,
+  "norm_topk_prob": true,
+  "num_attention_heads": 32,
+  "num_experts_per_tok": 8,
+  "num_hidden_layers": 28,
+  "num_key_value_heads": 4,
+  "num_local_experts": 64,
+  "output_router_logits": false,
+  "pad_token_id": null,
+  "quantization_config": {
+    "config_groups": {
+      "group_0": {
+        "format": "float-quantized",
+        "input_activations": {
+          "actorder": null,
+          "block_structure": null,
+          "dynamic": true,
+          "group_size": null,
+          "num_bits": 8,
+          "observer": null,
+          "observer_kwargs": {},
+          "scale_dtype": null,
+          "strategy": "token",
+          "symmetric": true,
+          "type": "float",
+          "zp_dtype": null
+        },
+        "output_activations": null,
+        "targets": [
+          "Linear"
+        ],
+        "weights": {
+          "actorder": null,
+          "block_structure": null,
+          "dynamic": false,
+          "group_size": null,
+          "num_bits": 8,
+          "observer": "memoryless_minmax",
+          "observer_kwargs": {},
+          "scale_dtype": null,
+          "strategy": "channel",
+          "symmetric": true,
+          "type": "float",
+          "zp_dtype": null
+        }
+      }
+    },
+    "format": "float-quantized",
+    "global_compression_ratio": null,
+    "ignore": [
+      "model.layers.0.mlp.gate",
+      "model.layers.1.mlp.gate",
+      "model.layers.2.mlp.gate",
+      "model.layers.3.mlp.gate",
+      "model.layers.4.mlp.gate",
+      "model.layers.5.mlp.gate",
+      "model.layers.6.mlp.gate",
+      "model.layers.7.mlp.gate",
+      "model.layers.8.mlp.gate",
+      "model.layers.9.mlp.gate",
+      "model.layers.10.mlp.gate",
+      "model.layers.11.mlp.gate",
+      "model.layers.12.mlp.gate",
+      "model.layers.13.mlp.gate",
+      "model.layers.14.mlp.gate",
+      "model.layers.15.mlp.gate",
+      "model.layers.16.mlp.gate",
+      "model.layers.17.mlp.gate",
+      "model.layers.18.mlp.gate",
+      "model.layers.19.mlp.gate",
+      "model.layers.20.mlp.gate",
+      "model.layers.21.mlp.gate",
+      "model.layers.22.mlp.gate",
+      "model.layers.23.mlp.gate",
+      "model.layers.24.mlp.gate",
+      "model.layers.25.mlp.gate",
+      "model.layers.26.mlp.gate",
+      "model.layers.27.mlp.gate",
+      "lm_head"
+    ],
+    "kv_cache_scheme": null,
+    "quant_method": "compressed-tensors",
+    "quantization_status": "compressed",
+    "sparsity_config": {},
+    "transform_config": {},
+    "version": "0.18.0"
+  },
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "full_attention": {
+      "attention_factor": 1.2772588722239782,
+      "beta_fast": 32.0,
+      "beta_slow": 1.0,
+      "factor": 16.0,
+      "original_max_position_embeddings": 8192,
+      "rope_theta": 500000.0,
+      "rope_type": "yarn"
+    },
+    "sliding_attention": {
+      "rope_theta": 500000.0,
+      "rope_type": "default"
+    }
+  },
+  "router_aux_loss_coef": 0.001,
+  "sliding_window": 1024,
+  "tie_word_embeddings": false,
+  "transformers_version": "5.14.1",
+  "use_cache": true,
+  "use_sliding_window": true,
+  "vocab_size": 98304
+}

generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 0,
+  "eos_token_id": 28,
+  "transformers_version": "5.14.1"
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:097077a620a34c291af51acbebe48e8af0422e156d1f4b6bdd1f624d640309e3
+size 12623670464

recipe.yaml

@@ -0,0 +1,8 @@
+default_stage:
+  default_modifiers:
+    QuantizationModifier:
+      targets: [Linear]
+      ignore: [lm_head]
+      scheme: FP8_DYNAMIC
+      bypass_divisibility_checks: false
+      requires_calibration_data: false

tokenizer_config.json

@@ -0,0 +1,12 @@
+{
+  "backend": "tokenizers",
+  "bos_token": "<|endoftext|>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "is_local": false,
+  "local_files_only": false,
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "tokenizer_class": "TokenizersBackend",
+  "unk_token": "<|endoftext|>"
+}