Initial W4A16 G64 release

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

LICENSE

@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

README.md

@@ -0,0 +1,268 @@
+---
+license: apache-2.0
+license_link: https://ai.google.dev/gemma/docs/gemma_4_license
+base_model:
+  - Mitchins/gemma-4-26B-A4B-it-qat-q4_0-unquantized-uncensored-heretic
+base_model_relation: quantized
+pipeline_tag: image-text-to-text
+library_name: vllm
+tags:
+  - gemma4
+  - gemma
+  - vision-language
+  - multimodal
+  - quantized
+  - qat
+  - heretic
+  - uncensored
+  - abliterated
+  - w4a16
+  - int4
+  - g64
+  - compressed-tensors
+  - vllm
+  - marlin
+  - ampere
+  - rtx-3090
+  - conversational
+---
+
+# Gemma 4 26B-A4B-IT QAT/Heretic — W4A16 G64, BF16 vision
+
+This is a calibrated W4A16 (INT4 weight-only) quantization of [Mitchins/gemma-4-26B-A4B-it-qat-q4_0-unquantized-uncensored-heretic](https://huggingface.co/Mitchins/gemma-4-26B-A4B-it-qat-q4_0-unquantized-uncensored-heretic), a QAT/Heretic tuned Gemma 4 26B-A4B instruction checkpoint. It is **not** a stock, bit-identical conversion of the ordinary public Google IT checkpoint. The model remains a Gemma 4 26B-A4B instruction/multimodal model; useful-quants performed the W4A16 conversion and packaging, not the original model development or the parent tuning.
+
+The language-model linear and MoE expert weights are packed with `compressed-tensors` W4A16/G64; activations remain BF16. The vision tower, routers, embeddings, and LM head remain BF16/unquantized. The packed payload is 14.58 GiB (`model.safetensors`); the full repository is approximately 14.61 GiB including tokenizer and processor files.
+
+Text generation and real-image inference were validated on one RTX 3090 24 GB with vLLM, Ampere Marlin dense/MoE kernels, and CUDA graphs. A 131,072-token total sequence window (128K) was also validated on that reference system.
+
+G64 is deliberate: the relevant Gemma 4 26B-A4B projection dimensions (`intermediate_size=2112`, `moe_intermediate_size=704`) divide cleanly by 64 but not by 128.
+
+| Property | Value |
+|---|---|
+| Architecture | Gemma 4 26B-A4B MoE instruction/multimodal |
+| Immediate source | [`Mitchins/gemma-4-26B-A4B-it-qat-q4_0-unquantized-uncensored-heretic`](https://huggingface.co/Mitchins/gemma-4-26B-A4B-it-qat-q4_0-unquantized-uncensored-heretic) at `d62fa836581401beafd08fdd624927e31dcac6ee` |
+| Quantization | GPTQ-style W4A16, symmetric INT4, group size 64 |
+| Runtime format | `compressed-tensors` `pack-quantized` |
+| Precision retained | BF16 activations, vision tower, routers, embeddings, LM head |
+| Primary runtime | vLLM (validated on a pinned main commit; see [Runtime compatibility](#runtime-compatibility)) |
+| Reference GPU | RTX 3090 24 GB (Ampere) |
+| Validated operational window | 131,072 total tokens, including generated tokens |
+| Architectural context limit | 262,144 tokens |
+| License | Apache-2.0, inherited from the Gemma parent |
+
+## Base model and provenance
+
+The immediate parent is [Mitchins/gemma-4-26B-A4B-it-qat-q4_0-unquantized-uncensored-heretic](https://huggingface.co/Mitchins/gemma-4-26B-A4B-it-qat-q4_0-unquantized-uncensored-heretic), revision `d62fa836581401beafd08fdd624927e31dcac6ee`. That parent is an uncensored/abliterated Heretic derivative of the QAT-trained `google/gemma-4-26B-A4B-it-qat-q4_0-unquantized` lineage, itself based on [Google Gemma 4 26B-A4B-IT](https://huggingface.co/google/gemma-4-26B-A4B-it). The local export was frozen before conversion, including source-file SHA256 hashes, tokenizer, processor, chat template, generation configuration, and architecture configuration. See [`provenance/calibration-and-source.json`](provenance/calibration-and-source.json).
+
+The parent model's Apache-2.0 license and Gemma 4 license link apply. This quantization does not change the parent model's intended use, safety guidance, or license terms.
+
+## What is quantized
+
+The saved `quantization_config` is the source of truth:
+
+- `Linear` language-model and MoE expert weights: 4-bit signed integer weights, W4A16, group-wise G64, symmetric, static activation ordering, `memoryless_minmax` observer, `pack-quantized` storage.
+- Dense language-model linears and MoE experts are W4A16. The artifact audit found the expected packed expert tensors and finite weight-scale tensors.
+- The vision tower is retained BF16.
+- Routers and token/image embeddings are retained BF16/unquantized. This Gemma export has no separate `lm_head` tensor; output weights are tied through the BF16 token embedding.
+- No reduced-precision KV-cache profile is claimed or recommended here.
+
+The GPTQ modifier also used a 128-token block size and `dampening_frac=0.01`; the complete public-safe recipe is in [`provenance/calibration-and-source.json`](provenance/calibration-and-source.json).
+
+## Calibration
+
+Calibration was multimodal rather than text-only so the conversion observed both normal language traffic and the image-processing/template path.
+
+- **Text:** 512 frozen 512-token windows, deterministic seed `20260812`.
+  - 192 windows from `Salesforce/wikitext`, `wikitext-103-raw-v1` train.
+  - 192 from `HuggingFaceTB/smoltalk`, `openhermes-100k`.
+  - 128 from `HuggingFaceTB/smoltalk`, `apigen-80k`.
+  - Each source is deterministically shuffled; a valid 512-token span is sampled from each selected row. Conversational rows are rendered as role-labelled text before tokenization.
+- **Images:** 1,024 deterministically selected, SHA256-verified images from COCO 2017 validation. The source processor converts each to RGB and applies its native preprocessing.
+- **Mixed layout:** the first 512 calibration samples pair one frozen text window with one image; the remaining 512 use image-instruction prompts. Every selected COCO image and every frozen text window is used once.
+- **Calibration sequence cap:** 1,024 tokens, allowing the 512-token text windows and Gemma image soft tokens to coexist.
+
+The exact datasets, sample counts, seed, selection method, image hash-manifest digest, exclusions, and software versions are in [`provenance/calibration-and-source.json`](provenance/calibration-and-source.json). Calibration media are deliberately not included in this release.
+
+## Quality: held-out perplexity
+
+| Model | Perplexity | Delta vs. BF16 |
+|---|---:|---:|
+| Tuned QAT/Heretic BF16 parent | 388.150 | — |
+| W4A16-G64 | 432.318 | +44.168 / +11.38% |
+
+The result uses held-out `Salesforce/wikitext` `wikitext-2-raw-v1` **test** text, which was not used for calibration. Both rows use the same frozen tokenizer, four contiguous non-empty 512-token test-corpus windows (2,044 scored next-token positions), next-token NLL arithmetic, and vLLM `prompt_logprobs=1` scorer. BF16 is executed with vLLM CPU offload because the parent checkpoint exceeds a 24 GB GPU. See [`eval/evaluate_gemma4_ppl_vllm.py`](eval/evaluate_gemma4_ppl_vllm.py) and [`benchmarks/perplexity.json`](benchmarks/perplexity.json).
+
+This is a compact quantization sanity evaluation, not a replacement for task-specific evaluation of the tuned parent.
+
+## Verified functional gates
+
+The final artifact was fresh-reloaded and passed:
+
+- deterministic short text generation;
+- real COCO image generation through the Gemma 4 processor/template path;
+- Marlin W4A16 dense-linear selection;
+- Marlin W4A16 MoE selection;
+- CUDA graph capture, without eager mode;
+- finite weight-scale and packed-tensor audit.
+
+For images, use the Gemma 4 processor or OpenAI-compatible multimodal request format. The source chat template emits Gemma's actual `<|image|>` placeholder semantics; do **not** manually write a raw `<image>` token into a text prompt.
+
+Gemma 4 supports video in its broader architecture, but video was neither required nor validated for this image-focused release profile. Unrestricted renderer warmup needlessly exercises a video path, so the validated image-serving profile explicitly uses:
+
+```bash
+--limit-mm-per-prompt '{"image":1,"video":0}'
+```
+
+This is a runtime warmup/configuration consideration, not a text or image generation failure.
+
+Peak observed VRAM in the full-checkpoint performance test was **23,914 MiB**; the benchmark conditions are below.
+
+## RTX 3090 24 GB performance
+
+Reference setup: physical CUDA:1 RTX 3090, vLLM main commit `7ca49fbe4bab019e55d57cdc4b7fd3d55c67c1a6`, CUDA graphs enabled, `max_model_len=1024`, `max_num_seqs=32`, `max_num_batched_tokens=8192`, prefix caching disabled, three repetitions per point, and fixed 128-token greedy `ignore_eos` outputs. Values are **aggregate verified output tokens/s**, not per-request rate.
+
+| Active requests | Text aggregate output tok/s | One image/request aggregate output tok/s |
+|---:|---:|---:|
+| 1 | 156.6 | 127.9 |
+| 4 | 486.9 | 262.6 |
+| 8 | 820.4 | 429.9 |
+| 16 | 1,269.7 | 518.5 |
+| 32 | 1,948.3 | 461.6 |
+
+The text test is decode-dominant. The one-image/request test includes image processing and multimodal prefill, so it is not directly comparable to the text-only decode number. B32 maximized measured text aggregate throughput. B16 is the measured image-workload knee; B32 was slower for that workload because it exceeds the efficient 8,192-token multimodal batching envelope. vLLM profiling allocated encoder cache for up to 29 maximum-size images in this configuration. Raw public-safe measurements: [`benchmarks/rtx3090-performance.json`](benchmarks/rtx3090-performance.json).
+
+## Long-context validation
+
+Gemma 4 advertises an architectural maximum of **262,144 tokens**. That is not the same as what this 24 GB reference deployment can serve.
+
+The validated RTX 3090 operational maximum is **131,072 total tokens (128K)**. A near-limit request with **130,816 input tokens** plus **64 generated tokens** completed with CUDA graphs, Marlin W4A16 dense/MoE paths, no OOM, and no allocator warning.
+
+- Validated fixed KV reservation: `6444425401` bytes.
+- Observed cache capacity: `135083` tokens.
+- Margin above the configured 128K window: `4011` tokens (about 3%).
+- A conventional 132K setting does not fit this fixed cache; 128K is the recommended validated ceiling for this reference configuration.
+
+`max_model_len` includes generation: `input_tokens + max_tokens <= 131072`. For example, with a 2,048-token generation budget, the maximum input is 129,024 tokens. The exact fixed reservation is hardware- and runtime-specific; do not assume every 24 GB GPU has the same headroom.
+
+## Recommended serving configurations
+
+### A. Mixed agentic/search and batch service
+
+This is the validated 128K RTX 3090 reference profile:
+
+```bash
+CUDA_VISIBLE_DEVICES=1 \
+vllm serve useful-quants/Gemma-4-26B-A4B-it-W4A16-G64-BF16Vision \
+  --quantization compressed-tensors \
+  --dtype bfloat16 \
+  --max-model-len 131072 \
+  --kv-cache-memory-bytes 6444425401 \
+  --max-num-seqs 17 \
+  --max-num-batched-tokens 8192 \
+  --enable-chunked-prefill \
+  --enable-prefix-caching \
+  --scheduling-policy priority \
+  --limit-mm-per-prompt '{"image":1,"video":0}' \
+  --host 0.0.0.0 \
+  --port 8000
+```
+
+Do not combine this exact fixed-KV profile with `--gpu-memory-utilization`: explicit cache bytes are authoritative. The byte value is a validated RTX 3090 reference setting, not a universal default.
+
+Admission policy matters as much as priority:
+
+- Set `max_num_seqs=17`.
+- Cap bulk ingress at **16** active requests, normally with `priority: 100`.
+- Submit interactive/Hermes/search traffic with `priority: -10`.
+- The free seventeenth slot lets the interactive request enter. Once admitted, priority scheduling can preempt/requeue lower-priority cache users if the long request requires cache. Priority alone should not be expected to create an admission slot when every sequence slot is occupied.
+
+Example OpenAI-compatible high-priority request, reserving a 2K response budget:
+
+```bash
+curl http://localhost:8000/v1/chat/completions \
+  -H 'Content-Type: application/json' \
+  -d '{
+    "model": "useful-quants/Gemma-4-26B-A4B-it-W4A16-G64-BF16Vision",
+    "messages": [{"role": "user", "content": "Summarize the supplied context."}],
+    "max_tokens": 2048,
+    "temperature": 0.2,
+    "priority": -10
+  }'
+```
+
+### B. Text-throughput batch profile
+
+For independent text-only jobs, use up to 32 active requests. The measured B32 point reached 1,948.3 aggregate verified output tok/s under the benchmark conditions above. This is a throughput profile, not the long-context mixed-service reservation.
+
+### C. Image-heavy batch profile
+
+For one image per request, B16 is the measured knee (518.5 aggregate output tok/s). Retain the 8,192-token batching envelope and disable video unless it is explicitly required:
+
+```bash
+--max-num-seqs 16 --max-num-batched-tokens 8192 \
+--limit-mm-per-prompt '{"image":1,"video":0}'
+```
+
+## Runtime compatibility
+
+Known-good final validation environment:
+
+| Component | Version |
+|---|---|
+| vLLM | main commit `7ca49fbe4bab019e55d57cdc4b7fd3d55c67c1a6` (`0.1.dev1+g7ca49fbe4`) |
+| Transformers | 5.15.1 |
+| PyTorch | 2.13.0+cu130 |
+| CUDA runtime | 13.0 |
+| compressed-tensors runtime | 0.17.0 |
+| Conversion artifact metadata | compressed-tensors 0.18.0 |
+
+vLLM 0.27.1 did not contain the necessary handling for Transformers-5.15-era heterogeneous Gemma 4 attention metadata. The artifact was not the cause. Use the pinned commit above or a later vLLM release that explicitly includes the corresponding Gemma 4 heterogeneous-config support. Transformers-native inference is not claimed as validated for this packed artifact; vLLM is the supported runtime documented here.
+
+## Image usage
+
+Use an OpenAI-compatible multimodal request and let vLLM/the model processor apply Gemma's chat template:
+
+```python
+from openai import OpenAI
+
+client = OpenAI(base_url="http://localhost:8000/v1", api_key="unused")
+response = client.chat.completions.create(
+    model="useful-quants/Gemma-4-26B-A4B-it-W4A16-G64-BF16Vision",
+    messages=[{
+        "role": "user",
+        "content": [
+            {"type": "image_url", "image_url": {"url": "https://example.org/image.jpg"}},
+            {"type": "text", "text": "Describe this image accurately and concisely."},
+        ],
+    }],
+    max_tokens=256,
+)
+print(response.choices[0].message.content)
+```
+
+Do not hand-insert raw image-placeholder text; the Gemma 4 processor/template is responsible for the image placement.
+
+## Limitations and caveats
+
+- Performance and KV-cache figures are measurements from one RTX 3090 24 GB, not universal guarantees.
+- The 128K claim means a validated 131,072-token **total** window, including output; it does not make 262K feasible on this reference card.
+- Video was intentionally disabled and not validated for this image-focused profile.
+- Image throughput includes processor and prefill work; do not compare it directly with text decode throughput.
+- The long-context reference profile retained the quality-tested BF16/auto KV-cache behavior. Reduced-precision KV cache has not been validated here.
+- This is a quantization of the linked QAT/Heretic parent, not a stock-Google checkpoint conversion.
+
+## Reproducibility
+
+Public-safe provenance, calibration, artifact-audit, benchmark, and perplexity files are included in this repository:
+
+- [`provenance/calibration-and-source.json`](provenance/calibration-and-source.json)
+- [`provenance/artifact-audit.json`](provenance/artifact-audit.json)
+- [`benchmarks/rtx3090-performance.json`](benchmarks/rtx3090-performance.json)
+- [`benchmarks/perplexity.json`](benchmarks/perplexity.json)
+- [`eval/evaluate_gemma4_ppl_vllm.py`](eval/evaluate_gemma4_ppl_vllm.py)
+
+## Citation and license
+
+Please cite the [QAT/Heretic parent model card](https://huggingface.co/Mitchins/gemma-4-26B-A4B-it-qat-q4_0-unquantized-uncensored-heretic), the original [Gemma 4 model card](https://huggingface.co/google/gemma-4-26B-A4B-it), and [technical report](https://arxiv.org/abs/2607.02770) for their respective methods and architecture. Google DeepMind developed Gemma; useful-quants packaged this W4A16 artifact from the QAT/Heretic parent checkpoint.
+
+This repository inherits the parent Apache-2.0 license; see [LICENSE](LICENSE) and the [Gemma 4 license](https://ai.google.dev/gemma/docs/gemma_4_license).

RELEASE_CHECKLIST.md

@@ -0,0 +1,35 @@
+# Release checklist — local staging only
+
+Status: **not uploaded**. This checklist maps public claims to local evidence.
+
+| Claim / release gate | Local evidence | Status |
+|---|---|---|
+| Immediate source is a tuned QAT/Heretic parent, not stock-Google weights | `Mitchins/gemma-4-26B-A4B-it-qat-q4_0-unquantized-uncensored-heretic` at `d62fa836581401beafd08fdd624927e31dcac6ee`; frozen source manifest | verified |
+| Original Gemma family base | Google `google/gemma-4-26B-A4B-it` upstream card/config | verified as inherited family lineage |
+| Apache-2.0 parent license | upstream Google model-card metadata; parent-license instruction | verified |
+| W4A16, G64, symmetric pack-quantized configuration | artifact `config.json`, `recipe.yaml` | verified |
+| Vision/router/embed/LM-head exclusions | artifact `config.json`, `recipe.yaml`, safetensors audit | verified; output head is tied to BF16 token embedding |
+| 512 text + 1,024 COCO calibration | `CALIBRATION_MANIFEST.json`, frozen manifests, conversion script | verified |
+| COCO selections hash verified | `image_manifest.jsonl` | verified; media intentionally excluded |
+| Fresh text and real-image inference | `gemma4-g64-full-canary.json` and corresponding vLLM log | verified |
+| Marlin dense and MoE paths, CUDA graphs | canary and full-validation logs | verified |
+| RTX 3090 throughput table | `gemma4-g64-full-speed.json` | verified |
+| 23,914 MiB peak benchmark VRAM | `gemma4-g64-full-speed.json` B32 records | verified |
+| 128K long-context claim | `gemma4-g64-128k-agent-fixed-cache.json` and log | verified |
+| Perplexity: same BF16/W4 text/windows/scoring | `eval/evaluate_gemma4_ppl_vllm.py`, `benchmarks/perplexity.json` | verified |
+| Readme YAML parses | local YAML parser | verified |
+| Public-safe files contain no absolute local paths/secrets | release-tree scan | verified |
+| Fresh staged text + image smoke | `benchmarks/functional-canary.json` | verified |
+| Model files unchanged from final artifact | same-inode hard link and SHA256 | verified |
+
+## Deliberate non-claims
+
+- No stock-Google checkpoint quality claim: this is a quantization of the linked tuned QAT/Heretic parent.
+- No video validation.
+- No Transformers-native packed-artifact serving validation.
+- No universal 24 GB or 128K guarantee.
+- No reduced-precision KV-cache validation.
+
+## Upload boundary
+
+Do not upload calibration images, raw calibration text, source BF16 shards, offload directories, complete logs, environment caches, local paths, credentials, or host-specific artifacts. See [`UPLOAD_MANIFEST.md`](UPLOAD_MANIFEST.md).

SHA256SUMS

@@ -0,0 +1,20 @@
+cfc7749b96f63bd31c3c42b5c471bf756814053e847c10f3eb003417bc523d30  LICENSE
+699ef4ff4940b5c61043fb363baf919d6d7957294dbecc97c64498f203fea63b  README.md
+61aa1b3e515ab2422f28c49cb98f0b9065089a9b0c7214f5e15cfb8da0309f5a  RELEASE_CHECKLIST.md
+a5a4ad69518a25f2ab78ebee60b4a7e675ee10dd6553d94f3a19369e92831caf  UPLOAD_MANIFEST.md
+2500eb83c5bab1125296c3b578c6ab16f22d9eeb66f411deeda8570d9b77048a  benchmarks/functional-canary.json
+b8568e3f7627a3789b291da2819641d00f2c5d434e985dc4b4d8dec1d25dd417  benchmarks/perplexity.json
+38afb77d9202ccc1aeaf24f43a99836bfe906b9097ad4a4175666af757cdcc85  benchmarks/rtx3090-128k-validation.json
+bf975291c1c2dfbe7cbf8a4dc1c2a02c880d915652caff670fe357a839406d50  benchmarks/rtx3090-performance.json
+ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4  chat_template.jinja
+3eb748bdc68e1139fced21067b3b6a36473930a5ac8ff85fb0538d5140ac2cd0  config.json
+2323e35e64bd70315112b1628e73c7f65774d2b2454b38bcb4dd6ed98b5cccda  eval/evaluate_gemma4_ppl_vllm.py
+5215a2bc388bc5fa3c0b71b1eea50c787feb13da07f3539aa64fa923d8b8ba62  examples/serve_vllm_3090_128k.sh
+3c205a3306d3c8a54e56e529f7338161bc012f1f2516e7175432cecd9c32b92d  generation_config.json
+4475b48270e4d2072886f084442e29f5e5eb16c7668d5a2a28404428e9c0b247  model.safetensors
+32bdf45d2ad4cc29a0822ddd157a182de76644f0419a6228d151495256e9813c  processor_config.json
+2cf5d3bf89731f262fb1873e1561cdca4dd67bbd06126faeb4fb0a4fcdc84290  provenance/artifact-audit.json
+e14f7e86544f46a80d36cb10e4597a5589fac8e04ddadda67dcf918c6e4ffbc1  provenance/calibration-and-source.json
+fd10b59552e882faf755509a8226ed10a6a36b9d5444372b4219dd41843f4a5c  recipe.yaml
+c2cf15ae878b9477ea6d714d02be642da51424c6b7fe0c1e2cf5a332b5c2f789  tokenizer.json
+ae79d5349f543e549ca2721755b030dbba92614e25585f4c7e9aaa3734dfa6ae  tokenizer_config.json

UPLOAD_MANIFEST.md

@@ -0,0 +1,28 @@
+# Upload manifest
+
+This local release candidate is intended to contain the following published payload.
+
+## Upload
+
+- `model.safetensors` — the final packed W4A16 model payload.
+- `config.json`, `generation_config.json` — architecture/generation metadata including the compressed-tensors quantization configuration.
+- `tokenizer.json`, `tokenizer_config.json`, `processor_config.json`, `chat_template.jinja` — inherited tokenizer, processor, and template files required for normal Gemma 4/vLLM use.
+- `README.md` — model card.
+- `LICENSE` — the inherited Apache-2.0 license text.
+- `SHA256SUMS` — checksums of all uploaded payload files.
+- `benchmarks/rtx3090-performance.json` — public-safe performance data.
+- `benchmarks/functional-canary.json` and `benchmarks/rtx3090-128k-validation.json` — public-safe functional and near-limit validation records.
+- `benchmarks/perplexity.json` — public-safe matched perplexity result.
+- `provenance/calibration-and-source.json` and `provenance/artifact-audit.json` — compact reproducibility records.
+- `eval/evaluate_gemma4_ppl_vllm.py` — perplexity implementation.
+- `examples/` — small public client/serve examples if added.
+
+## Keep private
+
+- All calibration images, image manifests containing captions/prompts, and text calibration windows.
+- The 49 GB BF16 tuned/QAT parent checkpoint and its 11 source shards.
+- Raw conversion logs, full vLLM logs, offload directories, environment caches, and compiler caches.
+- Original absolute-path provenance files, host/user/kernel details, and any credentials/tokens.
+- Trace artifacts, failed attempts, and internal planning notes.
+
+The staged `model.safetensors` must be the byte-identical final artifact; no model-weight file is modified by release packaging.

benchmarks/functional-canary.json

@@ -0,0 +1,23 @@
+{
+  "status": "passed",
+  "artifact_loaded_from_release_stage": true,
+  "text_generation": {
+    "status": "passed",
+    "generated_token_count": 8
+  },
+  "real_image_generation": {
+    "status": "passed",
+    "input": "one held-out COCO 2017 validation image",
+    "output": "A close-up shot of a large brown bear with thick, light brown fur, looking directly at the camera against a background of green grass."
+  },
+  "runtime": {
+    "quantization": "compressed-tensors",
+    "dense_kernel": "Marlin W4A16",
+    "moe_kernel": "Marlin W4A16",
+    "cuda_graphs": true,
+    "limit_mm_per_prompt": {
+      "image": 1,
+      "video": 0
+    }
+  }
+}

benchmarks/perplexity.json

@@ -0,0 +1,31 @@
+{
+  "methodology": {
+    "engine": "vLLM prompt_logprobs=1",
+    "dataset": "Salesforce/wikitext",
+    "dataset_config": "wikitext-2-raw-v1",
+    "split": "test",
+    "dataset_fingerprint": "a46124b21ac53738",
+    "window_selection": "first contiguous non-empty test-corpus token windows",
+    "num_windows": 4,
+    "window_tokens": 512,
+    "evaluated_next_tokens": 2044,
+    "tokenizer": "frozen Gemma 4 QAT/Heretic parent tokenizer",
+    "note": "BF16 used vLLM CPU offload on a 24 GB RTX 3090; W4A16 used the same vLLM prompt-logprob scorer without offload."
+  },
+  "results": {
+    "bf16_parent": {
+      "perplexity": 388.14988734463424,
+      "mean_nll": 5.961391572614971,
+      "nll_sum": 12185.084374425001,
+      "cpu_offload_gb": 30.0
+    },
+    "w4a16_g64": {
+      "perplexity": 432.31795682768376,
+      "mean_nll": 6.069161328695908,
+      "nll_sum": 12405.365755854436,
+      "cpu_offload_gb": 0.0
+    },
+    "delta_perplexity": 44.16806948304952,
+    "delta_percent": 11.379127219437565
+  }
+}

benchmarks/rtx3090-128k-validation.json

@@ -0,0 +1,22 @@
+{
+  "status": "passed",
+  "reference_gpu": "RTX 3090 24 GB",
+  "runtime": "vLLM main 7ca49fbe4bab019e55d57cdc4b7fd3d55c67c1a6",
+  "max_model_len": 131072,
+  "near_limit_input_tokens": 130816,
+  "generated_tokens": 64,
+  "max_num_seqs": 17,
+  "max_num_batched_tokens": 8192,
+  "kv_cache_memory_bytes": 6444425401,
+  "cache_capacity_tokens": 135083,
+  "cache_margin_tokens": 4011,
+  "cuda_graphs": true,
+  "quantization": "compressed-tensors W4A16 G64",
+  "limit_mm_per_prompt": {
+    "image": 1,
+    "video": 0
+  },
+  "oom": false,
+  "allocator_warning": false,
+  "wall_seconds": 92.90461822296493
+}

benchmarks/rtx3090-performance.json

@@ -0,0 +1,339 @@
+{
+  "runtime": "vLLM main 7ca49fbe4bab019e55d57cdc4b7fd3d55c67c1a6",
+  "gpu": "RTX 3090 24 GB (physical CUDA:1 in the reference run)",
+  "profile": {
+    "max_model_len": 1024,
+    "max_num_seqs": 32,
+    "max_num_batched_tokens": 8192,
+    "gpu_memory_utilization": 0.96,
+    "cuda_graphs": true,
+    "prefix_caching": false,
+    "limit_mm_per_prompt": {
+      "image": 1,
+      "video": 0
+    }
+  },
+  "method": "Three repetitions per point; short unique prompts; fixed 128-token greedy ignore-EOS outputs; output tokens divided by end-to-end batch completion time after warmup.",
+  "records": [
+    {
+      "workload": "text",
+      "concurrency": 1,
+      "repeats": 3,
+      "output_tokens_per_request": 128,
+      "total_verified_output_tokens": 384,
+      "aggregate_output_tok_s": 156.57567204605067,
+      "per_request_output_tok_s": 156.57567204605067,
+      "mean_completion_seconds": 0.8174960920005105,
+      "mean_tpot_ms": 6.386688218753989,
+      "peak_observed_gpu_memory_mib": 23010,
+      "runs": [
+        {
+          "generated_tokens": 128,
+          "wall_seconds": 0.81746169104008,
+          "output_tok_s": 156.58226116644306,
+          "gpu_memory_mib_after": 23010
+        },
+        {
+          "generated_tokens": 128,
+          "wall_seconds": 0.8176816419581883,
+          "output_tok_s": 156.54014158060946,
+          "gpu_memory_mib_after": 23010
+        },
+        {
+          "generated_tokens": 128,
+          "wall_seconds": 0.8173449430032633,
+          "output_tok_s": 156.60462708642333,
+          "gpu_memory_mib_after": 23010
+        }
+      ]
+    },
+    {
+      "workload": "one_image_per_request",
+      "concurrency": 1,
+      "repeats": 3,
+      "output_tokens_per_request": 128,
+      "total_verified_output_tokens": 384,
+      "aggregate_output_tok_s": 127.90552416322762,
+      "per_request_output_tok_s": 127.90552416322762,
+      "mean_completion_seconds": 1.0007386376576808,
+      "mean_tpot_ms": 7.818270606700632,
+      "peak_observed_gpu_memory_mib": 23010,
+      "runs": [
+        {
+          "generated_tokens": 128,
+          "wall_seconds": 0.9040946460445412,
+          "output_tok_s": 141.5780975587084,
+          "gpu_memory_mib_after": 23010
+        },
+        {
+          "generated_tokens": 128,
+          "wall_seconds": 1.0513607729808427,
+          "output_tok_s": 121.74698095030823,
+          "gpu_memory_mib_after": 23010
+        },
+        {
+          "generated_tokens": 128,
+          "wall_seconds": 1.0467604939476587,
+          "output_tok_s": 122.28203179246121,
+          "gpu_memory_mib_after": 23010
+        }
+      ]
+    },
+    {
+      "workload": "text",
+      "concurrency": 4,
+      "repeats": 3,
+      "output_tokens_per_request": 128,
+      "total_verified_output_tokens": 1536,
+      "aggregate_output_tok_s": 486.91511429228393,
+      "per_request_output_tok_s": 121.72877857307098,
+      "mean_completion_seconds": 1.0515179853149068,
+      "mean_tpot_ms": 8.21498426027271,
+      "peak_observed_gpu_memory_mib": 23054,
+      "runs": [
+        {
+          "generated_tokens": 512,
+          "wall_seconds": 1.0211215899907984,
+          "output_tok_s": 501.409435486144,
+          "gpu_memory_mib_after": 23010
+        },
+        {
+          "generated_tokens": 512,
+          "wall_seconds": 1.0497098179766908,
+          "output_tok_s": 487.7538451406283,
+          "gpu_memory_mib_after": 23010
+        },
+        {
+          "generated_tokens": 512,
+          "wall_seconds": 1.0837225479772314,
+          "output_tok_s": 472.4456466792614,
+          "gpu_memory_mib_after": 23054
+        }
+      ]
+    },
+    {
+      "workload": "one_image_per_request",
+      "concurrency": 4,
+      "repeats": 3,
+      "output_tokens_per_request": 128,
+      "total_verified_output_tokens": 1536,
+      "aggregate_output_tok_s": 262.6200812679751,
+      "per_request_output_tok_s": 65.65502031699377,
+      "mean_completion_seconds": 1.949584348340674,
+      "mean_tpot_ms": 15.231127721411514,
+      "peak_observed_gpu_memory_mib": 23054,
+      "runs": [
+        {
+          "generated_tokens": 512,
+          "wall_seconds": 1.9419829859980382,
+          "output_tok_s": 263.64803589505664,
+          "gpu_memory_mib_after": 23010
+        },
+        {
+          "generated_tokens": 512,
+          "wall_seconds": 1.9698979230015539,
+          "output_tok_s": 259.911944685875,
+          "gpu_memory_mib_after": 23054
+        },
+        {
+          "generated_tokens": 512,
+          "wall_seconds": 1.93687213602243,
+          "output_tok_s": 264.3437274344014,
+          "gpu_memory_mib_after": 23054
+        }
+      ]
+    },
+    {
+      "workload": "text",
+      "concurrency": 8,
+      "repeats": 3,
+      "output_tokens_per_request": 128,
+      "total_verified_output_tokens": 3072,
+      "aggregate_output_tok_s": 820.4449600227697,
+      "per_request_output_tok_s": 102.55562000284621,
+      "mean_completion_seconds": 1.2481032243425336,
+      "mean_tpot_ms": 9.750806440176044,
+      "peak_observed_gpu_memory_mib": 23138,
+      "runs": [
+        {
+          "generated_tokens": 1024,
+          "wall_seconds": 1.2773179819923826,
+          "output_tok_s": 801.6797809444029,
+          "gpu_memory_mib_after": 23054
+        },
+        {
+          "generated_tokens": 1024,
+          "wall_seconds": 1.277832499006763,
+          "output_tok_s": 801.3569859867684,
+          "gpu_memory_mib_after": 23138
+        },
+        {
+          "generated_tokens": 1024,
+          "wall_seconds": 1.1891591920284554,
+          "output_tok_s": 861.1126305581269,
+          "gpu_memory_mib_after": 23138
+        }
+      ]
+    },
+    {
+      "workload": "one_image_per_request",
+      "concurrency": 8,
+      "repeats": 3,
+      "output_tokens_per_request": 128,
+      "total_verified_output_tokens": 3072,
+      "aggregate_output_tok_s": 429.9295467387298,
+      "per_request_output_tok_s": 53.74119334234123,
+      "mean_completion_seconds": 2.3817855920060538,
+      "mean_tpot_ms": 18.607699937547295,
+      "peak_observed_gpu_memory_mib": 23138,
+      "runs": [
+        {
+          "generated_tokens": 1024,
+          "wall_seconds": 1.7241748640080914,
+          "output_tok_s": 593.9072778381453,
+          "gpu_memory_mib_after": 23138
+        },
+        {
+          "generated_tokens": 1024,
+          "wall_seconds": 2.370788298023399,
+          "output_tok_s": 431.92384611217335,
+          "gpu_memory_mib_after": 23138
+        },
+        {
+          "generated_tokens": 1024,
+          "wall_seconds": 3.050393613986671,
+          "output_tok_s": 335.6943822937319,
+          "gpu_memory_mib_after": 23138
+        }
+      ]
+    },
+    {
+      "workload": "text",
+      "concurrency": 16,
+      "repeats": 3,
+      "output_tokens_per_request": 128,
+      "total_verified_output_tokens": 6144,
+      "aggregate_output_tok_s": 1269.7205057297842,
+      "per_request_output_tok_s": 79.35753160811151,
+      "mean_completion_seconds": 1.6129533946708154,
+      "mean_tpot_ms": 12.601198395865746,
+      "peak_observed_gpu_memory_mib": 23430,
+      "runs": [
+        {
+          "generated_tokens": 2048,
+          "wall_seconds": 1.631988351000473,
+          "output_tok_s": 1254.9109181719928,
+          "gpu_memory_mib_after": 23138
+        },
+        {
+          "generated_tokens": 2048,
+          "wall_seconds": 1.5828196799848229,
+          "output_tok_s": 1293.8934395986519,
+          "gpu_memory_mib_after": 23430
+        },
+        {
+          "generated_tokens": 2048,
+          "wall_seconds": 1.6240521530271508,
+          "output_tok_s": 1261.0432467840592,
+          "gpu_memory_mib_after": 23430
+        }
+      ]
+    },
+    {
+      "workload": "one_image_per_request",
+      "concurrency": 16,
+      "repeats": 3,
+      "output_tokens_per_request": 128,
+      "total_verified_output_tokens": 6144,
+      "aggregate_output_tok_s": 518.5347963568223,
+      "per_request_output_tok_s": 32.408424772301395,
+      "mean_completion_seconds": 3.9495902963293097,
+      "mean_tpot_ms": 30.856174190072732,
+      "peak_observed_gpu_memory_mib": 23522,
+      "runs": [
+        {
+          "generated_tokens": 2048,
+          "wall_seconds": 2.9401068949955516,
+          "output_tok_s": 696.5733128567418,
+          "gpu_memory_mib_after": 23430
+        },
+        {
+          "generated_tokens": 2048,
+          "wall_seconds": 3.787013527995441,
+          "output_tok_s": 540.7955331715059,
+          "gpu_memory_mib_after": 23430
+        },
+        {
+          "generated_tokens": 2048,
+          "wall_seconds": 5.121650465996936,
+          "output_tok_s": 399.8710988961161,
+          "gpu_memory_mib_after": 23522
+        }
+      ]
+    },
+    {
+      "workload": "text",
+      "concurrency": 32,
+      "repeats": 3,
+      "output_tokens_per_request": 128,
+      "total_verified_output_tokens": 12288,
+      "aggregate_output_tok_s": 1948.2966906600047,
+      "per_request_output_tok_s": 60.88427158312515,
+      "mean_completion_seconds": 2.1023492056604787,
+      "mean_tpot_ms": 16.42460316922249,
+      "peak_observed_gpu_memory_mib": 23914,
+      "runs": [
+        {
+          "generated_tokens": 4096,
+          "wall_seconds": 2.0827834619558416,
+          "output_tok_s": 1966.5990607365604,
+          "gpu_memory_mib_after": 23522
+        },
+        {
+          "generated_tokens": 4096,
+          "wall_seconds": 2.070148140017409,
+          "output_tok_s": 1978.602362227833,
+          "gpu_memory_mib_after": 23914
+        },
+        {
+          "generated_tokens": 4096,
+          "wall_seconds": 2.154116015008185,
+          "output_tok_s": 1901.4760446801824,
+          "gpu_memory_mib_after": 23914
+        }
+      ]
+    },
+    {
+      "workload": "one_image_per_request",
+      "concurrency": 32,
+      "repeats": 3,
+      "output_tokens_per_request": 128,
+      "total_verified_output_tokens": 12288,
+      "aggregate_output_tok_s": 461.6294772584143,
+      "per_request_output_tok_s": 14.425921164325446,
+      "mean_completion_seconds": 8.872916921003101,
+      "mean_tpot_ms": 69.31966344533672,
+      "peak_observed_gpu_memory_mib": 23914,
+      "runs": [
+        {
+          "generated_tokens": 4096,
+          "wall_seconds": 8.780865803011693,
+          "output_tok_s": 466.46880750587707,
+          "gpu_memory_mib_after": 23914
+        },
+        {
+          "generated_tokens": 4096,
+          "wall_seconds": 8.837103814992588,
+          "output_tok_s": 463.5002695171388,
+          "gpu_memory_mib_after": 23914
+        },
+        {
+          "generated_tokens": 4096,
+          "wall_seconds": 9.000781145005021,
+          "output_tok_s": 455.07161367578334,
+          "gpu_memory_mib_after": 23914
+        }
+      ]
+    }
+  ]
+}

chat_template.jinja

@@ -0,0 +1,390 @@
+{#
+  Template: Google Gemma 4 Canonical Chat Template
+  Author: Google Gemma Engineering Team
+  Published: 2026-07-09
+  Context: Fixed tool-calling loops, turn closures, and thinking content-ordering.
+#}
+{%- macro format_parameters(properties, required, filter_keys=false) -%}
+    {%- set standard_keys = ['description', 'type', 'properties', 'required', 'nullable'] -%}
+    {%- set ns = namespace(found_first=false) -%}
+    {%- for key, value in properties | dictsort -%}
+        {%- set add_comma = false -%}
+        {%- if not filter_keys or key not in standard_keys -%}
+            {%- if ns.found_first %},{% endif -%}
+            {%- set ns.found_first = true -%}
+            {{ key }}:{
+            {%- if value['description'] -%}
+                description:<|"|>{{ value['description'] }}<|"|>
+                {%- set add_comma = true -%}
+            {%- endif -%}
+            {%- if value['type'] | upper == 'STRING' -%}
+                {%- if value['enum'] -%}
+                    {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%}
+                    enum:{{ format_argument(value['enum']) }}
+                {%- endif -%}
+            {%- elif value['type'] | upper == 'ARRAY' -%}
+                {%- if value['items'] is mapping and value['items'] -%}
+                    {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%}
+                    items:{
+                    {%- set ns_items = namespace(found_first=false) -%}
+                    {%- for item_key, item_value in value['items'] | dictsort -%}
+                        {%- if item_value is not none -%}
+                            {%- if ns_items.found_first %},{% endif -%}
+                            {%- set ns_items.found_first = true -%}
+                            {%- if item_key == 'properties' -%}
+                                properties:{
+                                {%- if item_value is mapping -%}
+                                    {{- format_parameters(item_value, value['items']['required'] | default([])) -}}
+                                {%- endif -%}
+                                }
+                            {%- elif item_key == 'required' -%}
+                                required:[
+                                {%- for req_item in item_value -%}
+                                    <|"|>{{- req_item -}}<|"|>
+                                    {%- if not loop.last %},{% endif -%}
+                                {%- endfor -%}
+                                ]
+                            {%- elif item_key == 'type' -%}
+                                {%- if item_value is string -%}
+                                    type:{{ format_argument(item_value | upper) }}
+                                {%- else -%}
+                                    type:{{ format_argument(item_value | map('upper') | list) }}
+                                {%- endif -%}
+                            {%- else -%}
+                                {{ item_key }}:{{ format_argument(item_value) }}
+                            {%- endif -%}
+                        {%- endif -%}
+                    {%- endfor -%}
+                    }
+                {%- endif -%}
+            {%- endif -%}
+            {%- if value['nullable'] %}
+                {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%}
+                nullable:true
+            {%- endif -%}
+            {%- if value['type'] | upper == 'OBJECT' -%}
+                {%- if value['properties'] is defined and value['properties'] is mapping -%}
+                    {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%}
+                    properties:{
+                    {{- format_parameters(value['properties'], value['required'] | default([])) -}}
+                    }
+                {%- elif value is mapping -%}
+                    {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%}
+                    properties:{
+                    {{- format_parameters(value, value['required'] | default([]), filter_keys=true) -}}
+                    }
+                {%- endif -%}
+                {%- if value['required'] -%}
+                    {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%}
+                    required:[
+                    {%- for item in value['required'] | default([]) -%}
+                        <|"|>{{- item -}}<|"|>
+                        {%- if not loop.last %},{% endif -%}
+                    {%- endfor -%}
+                    ]
+                {%- endif -%}
+            {%- endif -%}
+            {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%}
+            type:<|"|>{{ value['type'] | upper }}<|"|>}
+        {%- endif -%}
+    {%- endfor -%}
+{%- endmacro -%}
+{%- macro format_function_declaration(tool_data) -%}
+    declaration:{{- tool_data['function']['name'] -}}{description:<|"|>{{- tool_data['function']['description'] -}}<|"|>
+    {%- set params = tool_data['function']['parameters'] -%}
+    {%- if params -%}
+        ,parameters:{
+        {%- if params['properties'] -%}
+            properties:{ {{- format_parameters(params['properties'], params['required']) -}} },
+        {%- endif -%}
+        {%- if params['required'] -%}
+            required:[
+            {%- for item in params['required'] -%}
+                <|"|>{{- item -}}<|"|>
+                {{- ',' if not loop.last -}}
+            {%- endfor -%}
+            ],
+        {%- endif -%}
+        {%- if params['type'] -%}
+            type:<|"|>{{- params['type'] | upper -}}<|"|>}
+        {%- endif -%}
+    {%- endif -%}
+    {%- if 'response' in tool_data['function'] -%}
+        {%- set response_declaration = tool_data['function']['response'] -%}
+        ,response:{
+        {%- if response_declaration['description'] -%}
+            description:<|"|>{{- response_declaration['description'] -}}<|"|>,
+        {%- endif -%}
+        {%- if response_declaration['type'] | upper == 'OBJECT' -%}
+            type:<|"|>{{- response_declaration['type'] | upper -}}<|"|>}
+        {%- endif -%}
+    {%- endif -%}
+    }
+{%- endmacro -%}
+{%- macro format_argument(argument, escape_keys=True) -%}
+    {%- if argument is none -%}
+        {{- 'null' -}}
+    {%- elif argument is string -%}
+        {{- '<|"|>' + argument + '<|"|>' -}}
+    {%- elif argument is boolean -%}
+        {{- 'true' if argument else 'false' -}}
+    {%- elif argument is mapping -%}
+        {{- '{' -}}
+        {%- set ns = namespace(found_first=false) -%}
+        {%- for key, value in argument | dictsort -%}
+            {%- if ns.found_first %},{% endif -%}
+            {%- set ns.found_first = true -%}
+            {%- if escape_keys -%}
+                {{- '<|"|>' + key + '<|"|>' -}}
+            {%- else -%}
+                {{- key -}}
+            {%- endif -%}
+            :{{- format_argument(value, escape_keys=escape_keys) -}}
+        {%- endfor -%}
+        {{- '}' -}}
+    {%- elif argument is sequence -%}
+        {{- '[' -}}
+        {%- for item in argument -%}
+            {{- format_argument(item, escape_keys=escape_keys) -}}
+            {%- if not loop.last %},{% endif -%}
+        {%- endfor -%}
+        {{- ']' -}}
+    {%- else -%}
+        {{- argument -}}
+    {%- endif -%}
+{%- endmacro -%}
+{%- macro strip_thinking(text) -%}
+    {%- set ns = namespace(result='') -%}
+    {%- for part in text.split('<channel|>') -%}
+        {%- if '<|channel>' in part -%}
+            {%- set ns.result = ns.result + part.split('<|channel>')[0] -%}
+        {%- else -%}
+            {%- set ns.result = ns.result + part -%}
+        {%- endif -%}
+    {%- endfor -%}
+    {{- ns.result | trim -}}
+{%- endmacro -%}
+
+{%- macro format_tool_response_block(tool_name, response) -%}
+    {{- '<|tool_response>' -}}
+    {%- if response is mapping -%}
+        {{- 'response:' + tool_name + '{' -}}
+        {%- for key, value in response | dictsort -%}
+            {{- key -}}:{{- format_argument(value, escape_keys=False) -}}
+            {%- if not loop.last %},{% endif -%}
+        {%- endfor -%}
+        {{- '}' -}}
+    {%- else -%}
+        {{- 'response:' + tool_name + '{value:' + format_argument(response, escape_keys=False) + '}' -}}
+    {%- endif -%}
+    {{- '<tool_response|>' -}}
+{%- endmacro -%}
+
+{#- ===== SETUP ===== -#}
+{%- set ns = namespace(prev_message_type=None, prev_non_tool_role=None) -%}
+{%- set loop_messages = messages -%}
+{%- set enable_thinking = enable_thinking | default(false) -%}
+{%- set preserve_thinking = preserve_thinking | default(false) -%}
+{{- bos_token -}}
+{#- Handle System/Tool Definitions Block -#}
+{%- if enable_thinking or tools or (messages and messages[0]['role'] in ['system', 'developer']) -%}
+    {{- '<|turn>system\n' -}}
+    {#- Inject Thinking token at the very top of the FIRST system turn -#}
+    {%- if enable_thinking -%}
+        {{- '<|think|>\n' -}}
+        {%- set ns.prev_message_type = 'think' -%}
+    {%- endif -%}
+    {%- if messages and messages[0]['role'] in ['system', 'developer'] -%}
+        {%- if messages[0]['content'] is string -%}
+            {{- messages[0]['content'] | trim -}}
+        {%- elif messages[0]['content'] is sequence -%}
+            {%- for item in messages[0]['content'] -%}
+                {{- item['text'] | trim + ' '-}}
+            {%- endfor -%}
+        {%- endif -%}
+        {%- set loop_messages = messages[1:] -%}
+    {%- endif -%}
+    {%- if tools -%}
+        {%- for tool in tools %}
+            {{- '<|tool>' -}}
+            {{- format_function_declaration(tool) | trim -}}
+            {{- '<tool|>' -}}
+        {%- endfor %}
+        {%- set ns.prev_message_type = 'tool' -%}
+    {%- endif -%}
+    {{- '<turn|>\n' -}}
+{%- endif %}
+
+{#- Pre-scan: find last user message index for reasoning guard -#}
+{%- set ns_turn = namespace(last_user_idx=-1) -%}
+{%- for i in range(loop_messages | length) -%}
+    {%- if loop_messages[i]['role'] == 'user' -%}
+        {%- set ns_turn.last_user_idx = i -%}
+    {%- endif -%}
+{%- endfor -%}
+
+{#- Loop through messages -#}
+{%- for message in loop_messages -%}
+    {%- if message['role'] != 'tool' -%}
+    {%- set ns.prev_message_type = None -%}
+    {%- set role = 'model' if message['role'] == 'assistant' else message['role'] -%}
+    {#- Detect continuation using tracked state — O(1) instead of O(n) backward scan -#}
+    {%- set continue_same_model_turn = (role == 'model' and ns.prev_non_tool_role == 'assistant') -%}
+    {%- if not continue_same_model_turn -%}
+        {{- '<|turn>' + role + '\n' }}
+
+    {%- endif -%}
+
+    {#- Render reasoning/reasoning_content as thinking channel -#}
+    {%- set thinking_text = message.get('reasoning') or message.get('reasoning_content') -%}
+    {%- set thinking_gate = (loop.index0 > ns_turn.last_user_idx) or (preserve_thinking and message.get('tool_calls')) -%}
+    {%- if thinking_text and thinking_gate -%}
+        {{- '<|channel>thought\n' + thinking_text + '\n<channel|>' -}}
+    {%- endif -%}
+
+            {%- if message.get('tool_calls') -%}
+                {%- for tool_call in message.get('tool_calls') -%}
+                    {%- set function = tool_call['function'] -%}
+                    {{- '<|tool_call>call:' + function['name'] + '{' -}}
+                    {%- if function['arguments'] is mapping -%}
+                        {%- set ns_args = namespace(found_first=false) -%}
+                        {%- for key, value in function['arguments'] | dictsort -%}
+                            {%- if ns_args.found_first %},{% endif -%}
+                            {%- set ns_args.found_first = true -%}
+                            {{- key -}}:{{- format_argument(value, escape_keys=False) -}}
+                        {%- endfor -%}
+                    {%- elif function['arguments'] is none -%}
+                    {%- else -%}
+                        {{- raise_exception(
+                            "chat_template: tool_calls[].function.arguments must be a "
+                            "JSON object (mapping), not a string. Deserialize arguments "
+                            "before passing to the template."
+                        ) -}}
+                    {%- endif -%}
+                    {{- '}<tool_call|>' -}}
+                {%- endfor -%}
+                {%- set ns.prev_message_type = 'tool_call' -%}
+            {%- endif -%}
+
+            {%- set ns_tr_out = namespace(flag=false) -%}
+            {%- if message.get('tool_responses') -%}
+                {#- Legacy: tool_responses embedded on the assistant message (Google/Gemma native) -#}
+                {%- for tool_response in message.get('tool_responses') -%}
+                    {{- format_tool_response_block(tool_response['name'] | default('unknown', true), tool_response['response']) -}}
+                    {%- set ns_tr_out.flag = true -%}
+                    {%- set ns.prev_message_type = 'tool_response' -%}
+                {%- endfor -%}
+            {%- elif message.get('tool_calls') -%}
+                {#- OpenAI Chat Completions: forward-scan consecutive role:tool messages -#}
+                {%- set ns_tool_scan = namespace(stopped=false) -%}
+                {%- for k in range(loop.index0 + 1, loop_messages | length) -%}
+                    {%- if ns_tool_scan.stopped -%}
+                    {%- elif loop_messages[k]['role'] != 'tool' -%}
+                        {%- set ns_tool_scan.stopped = true -%}
+                    {%- else -%}
+                        {%- set follow = loop_messages[k] -%}
+                        {#- Resolve tool_call_id to function name -#}
+                        {%- set ns_tname = namespace(name=follow.get('name') or 'unknown') -%}
+                        {%- for tc in message.get('tool_calls') -%}
+                            {%- if tc.get('id') == follow.get('tool_call_id') -%}
+                                {%- set ns_tname.name = tc['function']['name'] -%}
+                            {%- endif -%}
+                        {%- endfor -%}
+                        {#- Handle content as string or content-parts array -#}
+                        {%- set tool_body = follow.get('content') -%}
+                        {%- if tool_body is string -%}
+                            {{- format_tool_response_block(ns_tname.name, tool_body) -}}
+                        {%- elif tool_body is sequence and tool_body is not string -%}
+                            {%- set ns_txt = namespace(s='') -%}
+                            {%- for part in tool_body -%}
+                                {%- if part.get('type') == 'text' -%}
+                                    {%- set ns_txt.s = ns_txt.s + (part.get('text') | default('')) -%}
+                                {%- endif -%}
+                            {%- endfor -%}
+                            {{- format_tool_response_block(ns_tname.name, ns_txt.s) -}}
+                            {%- for part in tool_body -%}
+                                {%- if part.get('type') in ['image', 'image_url'] -%}
+                                    {{- '<|image|>' -}}
+                                {%- elif part.get('type') in ['audio', 'input_audio'] -%}
+                                    {{- '<|audio|>' -}}
+                                {%- elif part.get('type') == 'video' -%}
+                                    {{- '<|video|>' -}}
+                                {%- endif -%}
+                            {%- endfor -%}
+                        {%- else -%}
+                            {{- format_tool_response_block(ns_tname.name, tool_body) -}}
+                        {%- endif -%}
+                        {%- set ns_tr_out.flag = true -%}
+                        {%- set ns.prev_message_type = 'tool_response' -%}
+                    {%- endif -%}
+                {%- endfor -%}
+            {%- endif -%}
+
+            {%- set captured_content -%}
+            {%- if message.get('content') is string -%}
+                {%- if role == 'model' -%}
+                    {{- strip_thinking(message['content']) -}}
+                {%- else -%}
+                    {{- message['content'] | trim -}}
+                {%- endif -%}
+            {%- elif message.get('content') is sequence -%}
+                {%- for item in message['content'] -%}
+                    {%- if item.get('type') == 'text' -%}
+                        {%- if role == 'model' -%}
+                            {{- strip_thinking(item['text']) -}}
+                        {%- else -%}
+                            {{- item['text'] | trim -}}
+                        {%- endif -%}
+                    {%- elif item.get('type') in ['image', 'image_url'] -%}
+                        {{- '<|image|>' -}}
+                    {%- elif item.get('type') in ['audio', 'input_audio'] -%}
+                        {{- '<|audio|>' -}}
+                    {%- elif item.get('type') == 'video' -%}
+                        {{- '<|video|>' -}}
+                    {%- endif -%}
+                {%- endfor -%}
+            {%- endif -%}
+            {%- endset -%}
+
+            {{- captured_content -}}
+            {%- set has_content = captured_content | trim | length > 0 -%}
+
+        {#- Forward-scan: find next non-tool message role for continuation detection -#}
+        {%- set next_nt = namespace(role=None, found=false) -%}
+        {%- for j in range(loop.index0 + 1, loop_messages | length) -%}
+            {%- if not next_nt.found -%}
+                {%- if loop_messages[j]['role'] != 'tool' -%}
+                    {%- set next_nt.role = loop_messages[j]['role'] -%}
+                    {%- set next_nt.found = true -%}
+                {%- endif -%}
+            {%- endif -%}
+        {%- endfor -%}
+
+        {%- set continues_into_next = (
+            role == 'model'
+            and next_nt.role == 'assistant'
+            and (not message.get('tool_calls') or ns_tr_out.flag)
+        ) -%}
+
+        {%- if ns.prev_message_type == 'tool_call' and not ns_tr_out.flag -%}
+            {{- '<|tool_response>' -}}
+        {%- elif continues_into_next -%}
+        {%- elif not (ns_tr_out.flag and not has_content and not next_nt.found) -%}
+            {{- '<turn|>\n' -}}
+        {%- endif -%}
+
+    {#- Track previous non-tool role for next iteration (avoids O(n) backward scan) -#}
+    {%- set ns.prev_non_tool_role = message['role'] -%}
+    {%- endif -%}
+{%- endfor -%}
+
+{%- if add_generation_prompt -%}
+    {%- if ns.prev_message_type != 'tool_response' and ns.prev_message_type != 'tool_call' -%}
+        {{- '<|turn>model\n' -}}
+        {%- if not enable_thinking -%}
+            {{- '<|channel>thought\n<channel|>' -}}
+        {%- endif -%}
+    {%- elif ns.prev_message_type == 'tool_response' and enable_thinking -%}
+        {{- '<|channel>thought\n' -}}
+    {%- endif -%}
+{%- endif -%}

config.json

@@ -0,0 +1,454 @@
+{
+  "architectures": [
+    "Gemma4ForConditionalGeneration"
+  ],
+  "audio_config": null,
+  "audio_token_id": 258881,
+  "boa_token_id": 256000,
+  "boi_token_id": 255999,
+  "dtype": "bfloat16",
+  "eoa_token_id": 258883,
+  "eoa_token_index": 258883,
+  "eoi_token_id": 258882,
+  "eos_token_id": [
+    1,
+    106
+  ],
+  "image_token_id": 258880,
+  "initializer_range": 0.02,
+  "model_type": "gemma4",
+  "quantization_config": {
+    "config_groups": {
+      "group_0": {
+        "format": "pack-quantized",
+        "input_activations": null,
+        "output_activations": null,
+        "targets": [
+          "Linear"
+        ],
+        "weights": {
+          "actorder": "static",
+          "block_structure": null,
+          "dynamic": false,
+          "group_size": 64,
+          "num_bits": 4,
+          "observer": "memoryless_minmax",
+          "observer_kwargs": {},
+          "scale_dtype": null,
+          "strategy": "group",
+          "symmetric": true,
+          "type": "int",
+          "zp_dtype": null
+        }
+      }
+    },
+    "format": "pack-quantized",
+    "global_compression_ratio": null,
+    "ignore": [
+      "model.vision_tower.patch_embedder.input_proj",
+      "model.vision_tower.encoder.layers.0.self_attn.q_proj.linear",
+      "model.vision_tower.encoder.layers.0.self_attn.k_proj.linear",
+      "model.vision_tower.encoder.layers.0.self_attn.v_proj.linear",
+      "model.vision_tower.encoder.layers.0.self_attn.o_proj.linear",
+      "model.vision_tower.encoder.layers.0.mlp.gate_proj.linear",
+      "model.vision_tower.encoder.layers.0.mlp.up_proj.linear",
+      "model.vision_tower.encoder.layers.0.mlp.down_proj.linear",
+      "model.vision_tower.encoder.layers.1.self_attn.q_proj.linear",
+      "model.vision_tower.encoder.layers.1.self_attn.k_proj.linear",
+      "model.vision_tower.encoder.layers.1.self_attn.v_proj.linear",
+      "model.vision_tower.encoder.layers.1.self_attn.o_proj.linear",
+      "model.vision_tower.encoder.layers.1.mlp.gate_proj.linear",
+      "model.vision_tower.encoder.layers.1.mlp.up_proj.linear",
+      "model.vision_tower.encoder.layers.1.mlp.down_proj.linear",
+      "model.vision_tower.encoder.layers.2.self_attn.q_proj.linear",
+      "model.vision_tower.encoder.layers.2.self_attn.k_proj.linear",
+      "model.vision_tower.encoder.layers.2.self_attn.v_proj.linear",
+      "model.vision_tower.encoder.layers.2.self_attn.o_proj.linear",
+      "model.vision_tower.encoder.layers.2.mlp.gate_proj.linear",
+      "model.vision_tower.encoder.layers.2.mlp.up_proj.linear",
+      "model.vision_tower.encoder.layers.2.mlp.down_proj.linear",
+      "model.vision_tower.encoder.layers.3.self_attn.q_proj.linear",
+      "model.vision_tower.encoder.layers.3.self_attn.k_proj.linear",
+      "model.vision_tower.encoder.layers.3.self_attn.v_proj.linear",
+      "model.vision_tower.encoder.layers.3.self_attn.o_proj.linear",
+      "model.vision_tower.encoder.layers.3.mlp.gate_proj.linear",
+      "model.vision_tower.encoder.layers.3.mlp.up_proj.linear",
+      "model.vision_tower.encoder.layers.3.mlp.down_proj.linear",
+      "model.vision_tower.encoder.layers.4.self_attn.q_proj.linear",
+      "model.vision_tower.encoder.layers.4.self_attn.k_proj.linear",
+      "model.vision_tower.encoder.layers.4.self_attn.v_proj.linear",
+      "model.vision_tower.encoder.layers.4.self_attn.o_proj.linear",
+      "model.vision_tower.encoder.layers.4.mlp.gate_proj.linear",
+      "model.vision_tower.encoder.layers.4.mlp.up_proj.linear",
+      "model.vision_tower.encoder.layers.4.mlp.down_proj.linear",
+      "model.vision_tower.encoder.layers.5.self_attn.q_proj.linear",
+      "model.vision_tower.encoder.layers.5.self_attn.k_proj.linear",
+      "model.vision_tower.encoder.layers.5.self_attn.v_proj.linear",
+      "model.vision_tower.encoder.layers.5.self_attn.o_proj.linear",
+      "model.vision_tower.encoder.layers.5.mlp.gate_proj.linear",
+      "model.vision_tower.encoder.layers.5.mlp.up_proj.linear",
+      "model.vision_tower.encoder.layers.5.mlp.down_proj.linear",
+      "model.vision_tower.encoder.layers.6.self_attn.q_proj.linear",
+      "model.vision_tower.encoder.layers.6.self_attn.k_proj.linear",
+      "model.vision_tower.encoder.layers.6.self_attn.v_proj.linear",
+      "model.vision_tower.encoder.layers.6.self_attn.o_proj.linear",
+      "model.vision_tower.encoder.layers.6.mlp.gate_proj.linear",
+      "model.vision_tower.encoder.layers.6.mlp.up_proj.linear",
+      "model.vision_tower.encoder.layers.6.mlp.down_proj.linear",
+      "model.vision_tower.encoder.layers.7.self_attn.q_proj.linear",
+      "model.vision_tower.encoder.layers.7.self_attn.k_proj.linear",
+      "model.vision_tower.encoder.layers.7.self_attn.v_proj.linear",
+      "model.vision_tower.encoder.layers.7.self_attn.o_proj.linear",
+      "model.vision_tower.encoder.layers.7.mlp.gate_proj.linear",
+      "model.vision_tower.encoder.layers.7.mlp.up_proj.linear",
+      "model.vision_tower.encoder.layers.7.mlp.down_proj.linear",
+      "model.vision_tower.encoder.layers.8.self_attn.q_proj.linear",
+      "model.vision_tower.encoder.layers.8.self_attn.k_proj.linear",
+      "model.vision_tower.encoder.layers.8.self_attn.v_proj.linear",
+      "model.vision_tower.encoder.layers.8.self_attn.o_proj.linear",
+      "model.vision_tower.encoder.layers.8.mlp.gate_proj.linear",
+      "model.vision_tower.encoder.layers.8.mlp.up_proj.linear",
+      "model.vision_tower.encoder.layers.8.mlp.down_proj.linear",
+      "model.vision_tower.encoder.layers.9.self_attn.q_proj.linear",
+      "model.vision_tower.encoder.layers.9.self_attn.k_proj.linear",
+      "model.vision_tower.encoder.layers.9.self_attn.v_proj.linear",
+      "model.vision_tower.encoder.layers.9.self_attn.o_proj.linear",
+      "model.vision_tower.encoder.layers.9.mlp.gate_proj.linear",
+      "model.vision_tower.encoder.layers.9.mlp.up_proj.linear",
+      "model.vision_tower.encoder.layers.9.mlp.down_proj.linear",
+      "model.vision_tower.encoder.layers.10.self_attn.q_proj.linear",
+      "model.vision_tower.encoder.layers.10.self_attn.k_proj.linear",
+      "model.vision_tower.encoder.layers.10.self_attn.v_proj.linear",
+      "model.vision_tower.encoder.layers.10.self_attn.o_proj.linear",
+      "model.vision_tower.encoder.layers.10.mlp.gate_proj.linear",
+      "model.vision_tower.encoder.layers.10.mlp.up_proj.linear",
+      "model.vision_tower.encoder.layers.10.mlp.down_proj.linear",
+      "model.vision_tower.encoder.layers.11.self_attn.q_proj.linear",
+      "model.vision_tower.encoder.layers.11.self_attn.k_proj.linear",
+      "model.vision_tower.encoder.layers.11.self_attn.v_proj.linear",
+      "model.vision_tower.encoder.layers.11.self_attn.o_proj.linear",
+      "model.vision_tower.encoder.layers.11.mlp.gate_proj.linear",
+      "model.vision_tower.encoder.layers.11.mlp.up_proj.linear",
+      "model.vision_tower.encoder.layers.11.mlp.down_proj.linear",
+      "model.vision_tower.encoder.layers.12.self_attn.q_proj.linear",
+      "model.vision_tower.encoder.layers.12.self_attn.k_proj.linear",
+      "model.vision_tower.encoder.layers.12.self_attn.v_proj.linear",
+      "model.vision_tower.encoder.layers.12.self_attn.o_proj.linear",
+      "model.vision_tower.encoder.layers.12.mlp.gate_proj.linear",
+      "model.vision_tower.encoder.layers.12.mlp.up_proj.linear",
+      "model.vision_tower.encoder.layers.12.mlp.down_proj.linear",
+      "model.vision_tower.encoder.layers.13.self_attn.q_proj.linear",
+      "model.vision_tower.encoder.layers.13.self_attn.k_proj.linear",
+      "model.vision_tower.encoder.layers.13.self_attn.v_proj.linear",
+      "model.vision_tower.encoder.layers.13.self_attn.o_proj.linear",
+      "model.vision_tower.encoder.layers.13.mlp.gate_proj.linear",
+      "model.vision_tower.encoder.layers.13.mlp.up_proj.linear",
+      "model.vision_tower.encoder.layers.13.mlp.down_proj.linear",
+      "model.vision_tower.encoder.layers.14.self_attn.q_proj.linear",
+      "model.vision_tower.encoder.layers.14.self_attn.k_proj.linear",
+      "model.vision_tower.encoder.layers.14.self_attn.v_proj.linear",
+      "model.vision_tower.encoder.layers.14.self_attn.o_proj.linear",
+      "model.vision_tower.encoder.layers.14.mlp.gate_proj.linear",
+      "model.vision_tower.encoder.layers.14.mlp.up_proj.linear",
+      "model.vision_tower.encoder.layers.14.mlp.down_proj.linear",
+      "model.vision_tower.encoder.layers.15.self_attn.q_proj.linear",
+      "model.vision_tower.encoder.layers.15.self_attn.k_proj.linear",
+      "model.vision_tower.encoder.layers.15.self_attn.v_proj.linear",
+      "model.vision_tower.encoder.layers.15.self_attn.o_proj.linear",
+      "model.vision_tower.encoder.layers.15.mlp.gate_proj.linear",
+      "model.vision_tower.encoder.layers.15.mlp.up_proj.linear",
+      "model.vision_tower.encoder.layers.15.mlp.down_proj.linear",
+      "model.vision_tower.encoder.layers.16.self_attn.q_proj.linear",
+      "model.vision_tower.encoder.layers.16.self_attn.k_proj.linear",
+      "model.vision_tower.encoder.layers.16.self_attn.v_proj.linear",
+      "model.vision_tower.encoder.layers.16.self_attn.o_proj.linear",
+      "model.vision_tower.encoder.layers.16.mlp.gate_proj.linear",
+      "model.vision_tower.encoder.layers.16.mlp.up_proj.linear",
+      "model.vision_tower.encoder.layers.16.mlp.down_proj.linear",
+      "model.vision_tower.encoder.layers.17.self_attn.q_proj.linear",
+      "model.vision_tower.encoder.layers.17.self_attn.k_proj.linear",
+      "model.vision_tower.encoder.layers.17.self_attn.v_proj.linear",
+      "model.vision_tower.encoder.layers.17.self_attn.o_proj.linear",
+      "model.vision_tower.encoder.layers.17.mlp.gate_proj.linear",
+      "model.vision_tower.encoder.layers.17.mlp.up_proj.linear",
+      "model.vision_tower.encoder.layers.17.mlp.down_proj.linear",
+      "model.vision_tower.encoder.layers.18.self_attn.q_proj.linear",
+      "model.vision_tower.encoder.layers.18.self_attn.k_proj.linear",
+      "model.vision_tower.encoder.layers.18.self_attn.v_proj.linear",
+      "model.vision_tower.encoder.layers.18.self_attn.o_proj.linear",
+      "model.vision_tower.encoder.layers.18.mlp.gate_proj.linear",
+      "model.vision_tower.encoder.layers.18.mlp.up_proj.linear",
+      "model.vision_tower.encoder.layers.18.mlp.down_proj.linear",
+      "model.vision_tower.encoder.layers.19.self_attn.q_proj.linear",
+      "model.vision_tower.encoder.layers.19.self_attn.k_proj.linear",
+      "model.vision_tower.encoder.layers.19.self_attn.v_proj.linear",
+      "model.vision_tower.encoder.layers.19.self_attn.o_proj.linear",
+      "model.vision_tower.encoder.layers.19.mlp.gate_proj.linear",
+      "model.vision_tower.encoder.layers.19.mlp.up_proj.linear",
+      "model.vision_tower.encoder.layers.19.mlp.down_proj.linear",
+      "model.vision_tower.encoder.layers.20.self_attn.q_proj.linear",
+      "model.vision_tower.encoder.layers.20.self_attn.k_proj.linear",
+      "model.vision_tower.encoder.layers.20.self_attn.v_proj.linear",
+      "model.vision_tower.encoder.layers.20.self_attn.o_proj.linear",
+      "model.vision_tower.encoder.layers.20.mlp.gate_proj.linear",
+      "model.vision_tower.encoder.layers.20.mlp.up_proj.linear",
+      "model.vision_tower.encoder.layers.20.mlp.down_proj.linear",
+      "model.vision_tower.encoder.layers.21.self_attn.q_proj.linear",
+      "model.vision_tower.encoder.layers.21.self_attn.k_proj.linear",
+      "model.vision_tower.encoder.layers.21.self_attn.v_proj.linear",
+      "model.vision_tower.encoder.layers.21.self_attn.o_proj.linear",
+      "model.vision_tower.encoder.layers.21.mlp.gate_proj.linear",
+      "model.vision_tower.encoder.layers.21.mlp.up_proj.linear",
+      "model.vision_tower.encoder.layers.21.mlp.down_proj.linear",
+      "model.vision_tower.encoder.layers.22.self_attn.q_proj.linear",
+      "model.vision_tower.encoder.layers.22.self_attn.k_proj.linear",
+      "model.vision_tower.encoder.layers.22.self_attn.v_proj.linear",
+      "model.vision_tower.encoder.layers.22.self_attn.o_proj.linear",
+      "model.vision_tower.encoder.layers.22.mlp.gate_proj.linear",
+      "model.vision_tower.encoder.layers.22.mlp.up_proj.linear",
+      "model.vision_tower.encoder.layers.22.mlp.down_proj.linear",
+      "model.vision_tower.encoder.layers.23.self_attn.q_proj.linear",
+      "model.vision_tower.encoder.layers.23.self_attn.k_proj.linear",
+      "model.vision_tower.encoder.layers.23.self_attn.v_proj.linear",
+      "model.vision_tower.encoder.layers.23.self_attn.o_proj.linear",
+      "model.vision_tower.encoder.layers.23.mlp.gate_proj.linear",
+      "model.vision_tower.encoder.layers.23.mlp.up_proj.linear",
+      "model.vision_tower.encoder.layers.23.mlp.down_proj.linear",
+      "model.vision_tower.encoder.layers.24.self_attn.q_proj.linear",
+      "model.vision_tower.encoder.layers.24.self_attn.k_proj.linear",
+      "model.vision_tower.encoder.layers.24.self_attn.v_proj.linear",
+      "model.vision_tower.encoder.layers.24.self_attn.o_proj.linear",
+      "model.vision_tower.encoder.layers.24.mlp.gate_proj.linear",
+      "model.vision_tower.encoder.layers.24.mlp.up_proj.linear",
+      "model.vision_tower.encoder.layers.24.mlp.down_proj.linear",
+      "model.vision_tower.encoder.layers.25.self_attn.q_proj.linear",
+      "model.vision_tower.encoder.layers.25.self_attn.k_proj.linear",
+      "model.vision_tower.encoder.layers.25.self_attn.v_proj.linear",
+      "model.vision_tower.encoder.layers.25.self_attn.o_proj.linear",
+      "model.vision_tower.encoder.layers.25.mlp.gate_proj.linear",
+      "model.vision_tower.encoder.layers.25.mlp.up_proj.linear",
+      "model.vision_tower.encoder.layers.25.mlp.down_proj.linear",
+      "model.vision_tower.encoder.layers.26.self_attn.q_proj.linear",
+      "model.vision_tower.encoder.layers.26.self_attn.k_proj.linear",
+      "model.vision_tower.encoder.layers.26.self_attn.v_proj.linear",
+      "model.vision_tower.encoder.layers.26.self_attn.o_proj.linear",
+      "model.vision_tower.encoder.layers.26.mlp.gate_proj.linear",
+      "model.vision_tower.encoder.layers.26.mlp.up_proj.linear",
+      "model.vision_tower.encoder.layers.26.mlp.down_proj.linear",
+      "model.language_model.layers.0.router",
+      "model.language_model.layers.0.router.proj",
+      "model.language_model.layers.1.router",
+      "model.language_model.layers.1.router.proj",
+      "model.language_model.layers.2.router",
+      "model.language_model.layers.2.router.proj",
+      "model.language_model.layers.3.router",
+      "model.language_model.layers.3.router.proj",
+      "model.language_model.layers.4.router",
+      "model.language_model.layers.4.router.proj",
+      "model.language_model.layers.5.router",
+      "model.language_model.layers.5.router.proj",
+      "model.language_model.layers.6.router",
+      "model.language_model.layers.6.router.proj",
+      "model.language_model.layers.7.router",
+      "model.language_model.layers.7.router.proj",
+      "model.language_model.layers.8.router",
+      "model.language_model.layers.8.router.proj",
+      "model.language_model.layers.9.router",
+      "model.language_model.layers.9.router.proj",
+      "model.language_model.layers.10.router",
+      "model.language_model.layers.10.router.proj",
+      "model.language_model.layers.11.router",
+      "model.language_model.layers.11.router.proj",
+      "model.language_model.layers.12.router",
+      "model.language_model.layers.12.router.proj",
+      "model.language_model.layers.13.router",
+      "model.language_model.layers.13.router.proj",
+      "model.language_model.layers.14.router",
+      "model.language_model.layers.14.router.proj",
+      "model.language_model.layers.15.router",
+      "model.language_model.layers.15.router.proj",
+      "model.language_model.layers.16.router",
+      "model.language_model.layers.16.router.proj",
+      "model.language_model.layers.17.router",
+      "model.language_model.layers.17.router.proj",
+      "model.language_model.layers.18.router",
+      "model.language_model.layers.18.router.proj",
+      "model.language_model.layers.19.router",
+      "model.language_model.layers.19.router.proj",
+      "model.language_model.layers.20.router",
+      "model.language_model.layers.20.router.proj",
+      "model.language_model.layers.21.router",
+      "model.language_model.layers.21.router.proj",
+      "model.language_model.layers.22.router",
+      "model.language_model.layers.22.router.proj",
+      "model.language_model.layers.23.router",
+      "model.language_model.layers.23.router.proj",
+      "model.language_model.layers.24.router",
+      "model.language_model.layers.24.router.proj",
+      "model.language_model.layers.25.router",
+      "model.language_model.layers.25.router.proj",
+      "model.language_model.layers.26.router",
+      "model.language_model.layers.26.router.proj",
+      "model.language_model.layers.27.router",
+      "model.language_model.layers.27.router.proj",
+      "model.language_model.layers.28.router",
+      "model.language_model.layers.28.router.proj",
+      "model.language_model.layers.29.router",
+      "model.language_model.layers.29.router.proj",
+      "model.embed_vision.embedding_projection",
+      "lm_head"
+    ],
+    "kv_cache_scheme": null,
+    "quant_method": "compressed-tensors",
+    "quantization_status": "compressed",
+    "sparsity_config": {},
+    "transform_config": {},
+    "version": "0.18.0"
+  },
+  "text_config": {
+    "attention_bias": false,
+    "attention_dropout": 0.0,
+    "attention_k_eq_v": true,
+    "bos_token_id": 2,
+    "dtype": "bfloat16",
+    "enable_moe_block": true,
+    "eos_token_id": 1,
+    "final_logit_softcapping": 30.0,
+    "head_dim": 256,
+    "hidden_activation": "gelu_pytorch_tanh",
+    "hidden_size": 2816,
+    "hidden_size_per_layer_input": 0,
+    "initializer_range": 0.02,
+    "intermediate_size": 2112,
+    "layer_types": [
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "full_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "full_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "full_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "full_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "full_attention"
+    ],
+    "max_position_embeddings": 262144,
+    "model_type": "gemma4_text",
+    "moe_intermediate_size": 704,
+    "num_attention_heads": 16,
+    "num_experts": 128,
+    "num_hidden_layers": 30,
+    "num_key_value_heads": 8,
+    "num_kv_shared_layers": 0,
+    "pad_token_id": 0,
+    "per_layer_config": {
+      "05": {
+        "head_dim": 512,
+        "num_key_value_heads": 2
+      },
+      "11": {
+        "head_dim": 512,
+        "num_key_value_heads": 2
+      },
+      "17": {
+        "head_dim": 512,
+        "num_key_value_heads": 2
+      },
+      "23": {
+        "head_dim": 512,
+        "num_key_value_heads": 2
+      },
+      "29": {
+        "head_dim": 512,
+        "num_key_value_heads": 2
+      }
+    },
+    "rms_norm_eps": 1e-06,
+    "rope_parameters": {
+      "full_attention": {
+        "partial_rotary_factor": 0.25,
+        "rope_theta": 1000000.0,
+        "rope_type": "proportional"
+      },
+      "sliding_attention": {
+        "rope_theta": 10000.0,
+        "rope_type": "default"
+      }
+    },
+    "sliding_window": 1024,
+    "tie_word_embeddings": true,
+    "top_k_experts": 8,
+    "use_bidirectional_attention": "vision",
+    "use_cache": true,
+    "use_double_wide_mlp": false,
+    "vocab_size": 262144,
+    "vocab_size_per_layer_input": 262144
+  },
+  "tie_word_embeddings": true,
+  "transformers_version": "5.15.1",
+  "video_token_id": 258884,
+  "vision_config": {
+    "_name_or_path": "",
+    "architectures": null,
+    "attention_bias": false,
+    "attention_dropout": 0.0,
+    "chunk_size_feed_forward": 0,
+    "default_output_length": 280,
+    "dtype": "bfloat16",
+    "global_head_dim": 72,
+    "head_dim": 72,
+    "hidden_activation": "gelu_pytorch_tanh",
+    "hidden_size": 1152,
+    "id2label": {
+      "0": "LABEL_0",
+      "1": "LABEL_1"
+    },
+    "initializer_range": 0.02,
+    "intermediate_size": 4304,
+    "is_encoder_decoder": false,
+    "label2id": {
+      "LABEL_0": 0,
+      "LABEL_1": 1
+    },
+    "max_position_embeddings": 131072,
+    "model_type": "gemma4_vision",
+    "num_attention_heads": 16,
+    "num_hidden_layers": 27,
+    "num_key_value_heads": 16,
+    "output_attentions": false,
+    "output_hidden_states": false,
+    "patch_size": 16,
+    "pooling_kernel_size": 3,
+    "position_embedding_size": 10240,
+    "problem_type": null,
+    "return_dict": true,
+    "rms_norm_eps": 1e-06,
+    "rope_parameters": {
+      "rope_theta": 100.0,
+      "rope_type": "default"
+    },
+    "standardize": true,
+    "use_clipped_linears": false
+  },
+  "vision_soft_tokens_per_image": 280
+}

eval/evaluate_gemma4_ppl_vllm.py

@@ -0,0 +1,122 @@
+#!/usr/bin/env python3
+"""Perplexity via vLLM prompt logprobs for a Gemma 4 text-only path.
+
+This is intentionally shared by BF16 and compressed-tensors W4A16 runs.  It
+scores the observed next token at every noninitial position in deterministic,
+contiguous held-out WikiText-2 test windows.  ``--cpu-offload-gb`` makes the
+otherwise too-large BF16 parent testable on a 24 GB card without changing the
+model or scoring implementation.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import math
+from datetime import datetime, timezone
+from pathlib import Path
+
+from datasets import load_dataset
+from transformers import AutoTokenizer
+from vllm import LLM, SamplingParams
+from vllm.inputs import TokensPrompt
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--model", type=Path, required=True)
+    parser.add_argument("--tokenizer", type=Path, required=True)
+    parser.add_argument("--label", required=True)
+    parser.add_argument("--output", type=Path, required=True)
+    parser.add_argument("--cache-dir", type=Path, required=True)
+    parser.add_argument("--num-windows", type=int, default=4)
+    parser.add_argument("--window-tokens", type=int, default=512)
+    parser.add_argument("--quantization", default=None)
+    parser.add_argument("--cpu-offload-gb", type=float, default=0.0)
+    return parser.parse_args()
+
+
+def held_out_windows(tokenizer, cache_dir: Path, num_windows: int, size: int):
+    dataset = load_dataset(
+        "Salesforce/wikitext",
+        "wikitext-2-raw-v1",
+        split="test",
+        cache_dir=str(cache_dir),
+    )
+    text = "\n\n".join(row["text"] for row in dataset if row["text"].strip())
+    ids = tokenizer(text, add_special_tokens=False)["input_ids"]
+    needed = num_windows * size
+    if len(ids) < needed:
+        raise RuntimeError(f"Need {needed} tokens, corpus yielded {len(ids)}")
+    return dataset, [ids[i * size : (i + 1) * size] for i in range(num_windows)]
+
+
+def main() -> None:
+    args = parse_args()
+    if args.window_tokens < 2:
+        raise ValueError("--window-tokens must be at least 2")
+    tokenizer = AutoTokenizer.from_pretrained(args.tokenizer, local_files_only=True)
+    dataset, windows = held_out_windows(
+        tokenizer, args.cache_dir, args.num_windows, args.window_tokens
+    )
+    llm = LLM(
+        model=str(args.model),
+        tokenizer=str(args.tokenizer),
+        dtype="bfloat16",
+        quantization=args.quantization,
+        max_model_len=args.window_tokens + 1,
+        max_num_seqs=1,
+        max_num_batched_tokens=args.window_tokens + 1,
+        gpu_memory_utilization=0.80,
+        cpu_offload_gb=args.cpu_offload_gb,
+        language_model_only=True,
+        limit_mm_per_prompt={"image": 0, "video": 0},
+    )
+    params = SamplingParams(
+        temperature=0.0,
+        max_tokens=1,
+        ignore_eos=True,
+        prompt_logprobs=1,
+        detokenize=False,
+    )
+    outputs = llm.generate(
+        [TokensPrompt(prompt_token_ids=ids) for ids in windows], params, use_tqdm=False
+    )
+    nll = 0.0
+    token_count = 0
+    for window, output in zip(windows, outputs, strict=True):
+        values = output.prompt_logprobs
+        if values is None or len(values) != len(window):
+            raise RuntimeError("vLLM did not return one prompt-logprob entry per prompt token")
+        for token_id, entry in zip(window[1:], values[1:], strict=True):
+            if entry is None or token_id not in entry:
+                raise RuntimeError("observed token missing from prompt-logprob response")
+            nll -= entry[token_id].logprob
+            token_count += 1
+    result = {
+        "label": args.label,
+        "model": args.model.name,
+        "tokenizer": args.tokenizer.name,
+        "dataset": "Salesforce/wikitext",
+        "dataset_config": "wikitext-2-raw-v1",
+        "split": "test",
+        "dataset_fingerprint": dataset._fingerprint,
+        "window_selection": "first contiguous non-empty test-corpus token windows",
+        "num_windows": args.num_windows,
+        "window_tokens": args.window_tokens,
+        "evaluated_next_tokens": token_count,
+        "nll_sum": nll,
+        "mean_nll": nll / token_count,
+        "perplexity": math.exp(nll / token_count),
+        "engine": "vLLM prompt_logprobs=1",
+        "quantization": args.quantization,
+        "cpu_offload_gb": args.cpu_offload_gb,
+        "utc": datetime.now(timezone.utc).isoformat(),
+    }
+    args.output.parent.mkdir(parents=True, exist_ok=True)
+    args.output.write_text(json.dumps(result, indent=2) + "\n")
+    print(json.dumps(result, indent=2))
+
+
+if __name__ == "__main__":
+    main()

examples/serve_vllm_3090_128k.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+# Validated reference profile: one RTX 3090 24 GB, image-focused Gemma 4 use.
+set -euo pipefail
+
+CUDA_VISIBLE_DEVICES=1 vllm serve useful-quants/Gemma-4-26B-A4B-it-W4A16-G64-BF16Vision \
+  --quantization compressed-tensors \
+  --dtype bfloat16 \
+  --max-model-len 131072 \
+  --kv-cache-memory-bytes 6444425401 \
+  --max-num-seqs 17 \
+  --max-num-batched-tokens 8192 \
+  --enable-chunked-prefill \
+  --enable-prefix-caching \
+  --scheduling-policy priority \
+  --limit-mm-per-prompt '{"image":1,"video":0}' \
+  --host 0.0.0.0 \
+  --port 8000

generation_config.json

@@ -0,0 +1,13 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 2,
+  "eos_token_id": [
+    1,
+    106
+  ],
+  "output_attentions": false,
+  "output_hidden_states": false,
+  "pad_token_id": 0,
+  "transformers_version": "5.15.1",
+  "use_cache": true
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4475b48270e4d2072886f084442e29f5e5eb16c7668d5a2a28404428e9c0b247
+size 15656878852

processor_config.json

@@ -0,0 +1,75 @@
+{
+  "audio_ms_per_token": 40,
+  "audio_seq_length": 750,
+  "feature_extractor": {
+    "dither": 0.0,
+    "feature_extractor_type": "Gemma4AudioFeatureExtractor",
+    "feature_size": 128,
+    "fft_length": 512,
+    "fft_overdrive": false,
+    "frame_length": 320,
+    "hop_length": 160,
+    "input_scale_factor": 1.0,
+    "max_frequency": 8000.0,
+    "mel_floor": 0.001,
+    "min_frequency": 0.0,
+    "padding_side": "right",
+    "padding_value": 0.0,
+    "per_bin_mean": null,
+    "per_bin_stddev": null,
+    "preemphasis": 0.0,
+    "preemphasis_htk_flavor": true,
+    "return_attention_mask": true,
+    "sampling_rate": 16000
+  },
+  "image_processor": {
+    "do_convert_rgb": true,
+    "do_normalize": false,
+    "do_rescale": true,
+    "do_resize": true,
+    "image_mean": [
+      0.0,
+      0.0,
+      0.0
+    ],
+    "image_processor_type": "Gemma4ImageProcessor",
+    "image_seq_length": 280,
+    "image_std": [
+      1.0,
+      1.0,
+      1.0
+    ],
+    "max_soft_tokens": 280,
+    "patch_size": 16,
+    "pooling_kernel_size": 3,
+    "resample": 3,
+    "rescale_factor": 0.00392156862745098
+  },
+  "image_seq_length": 280,
+  "processor_class": "Gemma4Processor",
+  "video_processor": {
+    "do_convert_rgb": true,
+    "do_normalize": true,
+    "do_rescale": true,
+    "do_resize": true,
+    "do_sample_frames": true,
+    "image_mean": [
+      0.0,
+      0.0,
+      0.0
+    ],
+    "image_std": [
+      1.0,
+      1.0,
+      1.0
+    ],
+    "max_soft_tokens": 70,
+    "num_frames": 32,
+    "patch_size": 16,
+    "pooling_kernel_size": 3,
+    "resample": 3,
+    "rescale_factor": 0.00392156862745098,
+    "return_metadata": false,
+    "video_processor_type": "Gemma4VideoProcessor"
+  }
+}

provenance/artifact-audit.json

@@ -0,0 +1,312 @@
+{
+  "model_file": "model.safetensors",
+  "model_file_bytes": 15656878852,
+  "model_file_sha256": "4475b48270e4d2072886f084442e29f5e5eb16c7668d5a2a28404428e9c0b247",
+  "total_tensor_count": 35923,
+  "weight_packed_tensor_count": 11725,
+  "weight_scale_tensor_count": 11725,
+  "weight_scale_nonfinite_tensor_count": 0,
+  "weight_scale_nonfinite_value_count": 0,
+  "packed_exclusion_counts": {
+    "vision_tower": 0,
+    "router": 0,
+    "token_embedding": 0,
+    "vision_embedding": 0,
+    "lm_head": 0
+  },
+  "representative_excluded_weight_dtypes": {
+    "token_embedding": "bfloat16",
+    "vision_projection": "bfloat16",
+    "router": "bfloat16",
+    "vision_mlp": "bfloat16"
+  },
+  "separate_lm_head_tensor_present": false,
+  "quantization_config": {
+    "config_groups": {
+      "group_0": {
+        "format": "pack-quantized",
+        "input_activations": null,
+        "output_activations": null,
+        "targets": [
+          "Linear"
+        ],
+        "weights": {
+          "actorder": "static",
+          "block_structure": null,
+          "dynamic": false,
+          "group_size": 64,
+          "num_bits": 4,
+          "observer": "memoryless_minmax",
+          "observer_kwargs": {},
+          "scale_dtype": null,
+          "strategy": "group",
+          "symmetric": true,
+          "type": "int",
+          "zp_dtype": null
+        }
+      }
+    },
+    "format": "pack-quantized",
+    "global_compression_ratio": null,
+    "ignore": [
+      "model.vision_tower.patch_embedder.input_proj",
+      "model.vision_tower.encoder.layers.0.self_attn.q_proj.linear",
+      "model.vision_tower.encoder.layers.0.self_attn.k_proj.linear",
+      "model.vision_tower.encoder.layers.0.self_attn.v_proj.linear",
+      "model.vision_tower.encoder.layers.0.self_attn.o_proj.linear",
+      "model.vision_tower.encoder.layers.0.mlp.gate_proj.linear",
+      "model.vision_tower.encoder.layers.0.mlp.up_proj.linear",
+      "model.vision_tower.encoder.layers.0.mlp.down_proj.linear",
+      "model.vision_tower.encoder.layers.1.self_attn.q_proj.linear",
+      "model.vision_tower.encoder.layers.1.self_attn.k_proj.linear",
+      "model.vision_tower.encoder.layers.1.self_attn.v_proj.linear",
+      "model.vision_tower.encoder.layers.1.self_attn.o_proj.linear",
+      "model.vision_tower.encoder.layers.1.mlp.gate_proj.linear",
+      "model.vision_tower.encoder.layers.1.mlp.up_proj.linear",
+      "model.vision_tower.encoder.layers.1.mlp.down_proj.linear",
+      "model.vision_tower.encoder.layers.2.self_attn.q_proj.linear",
+      "model.vision_tower.encoder.layers.2.self_attn.k_proj.linear",
+      "model.vision_tower.encoder.layers.2.self_attn.v_proj.linear",
+      "model.vision_tower.encoder.layers.2.self_attn.o_proj.linear",
+      "model.vision_tower.encoder.layers.2.mlp.gate_proj.linear",
+      "model.vision_tower.encoder.layers.2.mlp.up_proj.linear",
+      "model.vision_tower.encoder.layers.2.mlp.down_proj.linear",
+      "model.vision_tower.encoder.layers.3.self_attn.q_proj.linear",
+      "model.vision_tower.encoder.layers.3.self_attn.k_proj.linear",
+      "model.vision_tower.encoder.layers.3.self_attn.v_proj.linear",
+      "model.vision_tower.encoder.layers.3.self_attn.o_proj.linear",
+      "model.vision_tower.encoder.layers.3.mlp.gate_proj.linear",
+      "model.vision_tower.encoder.layers.3.mlp.up_proj.linear",
+      "model.vision_tower.encoder.layers.3.mlp.down_proj.linear",
+      "model.vision_tower.encoder.layers.4.self_attn.q_proj.linear",
+      "model.vision_tower.encoder.layers.4.self_attn.k_proj.linear",
+      "model.vision_tower.encoder.layers.4.self_attn.v_proj.linear",
+      "model.vision_tower.encoder.layers.4.self_attn.o_proj.linear",
+      "model.vision_tower.encoder.layers.4.mlp.gate_proj.linear",
+      "model.vision_tower.encoder.layers.4.mlp.up_proj.linear",
+      "model.vision_tower.encoder.layers.4.mlp.down_proj.linear",
+      "model.vision_tower.encoder.layers.5.self_attn.q_proj.linear",
+      "model.vision_tower.encoder.layers.5.self_attn.k_proj.linear",
+      "model.vision_tower.encoder.layers.5.self_attn.v_proj.linear",
+      "model.vision_tower.encoder.layers.5.self_attn.o_proj.linear",
+      "model.vision_tower.encoder.layers.5.mlp.gate_proj.linear",
+      "model.vision_tower.encoder.layers.5.mlp.up_proj.linear",
+      "model.vision_tower.encoder.layers.5.mlp.down_proj.linear",
+      "model.vision_tower.encoder.layers.6.self_attn.q_proj.linear",
+      "model.vision_tower.encoder.layers.6.self_attn.k_proj.linear",
+      "model.vision_tower.encoder.layers.6.self_attn.v_proj.linear",
+      "model.vision_tower.encoder.layers.6.self_attn.o_proj.linear",
+      "model.vision_tower.encoder.layers.6.mlp.gate_proj.linear",
+      "model.vision_tower.encoder.layers.6.mlp.up_proj.linear",
+      "model.vision_tower.encoder.layers.6.mlp.down_proj.linear",
+      "model.vision_tower.encoder.layers.7.self_attn.q_proj.linear",
+      "model.vision_tower.encoder.layers.7.self_attn.k_proj.linear",
+      "model.vision_tower.encoder.layers.7.self_attn.v_proj.linear",
+      "model.vision_tower.encoder.layers.7.self_attn.o_proj.linear",
+      "model.vision_tower.encoder.layers.7.mlp.gate_proj.linear",
+      "model.vision_tower.encoder.layers.7.mlp.up_proj.linear",
+      "model.vision_tower.encoder.layers.7.mlp.down_proj.linear",
+      "model.vision_tower.encoder.layers.8.self_attn.q_proj.linear",
+      "model.vision_tower.encoder.layers.8.self_attn.k_proj.linear",
+      "model.vision_tower.encoder.layers.8.self_attn.v_proj.linear",
+      "model.vision_tower.encoder.layers.8.self_attn.o_proj.linear",
+      "model.vision_tower.encoder.layers.8.mlp.gate_proj.linear",
+      "model.vision_tower.encoder.layers.8.mlp.up_proj.linear",
+      "model.vision_tower.encoder.layers.8.mlp.down_proj.linear",
+      "model.vision_tower.encoder.layers.9.self_attn.q_proj.linear",
+      "model.vision_tower.encoder.layers.9.self_attn.k_proj.linear",
+      "model.vision_tower.encoder.layers.9.self_attn.v_proj.linear",
+      "model.vision_tower.encoder.layers.9.self_attn.o_proj.linear",
+      "model.vision_tower.encoder.layers.9.mlp.gate_proj.linear",
+      "model.vision_tower.encoder.layers.9.mlp.up_proj.linear",
+      "model.vision_tower.encoder.layers.9.mlp.down_proj.linear",
+      "model.vision_tower.encoder.layers.10.self_attn.q_proj.linear",
+      "model.vision_tower.encoder.layers.10.self_attn.k_proj.linear",
+      "model.vision_tower.encoder.layers.10.self_attn.v_proj.linear",
+      "model.vision_tower.encoder.layers.10.self_attn.o_proj.linear",
+      "model.vision_tower.encoder.layers.10.mlp.gate_proj.linear",
+      "model.vision_tower.encoder.layers.10.mlp.up_proj.linear",
+      "model.vision_tower.encoder.layers.10.mlp.down_proj.linear",
+      "model.vision_tower.encoder.layers.11.self_attn.q_proj.linear",
+      "model.vision_tower.encoder.layers.11.self_attn.k_proj.linear",
+      "model.vision_tower.encoder.layers.11.self_attn.v_proj.linear",
+      "model.vision_tower.encoder.layers.11.self_attn.o_proj.linear",
+      "model.vision_tower.encoder.layers.11.mlp.gate_proj.linear",
+      "model.vision_tower.encoder.layers.11.mlp.up_proj.linear",
+      "model.vision_tower.encoder.layers.11.mlp.down_proj.linear",
+      "model.vision_tower.encoder.layers.12.self_attn.q_proj.linear",
+      "model.vision_tower.encoder.layers.12.self_attn.k_proj.linear",
+      "model.vision_tower.encoder.layers.12.self_attn.v_proj.linear",
+      "model.vision_tower.encoder.layers.12.self_attn.o_proj.linear",
+      "model.vision_tower.encoder.layers.12.mlp.gate_proj.linear",
+      "model.vision_tower.encoder.layers.12.mlp.up_proj.linear",
+      "model.vision_tower.encoder.layers.12.mlp.down_proj.linear",
+      "model.vision_tower.encoder.layers.13.self_attn.q_proj.linear",
+      "model.vision_tower.encoder.layers.13.self_attn.k_proj.linear",
+      "model.vision_tower.encoder.layers.13.self_attn.v_proj.linear",
+      "model.vision_tower.encoder.layers.13.self_attn.o_proj.linear",
+      "model.vision_tower.encoder.layers.13.mlp.gate_proj.linear",
+      "model.vision_tower.encoder.layers.13.mlp.up_proj.linear",
+      "model.vision_tower.encoder.layers.13.mlp.down_proj.linear",
+      "model.vision_tower.encoder.layers.14.self_attn.q_proj.linear",
+      "model.vision_tower.encoder.layers.14.self_attn.k_proj.linear",
+      "model.vision_tower.encoder.layers.14.self_attn.v_proj.linear",
+      "model.vision_tower.encoder.layers.14.self_attn.o_proj.linear",
+      "model.vision_tower.encoder.layers.14.mlp.gate_proj.linear",
+      "model.vision_tower.encoder.layers.14.mlp.up_proj.linear",
+      "model.vision_tower.encoder.layers.14.mlp.down_proj.linear",
+      "model.vision_tower.encoder.layers.15.self_attn.q_proj.linear",
+      "model.vision_tower.encoder.layers.15.self_attn.k_proj.linear",
+      "model.vision_tower.encoder.layers.15.self_attn.v_proj.linear",
+      "model.vision_tower.encoder.layers.15.self_attn.o_proj.linear",
+      "model.vision_tower.encoder.layers.15.mlp.gate_proj.linear",
+      "model.vision_tower.encoder.layers.15.mlp.up_proj.linear",
+      "model.vision_tower.encoder.layers.15.mlp.down_proj.linear",
+      "model.vision_tower.encoder.layers.16.self_attn.q_proj.linear",
+      "model.vision_tower.encoder.layers.16.self_attn.k_proj.linear",
+      "model.vision_tower.encoder.layers.16.self_attn.v_proj.linear",
+      "model.vision_tower.encoder.layers.16.self_attn.o_proj.linear",
+      "model.vision_tower.encoder.layers.16.mlp.gate_proj.linear",
+      "model.vision_tower.encoder.layers.16.mlp.up_proj.linear",
+      "model.vision_tower.encoder.layers.16.mlp.down_proj.linear",
+      "model.vision_tower.encoder.layers.17.self_attn.q_proj.linear",
+      "model.vision_tower.encoder.layers.17.self_attn.k_proj.linear",
+      "model.vision_tower.encoder.layers.17.self_attn.v_proj.linear",
+      "model.vision_tower.encoder.layers.17.self_attn.o_proj.linear",
+      "model.vision_tower.encoder.layers.17.mlp.gate_proj.linear",
+      "model.vision_tower.encoder.layers.17.mlp.up_proj.linear",
+      "model.vision_tower.encoder.layers.17.mlp.down_proj.linear",
+      "model.vision_tower.encoder.layers.18.self_attn.q_proj.linear",
+      "model.vision_tower.encoder.layers.18.self_attn.k_proj.linear",
+      "model.vision_tower.encoder.layers.18.self_attn.v_proj.linear",
+      "model.vision_tower.encoder.layers.18.self_attn.o_proj.linear",
+      "model.vision_tower.encoder.layers.18.mlp.gate_proj.linear",
+      "model.vision_tower.encoder.layers.18.mlp.up_proj.linear",
+      "model.vision_tower.encoder.layers.18.mlp.down_proj.linear",
+      "model.vision_tower.encoder.layers.19.self_attn.q_proj.linear",
+      "model.vision_tower.encoder.layers.19.self_attn.k_proj.linear",
+      "model.vision_tower.encoder.layers.19.self_attn.v_proj.linear",
+      "model.vision_tower.encoder.layers.19.self_attn.o_proj.linear",
+      "model.vision_tower.encoder.layers.19.mlp.gate_proj.linear",
+      "model.vision_tower.encoder.layers.19.mlp.up_proj.linear",
+      "model.vision_tower.encoder.layers.19.mlp.down_proj.linear",
+      "model.vision_tower.encoder.layers.20.self_attn.q_proj.linear",
+      "model.vision_tower.encoder.layers.20.self_attn.k_proj.linear",
+      "model.vision_tower.encoder.layers.20.self_attn.v_proj.linear",
+      "model.vision_tower.encoder.layers.20.self_attn.o_proj.linear",
+      "model.vision_tower.encoder.layers.20.mlp.gate_proj.linear",
+      "model.vision_tower.encoder.layers.20.mlp.up_proj.linear",
+      "model.vision_tower.encoder.layers.20.mlp.down_proj.linear",
+      "model.vision_tower.encoder.layers.21.self_attn.q_proj.linear",
+      "model.vision_tower.encoder.layers.21.self_attn.k_proj.linear",
+      "model.vision_tower.encoder.layers.21.self_attn.v_proj.linear",
+      "model.vision_tower.encoder.layers.21.self_attn.o_proj.linear",
+      "model.vision_tower.encoder.layers.21.mlp.gate_proj.linear",
+      "model.vision_tower.encoder.layers.21.mlp.up_proj.linear",
+      "model.vision_tower.encoder.layers.21.mlp.down_proj.linear",
+      "model.vision_tower.encoder.layers.22.self_attn.q_proj.linear",
+      "model.vision_tower.encoder.layers.22.self_attn.k_proj.linear",
+      "model.vision_tower.encoder.layers.22.self_attn.v_proj.linear",
+      "model.vision_tower.encoder.layers.22.self_attn.o_proj.linear",
+      "model.vision_tower.encoder.layers.22.mlp.gate_proj.linear",
+      "model.vision_tower.encoder.layers.22.mlp.up_proj.linear",
+      "model.vision_tower.encoder.layers.22.mlp.down_proj.linear",
+      "model.vision_tower.encoder.layers.23.self_attn.q_proj.linear",
+      "model.vision_tower.encoder.layers.23.self_attn.k_proj.linear",
+      "model.vision_tower.encoder.layers.23.self_attn.v_proj.linear",
+      "model.vision_tower.encoder.layers.23.self_attn.o_proj.linear",
+      "model.vision_tower.encoder.layers.23.mlp.gate_proj.linear",
+      "model.vision_tower.encoder.layers.23.mlp.up_proj.linear",
+      "model.vision_tower.encoder.layers.23.mlp.down_proj.linear",
+      "model.vision_tower.encoder.layers.24.self_attn.q_proj.linear",
+      "model.vision_tower.encoder.layers.24.self_attn.k_proj.linear",
+      "model.vision_tower.encoder.layers.24.self_attn.v_proj.linear",
+      "model.vision_tower.encoder.layers.24.self_attn.o_proj.linear",
+      "model.vision_tower.encoder.layers.24.mlp.gate_proj.linear",
+      "model.vision_tower.encoder.layers.24.mlp.up_proj.linear",
+      "model.vision_tower.encoder.layers.24.mlp.down_proj.linear",
+      "model.vision_tower.encoder.layers.25.self_attn.q_proj.linear",
+      "model.vision_tower.encoder.layers.25.self_attn.k_proj.linear",
+      "model.vision_tower.encoder.layers.25.self_attn.v_proj.linear",
+      "model.vision_tower.encoder.layers.25.self_attn.o_proj.linear",
+      "model.vision_tower.encoder.layers.25.mlp.gate_proj.linear",
+      "model.vision_tower.encoder.layers.25.mlp.up_proj.linear",
+      "model.vision_tower.encoder.layers.25.mlp.down_proj.linear",
+      "model.vision_tower.encoder.layers.26.self_attn.q_proj.linear",
+      "model.vision_tower.encoder.layers.26.self_attn.k_proj.linear",
+      "model.vision_tower.encoder.layers.26.self_attn.v_proj.linear",
+      "model.vision_tower.encoder.layers.26.self_attn.o_proj.linear",
+      "model.vision_tower.encoder.layers.26.mlp.gate_proj.linear",
+      "model.vision_tower.encoder.layers.26.mlp.up_proj.linear",
+      "model.vision_tower.encoder.layers.26.mlp.down_proj.linear",
+      "model.language_model.layers.0.router",
+      "model.language_model.layers.0.router.proj",
+      "model.language_model.layers.1.router",
+      "model.language_model.layers.1.router.proj",
+      "model.language_model.layers.2.router",
+      "model.language_model.layers.2.router.proj",
+      "model.language_model.layers.3.router",
+      "model.language_model.layers.3.router.proj",
+      "model.language_model.layers.4.router",
+      "model.language_model.layers.4.router.proj",
+      "model.language_model.layers.5.router",
+      "model.language_model.layers.5.router.proj",
+      "model.language_model.layers.6.router",
+      "model.language_model.layers.6.router.proj",
+      "model.language_model.layers.7.router",
+      "model.language_model.layers.7.router.proj",
+      "model.language_model.layers.8.router",
+      "model.language_model.layers.8.router.proj",
+      "model.language_model.layers.9.router",
+      "model.language_model.layers.9.router.proj",
+      "model.language_model.layers.10.router",
+      "model.language_model.layers.10.router.proj",
+      "model.language_model.layers.11.router",
+      "model.language_model.layers.11.router.proj",
+      "model.language_model.layers.12.router",
+      "model.language_model.layers.12.router.proj",
+      "model.language_model.layers.13.router",
+      "model.language_model.layers.13.router.proj",
+      "model.language_model.layers.14.router",
+      "model.language_model.layers.14.router.proj",
+      "model.language_model.layers.15.router",
+      "model.language_model.layers.15.router.proj",
+      "model.language_model.layers.16.router",
+      "model.language_model.layers.16.router.proj",
+      "model.language_model.layers.17.router",
+      "model.language_model.layers.17.router.proj",
+      "model.language_model.layers.18.router",
+      "model.language_model.layers.18.router.proj",
+      "model.language_model.layers.19.router",
+      "model.language_model.layers.19.router.proj",
+      "model.language_model.layers.20.router",
+      "model.language_model.layers.20.router.proj",
+      "model.language_model.layers.21.router",
+      "model.language_model.layers.21.router.proj",
+      "model.language_model.layers.22.router",
+      "model.language_model.layers.22.router.proj",
+      "model.language_model.layers.23.router",
+      "model.language_model.layers.23.router.proj",
+      "model.language_model.layers.24.router",
+      "model.language_model.layers.24.router.proj",
+      "model.language_model.layers.25.router",
+      "model.language_model.layers.25.router.proj",
+      "model.language_model.layers.26.router",
+      "model.language_model.layers.26.router.proj",
+      "model.language_model.layers.27.router",
+      "model.language_model.layers.27.router.proj",
+      "model.language_model.layers.28.router",
+      "model.language_model.layers.28.router.proj",
+      "model.language_model.layers.29.router",
+      "model.language_model.layers.29.router.proj",
+      "model.embed_vision.embedding_projection",
+      "lm_head"
+    ],
+    "kv_cache_scheme": null,
+    "quant_method": "compressed-tensors",
+    "quantization_status": "compressed",
+    "sparsity_config": {},
+    "transform_config": {},
+    "version": "0.18.0"
+  }
+}

provenance/calibration-and-source.json

@@ -0,0 +1,384 @@
+{
+  "lineage": {
+    "original_model_family": "google/gemma-4-26B-A4B-it",
+    "qat_parent_lineage": "google/gemma-4-26B-A4B-it-qat-q4_0-unquantized",
+    "immediate_quantization_parent": "Mitchins/gemma-4-26B-A4B-it-qat-q4_0-unquantized-uncensored-heretic",
+    "immediate_parent_revision": "d62fa836581401beafd08fdd624927e31dcac6ee",
+    "immediate_parent_relation": "QAT/Heretic tuned and uncensored derivative; this W4A16 artifact quantizes that parent, not stock Google IT weights.",
+    "license": "Apache-2.0 inherited from the parent"
+  },
+  "source_file_sha256": {
+    "model-00001-of-00011.safetensors": "0a8ef8c2e3a967601b62b2bd052f9327c5e80d844bff495cdabba0a32c988fd9",
+    "model-00002-of-00011.safetensors": "b643e01d2a56cffd216f4cb4025bf312142af46127a3f3701fe1548758edfdfe",
+    "model-00003-of-00011.safetensors": "724dd311e180744593e7cff6dc4a3dbd51cccf39c3fe23e8589121f7a125576e",
+    "model-00004-of-00011.safetensors": "01e459f49f7b5c7dfa9f4fba53a78af0f9978ca49b56b0404008116a6b171abf",
+    "model-00005-of-00011.safetensors": "e800b4e729edc6f54672006150ad9bcec046b0b5b2d620cabe277242b52acaab",
+    "model-00006-of-00011.safetensors": "0a90f287557e40ebedee1d595035b760642217c8960f8cf1b494034c6d4cee86",
+    "model-00007-of-00011.safetensors": "3b796785e6f9f67d74d75facaf1ec95651d3409def2c67afc1a3b4237a9f544b",
+    "model-00008-of-00011.safetensors": "5dd1bf454448f1e1f7f58e68eb4908b557d89e6cfd04cc13731b490635dc64e6",
+    "model-00009-of-00011.safetensors": "1887636919fe8d0326964a3645a867bbe9ecf8c4c9202c2a8d2927cfa2725b46",
+    "model-00010-of-00011.safetensors": "36498727d58f158cb29e7d93dbbf457bf8fb0e883859888f6db2cd3f5fe585b2",
+    "model-00011-of-00011.safetensors": "33918a3f3e04864eba494e5515fd9b0575c2a923d0cf080d12b6fb922d41bbeb",
+    "config.json": "fb61827d95f23593a13dbc03023ac4892f31495b0348f2f5ef9a720c7568f772",
+    "generation_config.json": "4985d1a588cdbd922d99c3ccad042b0493773a499937fa248d5849ef4afb4101",
+    "model.safetensors.index.json": "1e9e4718c49193ff47e2ca0847dfad3cfa5e57e6799463994284fc3284f93c7d",
+    "processor_config.json": "32bdf45d2ad4cc29a0822ddd157a182de76644f0419a6228d151495256e9813c",
+    "tokenizer.json": "cc8d3a0ce36466ccc1278bf987df5f71db1719b9ca6b4118264f45cb627bfe0f",
+    "tokenizer_config.json": "b8045a4576903e86903291d5cbdd4adfc8859e9ce3c98621bdbd957f73ed394b",
+    "chat_template.jinja": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4"
+  },
+  "source_architecture": {
+    "model_type": "gemma4",
+    "architectures": [
+      "Gemma4ForConditionalGeneration"
+    ],
+    "text_max_position_embeddings": 262144,
+    "intermediate_size": 2112,
+    "moe_intermediate_size": 704
+  },
+  "calibration": {
+    "seed": 20260812,
+    "text_windows": 512,
+    "tokens_per_text_window": 512,
+    "image_samples": 1024,
+    "image_source": "COCO 2017 validation",
+    "source_urls": [
+      "https://s3.amazonaws.com/images.cocodataset.org/zips/val2017.zip",
+      "https://s3.amazonaws.com/images.cocodataset.org/annotations/annotations_trainval2017.zip"
+    ],
+    "text_sources": [
+      {
+        "dataset": "Salesforce/wikitext",
+        "config": "wikitext-103-raw-v1",
+        "count": 192
+      },
+      {
+        "dataset": "HuggingFaceTB/smoltalk",
+        "config": "openhermes-100k",
+        "count": 192
+      },
+      {
+        "dataset": "HuggingFaceTB/smoltalk",
+        "config": "apigen-80k",
+        "count": 128
+      }
+    ]
+  },
+  "quantization_run": {
+    "weight_bits": 4,
+    "activation_bits": 16,
+    "group_size": 64,
+    "algorithm": "GPTQ",
+    "pipeline": "sequential",
+    "max_seq_length": 1024,
+    "moe_calibrate_all_experts": true,
+    "ignore": [
+      "re:.*lm_head$",
+      "re:.*vision_tower.*",
+      "re:.*router.*",
+      "re:.*embed_tokens.*",
+      "re:.*embed_vision.*"
+    ],
+    "sample_layout": "first 512 image+text, remaining 512 image-instruction",
+    "limit_mm_per_prompt": {
+      "image": 1,
+      "video": 0
+    }
+  },
+  "conversion_recipe": {
+    "config_groups": {
+      "group_0": {
+        "format": "pack-quantized",
+        "input_activations": null,
+        "output_activations": null,
+        "targets": [
+          "Linear"
+        ],
+        "weights": {
+          "actorder": "static",
+          "block_structure": null,
+          "dynamic": false,
+          "group_size": 64,
+          "num_bits": 4,
+          "observer": "memoryless_minmax",
+          "observer_kwargs": {},
+          "scale_dtype": null,
+          "strategy": "group",
+          "symmetric": true,
+          "type": "int",
+          "zp_dtype": null
+        }
+      }
+    },
+    "format": "pack-quantized",
+    "global_compression_ratio": null,
+    "ignore": [
+      "model.vision_tower.patch_embedder.input_proj",
+      "model.vision_tower.encoder.layers.0.self_attn.q_proj.linear",
+      "model.vision_tower.encoder.layers.0.self_attn.k_proj.linear",
+      "model.vision_tower.encoder.layers.0.self_attn.v_proj.linear",
+      "model.vision_tower.encoder.layers.0.self_attn.o_proj.linear",
+      "model.vision_tower.encoder.layers.0.mlp.gate_proj.linear",
+      "model.vision_tower.encoder.layers.0.mlp.up_proj.linear",
+      "model.vision_tower.encoder.layers.0.mlp.down_proj.linear",
+      "model.vision_tower.encoder.layers.1.self_attn.q_proj.linear",
+      "model.vision_tower.encoder.layers.1.self_attn.k_proj.linear",
+      "model.vision_tower.encoder.layers.1.self_attn.v_proj.linear",
+      "model.vision_tower.encoder.layers.1.self_attn.o_proj.linear",
+      "model.vision_tower.encoder.layers.1.mlp.gate_proj.linear",
+      "model.vision_tower.encoder.layers.1.mlp.up_proj.linear",
+      "model.vision_tower.encoder.layers.1.mlp.down_proj.linear",
+      "model.vision_tower.encoder.layers.2.self_attn.q_proj.linear",
+      "model.vision_tower.encoder.layers.2.self_attn.k_proj.linear",
+      "model.vision_tower.encoder.layers.2.self_attn.v_proj.linear",
+      "model.vision_tower.encoder.layers.2.self_attn.o_proj.linear",
+      "model.vision_tower.encoder.layers.2.mlp.gate_proj.linear",
+      "model.vision_tower.encoder.layers.2.mlp.up_proj.linear",
+      "model.vision_tower.encoder.layers.2.mlp.down_proj.linear",
+      "model.vision_tower.encoder.layers.3.self_attn.q_proj.linear",
+      "model.vision_tower.encoder.layers.3.self_attn.k_proj.linear",
+      "model.vision_tower.encoder.layers.3.self_attn.v_proj.linear",
+      "model.vision_tower.encoder.layers.3.self_attn.o_proj.linear",
+      "model.vision_tower.encoder.layers.3.mlp.gate_proj.linear",
+      "model.vision_tower.encoder.layers.3.mlp.up_proj.linear",
+      "model.vision_tower.encoder.layers.3.mlp.down_proj.linear",
+      "model.vision_tower.encoder.layers.4.self_attn.q_proj.linear",
+      "model.vision_tower.encoder.layers.4.self_attn.k_proj.linear",
+      "model.vision_tower.encoder.layers.4.self_attn.v_proj.linear",
+      "model.vision_tower.encoder.layers.4.self_attn.o_proj.linear",
+      "model.vision_tower.encoder.layers.4.mlp.gate_proj.linear",
+      "model.vision_tower.encoder.layers.4.mlp.up_proj.linear",
+      "model.vision_tower.encoder.layers.4.mlp.down_proj.linear",
+      "model.vision_tower.encoder.layers.5.self_attn.q_proj.linear",
+      "model.vision_tower.encoder.layers.5.self_attn.k_proj.linear",
+      "model.vision_tower.encoder.layers.5.self_attn.v_proj.linear",
+      "model.vision_tower.encoder.layers.5.self_attn.o_proj.linear",
+      "model.vision_tower.encoder.layers.5.mlp.gate_proj.linear",
+      "model.vision_tower.encoder.layers.5.mlp.up_proj.linear",
+      "model.vision_tower.encoder.layers.5.mlp.down_proj.linear",
+      "model.vision_tower.encoder.layers.6.self_attn.q_proj.linear",
+      "model.vision_tower.encoder.layers.6.self_attn.k_proj.linear",
+      "model.vision_tower.encoder.layers.6.self_attn.v_proj.linear",
+      "model.vision_tower.encoder.layers.6.self_attn.o_proj.linear",
+      "model.vision_tower.encoder.layers.6.mlp.gate_proj.linear",
+      "model.vision_tower.encoder.layers.6.mlp.up_proj.linear",
+      "model.vision_tower.encoder.layers.6.mlp.down_proj.linear",
+      "model.vision_tower.encoder.layers.7.self_attn.q_proj.linear",
+      "model.vision_tower.encoder.layers.7.self_attn.k_proj.linear",
+      "model.vision_tower.encoder.layers.7.self_attn.v_proj.linear",
+      "model.vision_tower.encoder.layers.7.self_attn.o_proj.linear",
+      "model.vision_tower.encoder.layers.7.mlp.gate_proj.linear",
+      "model.vision_tower.encoder.layers.7.mlp.up_proj.linear",
+      "model.vision_tower.encoder.layers.7.mlp.down_proj.linear",
+      "model.vision_tower.encoder.layers.8.self_attn.q_proj.linear",
+      "model.vision_tower.encoder.layers.8.self_attn.k_proj.linear",
+      "model.vision_tower.encoder.layers.8.self_attn.v_proj.linear",
+      "model.vision_tower.encoder.layers.8.self_attn.o_proj.linear",
+      "model.vision_tower.encoder.layers.8.mlp.gate_proj.linear",
+      "model.vision_tower.encoder.layers.8.mlp.up_proj.linear",
+      "model.vision_tower.encoder.layers.8.mlp.down_proj.linear",
+      "model.vision_tower.encoder.layers.9.self_attn.q_proj.linear",
+      "model.vision_tower.encoder.layers.9.self_attn.k_proj.linear",
+      "model.vision_tower.encoder.layers.9.self_attn.v_proj.linear",
+      "model.vision_tower.encoder.layers.9.self_attn.o_proj.linear",
+      "model.vision_tower.encoder.layers.9.mlp.gate_proj.linear",
+      "model.vision_tower.encoder.layers.9.mlp.up_proj.linear",
+      "model.vision_tower.encoder.layers.9.mlp.down_proj.linear",
+      "model.vision_tower.encoder.layers.10.self_attn.q_proj.linear",
+      "model.vision_tower.encoder.layers.10.self_attn.k_proj.linear",
+      "model.vision_tower.encoder.layers.10.self_attn.v_proj.linear",
+      "model.vision_tower.encoder.layers.10.self_attn.o_proj.linear",
+      "model.vision_tower.encoder.layers.10.mlp.gate_proj.linear",
+      "model.vision_tower.encoder.layers.10.mlp.up_proj.linear",
+      "model.vision_tower.encoder.layers.10.mlp.down_proj.linear",
+      "model.vision_tower.encoder.layers.11.self_attn.q_proj.linear",
+      "model.vision_tower.encoder.layers.11.self_attn.k_proj.linear",
+      "model.vision_tower.encoder.layers.11.self_attn.v_proj.linear",
+      "model.vision_tower.encoder.layers.11.self_attn.o_proj.linear",
+      "model.vision_tower.encoder.layers.11.mlp.gate_proj.linear",
+      "model.vision_tower.encoder.layers.11.mlp.up_proj.linear",
+      "model.vision_tower.encoder.layers.11.mlp.down_proj.linear",
+      "model.vision_tower.encoder.layers.12.self_attn.q_proj.linear",
+      "model.vision_tower.encoder.layers.12.self_attn.k_proj.linear",
+      "model.vision_tower.encoder.layers.12.self_attn.v_proj.linear",
+      "model.vision_tower.encoder.layers.12.self_attn.o_proj.linear",
+      "model.vision_tower.encoder.layers.12.mlp.gate_proj.linear",
+      "model.vision_tower.encoder.layers.12.mlp.up_proj.linear",
+      "model.vision_tower.encoder.layers.12.mlp.down_proj.linear",
+      "model.vision_tower.encoder.layers.13.self_attn.q_proj.linear",
+      "model.vision_tower.encoder.layers.13.self_attn.k_proj.linear",
+      "model.vision_tower.encoder.layers.13.self_attn.v_proj.linear",
+      "model.vision_tower.encoder.layers.13.self_attn.o_proj.linear",
+      "model.vision_tower.encoder.layers.13.mlp.gate_proj.linear",
+      "model.vision_tower.encoder.layers.13.mlp.up_proj.linear",
+      "model.vision_tower.encoder.layers.13.mlp.down_proj.linear",
+      "model.vision_tower.encoder.layers.14.self_attn.q_proj.linear",
+      "model.vision_tower.encoder.layers.14.self_attn.k_proj.linear",
+      "model.vision_tower.encoder.layers.14.self_attn.v_proj.linear",
+      "model.vision_tower.encoder.layers.14.self_attn.o_proj.linear",
+      "model.vision_tower.encoder.layers.14.mlp.gate_proj.linear",
+      "model.vision_tower.encoder.layers.14.mlp.up_proj.linear",
+      "model.vision_tower.encoder.layers.14.mlp.down_proj.linear",
+      "model.vision_tower.encoder.layers.15.self_attn.q_proj.linear",
+      "model.vision_tower.encoder.layers.15.self_attn.k_proj.linear",
+      "model.vision_tower.encoder.layers.15.self_attn.v_proj.linear",
+      "model.vision_tower.encoder.layers.15.self_attn.o_proj.linear",
+      "model.vision_tower.encoder.layers.15.mlp.gate_proj.linear",
+      "model.vision_tower.encoder.layers.15.mlp.up_proj.linear",
+      "model.vision_tower.encoder.layers.15.mlp.down_proj.linear",
+      "model.vision_tower.encoder.layers.16.self_attn.q_proj.linear",
+      "model.vision_tower.encoder.layers.16.self_attn.k_proj.linear",
+      "model.vision_tower.encoder.layers.16.self_attn.v_proj.linear",
+      "model.vision_tower.encoder.layers.16.self_attn.o_proj.linear",
+      "model.vision_tower.encoder.layers.16.mlp.gate_proj.linear",
+      "model.vision_tower.encoder.layers.16.mlp.up_proj.linear",
+      "model.vision_tower.encoder.layers.16.mlp.down_proj.linear",
+      "model.vision_tower.encoder.layers.17.self_attn.q_proj.linear",
+      "model.vision_tower.encoder.layers.17.self_attn.k_proj.linear",
+      "model.vision_tower.encoder.layers.17.self_attn.v_proj.linear",
+      "model.vision_tower.encoder.layers.17.self_attn.o_proj.linear",
+      "model.vision_tower.encoder.layers.17.mlp.gate_proj.linear",
+      "model.vision_tower.encoder.layers.17.mlp.up_proj.linear",
+      "model.vision_tower.encoder.layers.17.mlp.down_proj.linear",
+      "model.vision_tower.encoder.layers.18.self_attn.q_proj.linear",
+      "model.vision_tower.encoder.layers.18.self_attn.k_proj.linear",
+      "model.vision_tower.encoder.layers.18.self_attn.v_proj.linear",
+      "model.vision_tower.encoder.layers.18.self_attn.o_proj.linear",
+      "model.vision_tower.encoder.layers.18.mlp.gate_proj.linear",
+      "model.vision_tower.encoder.layers.18.mlp.up_proj.linear",
+      "model.vision_tower.encoder.layers.18.mlp.down_proj.linear",
+      "model.vision_tower.encoder.layers.19.self_attn.q_proj.linear",
+      "model.vision_tower.encoder.layers.19.self_attn.k_proj.linear",
+      "model.vision_tower.encoder.layers.19.self_attn.v_proj.linear",
+      "model.vision_tower.encoder.layers.19.self_attn.o_proj.linear",
+      "model.vision_tower.encoder.layers.19.mlp.gate_proj.linear",
+      "model.vision_tower.encoder.layers.19.mlp.up_proj.linear",
+      "model.vision_tower.encoder.layers.19.mlp.down_proj.linear",
+      "model.vision_tower.encoder.layers.20.self_attn.q_proj.linear",
+      "model.vision_tower.encoder.layers.20.self_attn.k_proj.linear",
+      "model.vision_tower.encoder.layers.20.self_attn.v_proj.linear",
+      "model.vision_tower.encoder.layers.20.self_attn.o_proj.linear",
+      "model.vision_tower.encoder.layers.20.mlp.gate_proj.linear",
+      "model.vision_tower.encoder.layers.20.mlp.up_proj.linear",
+      "model.vision_tower.encoder.layers.20.mlp.down_proj.linear",
+      "model.vision_tower.encoder.layers.21.self_attn.q_proj.linear",
+      "model.vision_tower.encoder.layers.21.self_attn.k_proj.linear",
+      "model.vision_tower.encoder.layers.21.self_attn.v_proj.linear",
+      "model.vision_tower.encoder.layers.21.self_attn.o_proj.linear",
+      "model.vision_tower.encoder.layers.21.mlp.gate_proj.linear",
+      "model.vision_tower.encoder.layers.21.mlp.up_proj.linear",
+      "model.vision_tower.encoder.layers.21.mlp.down_proj.linear",
+      "model.vision_tower.encoder.layers.22.self_attn.q_proj.linear",
+      "model.vision_tower.encoder.layers.22.self_attn.k_proj.linear",
+      "model.vision_tower.encoder.layers.22.self_attn.v_proj.linear",
+      "model.vision_tower.encoder.layers.22.self_attn.o_proj.linear",
+      "model.vision_tower.encoder.layers.22.mlp.gate_proj.linear",
+      "model.vision_tower.encoder.layers.22.mlp.up_proj.linear",
+      "model.vision_tower.encoder.layers.22.mlp.down_proj.linear",
+      "model.vision_tower.encoder.layers.23.self_attn.q_proj.linear",
+      "model.vision_tower.encoder.layers.23.self_attn.k_proj.linear",
+      "model.vision_tower.encoder.layers.23.self_attn.v_proj.linear",
+      "model.vision_tower.encoder.layers.23.self_attn.o_proj.linear",
+      "model.vision_tower.encoder.layers.23.mlp.gate_proj.linear",
+      "model.vision_tower.encoder.layers.23.mlp.up_proj.linear",
+      "model.vision_tower.encoder.layers.23.mlp.down_proj.linear",
+      "model.vision_tower.encoder.layers.24.self_attn.q_proj.linear",
+      "model.vision_tower.encoder.layers.24.self_attn.k_proj.linear",
+      "model.vision_tower.encoder.layers.24.self_attn.v_proj.linear",
+      "model.vision_tower.encoder.layers.24.self_attn.o_proj.linear",
+      "model.vision_tower.encoder.layers.24.mlp.gate_proj.linear",
+      "model.vision_tower.encoder.layers.24.mlp.up_proj.linear",
+      "model.vision_tower.encoder.layers.24.mlp.down_proj.linear",
+      "model.vision_tower.encoder.layers.25.self_attn.q_proj.linear",
+      "model.vision_tower.encoder.layers.25.self_attn.k_proj.linear",
+      "model.vision_tower.encoder.layers.25.self_attn.v_proj.linear",
+      "model.vision_tower.encoder.layers.25.self_attn.o_proj.linear",
+      "model.vision_tower.encoder.layers.25.mlp.gate_proj.linear",
+      "model.vision_tower.encoder.layers.25.mlp.up_proj.linear",
+      "model.vision_tower.encoder.layers.25.mlp.down_proj.linear",
+      "model.vision_tower.encoder.layers.26.self_attn.q_proj.linear",
+      "model.vision_tower.encoder.layers.26.self_attn.k_proj.linear",
+      "model.vision_tower.encoder.layers.26.self_attn.v_proj.linear",
+      "model.vision_tower.encoder.layers.26.self_attn.o_proj.linear",
+      "model.vision_tower.encoder.layers.26.mlp.gate_proj.linear",
+      "model.vision_tower.encoder.layers.26.mlp.up_proj.linear",
+      "model.vision_tower.encoder.layers.26.mlp.down_proj.linear",
+      "model.language_model.layers.0.router",
+      "model.language_model.layers.0.router.proj",
+      "model.language_model.layers.1.router",
+      "model.language_model.layers.1.router.proj",
+      "model.language_model.layers.2.router",
+      "model.language_model.layers.2.router.proj",
+      "model.language_model.layers.3.router",
+      "model.language_model.layers.3.router.proj",
+      "model.language_model.layers.4.router",
+      "model.language_model.layers.4.router.proj",
+      "model.language_model.layers.5.router",
+      "model.language_model.layers.5.router.proj",
+      "model.language_model.layers.6.router",
+      "model.language_model.layers.6.router.proj",
+      "model.language_model.layers.7.router",
+      "model.language_model.layers.7.router.proj",
+      "model.language_model.layers.8.router",
+      "model.language_model.layers.8.router.proj",
+      "model.language_model.layers.9.router",
+      "model.language_model.layers.9.router.proj",
+      "model.language_model.layers.10.router",
+      "model.language_model.layers.10.router.proj",
+      "model.language_model.layers.11.router",
+      "model.language_model.layers.11.router.proj",
+      "model.language_model.layers.12.router",
+      "model.language_model.layers.12.router.proj",
+      "model.language_model.layers.13.router",
+      "model.language_model.layers.13.router.proj",
+      "model.language_model.layers.14.router",
+      "model.language_model.layers.14.router.proj",
+      "model.language_model.layers.15.router",
+      "model.language_model.layers.15.router.proj",
+      "model.language_model.layers.16.router",
+      "model.language_model.layers.16.router.proj",
+      "model.language_model.layers.17.router",
+      "model.language_model.layers.17.router.proj",
+      "model.language_model.layers.18.router",
+      "model.language_model.layers.18.router.proj",
+      "model.language_model.layers.19.router",
+      "model.language_model.layers.19.router.proj",
+      "model.language_model.layers.20.router",
+      "model.language_model.layers.20.router.proj",
+      "model.language_model.layers.21.router",
+      "model.language_model.layers.21.router.proj",
+      "model.language_model.layers.22.router",
+      "model.language_model.layers.22.router.proj",
+      "model.language_model.layers.23.router",
+      "model.language_model.layers.23.router.proj",
+      "model.language_model.layers.24.router",
+      "model.language_model.layers.24.router.proj",
+      "model.language_model.layers.25.router",
+      "model.language_model.layers.25.router.proj",
+      "model.language_model.layers.26.router",
+      "model.language_model.layers.26.router.proj",
+      "model.language_model.layers.27.router",
+      "model.language_model.layers.27.router.proj",
+      "model.language_model.layers.28.router",
+      "model.language_model.layers.28.router.proj",
+      "model.language_model.layers.29.router",
+      "model.language_model.layers.29.router.proj",
+      "model.embed_vision.embedding_projection",
+      "lm_head"
+    ],
+    "kv_cache_scheme": null,
+    "quant_method": "compressed-tensors",
+    "quantization_status": "compressed",
+    "sparsity_config": {},
+    "transform_config": {},
+    "version": "0.18.0"
+  },
+  "software": {
+    "conversion_artifact_compressed_tensors_version": "0.18.0",
+    "validated_vllm_commit": "7ca49fbe4bab019e55d57cdc4b7fd3d55c67c1a6",
+    "validated_transformers": "5.15.1",
+    "validated_pytorch": "2.13.0+cu130",
+    "validated_cuda_runtime": "13.0",
+    "validated_compressed_tensors_runtime": "0.17.0"
+  }
+}

recipe.yaml

@@ -0,0 +1,32 @@
+default_stage:
+  default_modifiers:
+    GPTQModifier:
+      config_groups:
+        w4a16_g64:
+          targets: [Linear]
+          weights:
+            num_bits: 4
+            type: int
+            symmetric: true
+            group_size: 64
+            strategy: group
+            block_structure: null
+            dynamic: false
+            actorder: !!python/object/apply:compressed_tensors.quantization.quant_args.ActivationOrdering [
+              static]
+            scale_dtype: null
+            zp_dtype: null
+            observer: memoryless_minmax
+            observer_kwargs: {}
+          input_activations: null
+          output_activations: null
+          format: null
+      targets: [Linear]
+      ignore: ['re:.*lm_head$', 're:.*vision_tower.*', 're:.*router.*', 're:.*embed_tokens.*',
+        're:.*embed_vision.*']
+      bypass_divisibility_checks: false
+      requires_calibration_data: true
+      block_size: 128
+      dampening_frac: 0.01
+      actorder: static
+      offload_hessians: false

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c2cf15ae878b9477ea6d714d02be642da51424c6b7fe0c1e2cf5a332b5c2f789
+size 32169879

tokenizer_config.json

@@ -0,0 +1,142 @@
+{
+  "audio_token": "<|audio|>",
+  "backend": "tokenizers",
+  "boa_token": "<|audio>",
+  "boi_token": "<|image>",
+  "bos_token": "<bos>",
+  "eoa_token": "<audio|>",
+  "eoc_token": "<channel|>",
+  "eoi_token": "<image|>",
+  "eos_token": "<eos>",
+  "eot_token": "<turn|>",
+  "escape_token": "<|\"|>",
+  "etc_token": "<tool_call|>",
+  "etd_token": "<tool|>",
+  "etr_token": "<tool_response|>",
+  "extra_special_tokens": [
+    "<|video|>"
+  ],
+  "image_token": "<|image|>",
+  "is_local": true,
+  "local_files_only": true,
+  "mask_token": "<mask>",
+  "model_max_length": 1000000000000000019884624838656,
+  "model_specific_special_tokens": {
+    "audio_token": "<|audio|>",
+    "boa_token": "<|audio>",
+    "boi_token": "<|image>",
+    "eoa_token": "<audio|>",
+    "eoc_token": "<channel|>",
+    "eoi_token": "<image|>",
+    "eot_token": "<turn|>",
+    "escape_token": "<|\"|>",
+    "etc_token": "<tool_call|>",
+    "etd_token": "<tool|>",
+    "etr_token": "<tool_response|>",
+    "image_token": "<|image|>",
+    "soc_token": "<|channel>",
+    "sot_token": "<|turn>",
+    "stc_token": "<|tool_call>",
+    "std_token": "<|tool>",
+    "str_token": "<|tool_response>",
+    "think_token": "<|think|>"
+  },
+  "pad_token": "<pad>",
+  "padding_side": "left",
+  "processor_class": "Gemma4Processor",
+  "response_schema": {
+    "properties": {
+      "content": {
+        "type": "string"
+      },
+      "role": {
+        "const": "assistant"
+      },
+      "thinking": {
+        "type": "string"
+      },
+      "tool_calls": {
+        "items": {
+          "properties": {
+            "function": {
+              "properties": {
+                "arguments": {
+                  "additionalProperties": {},
+                  "type": "object",
+                  "x-parser": "gemma4-tool-call"
+                },
+                "name": {
+                  "type": "string"
+                }
+              },
+              "type": "object",
+              "x-regex": "call\\:(?P<name>\\w+)(?P<arguments>\\{.*\\})"
+            },
+            "type": {
+              "const": "function"
+            }
+          },
+          "type": "object"
+        },
+        "type": "array",
+        "x-regex-iterator": "<\\|tool_call>(.*?)<tool_call\\|>"
+      }
+    },
+    "type": "object",
+    "x-regex": "(\\<\\|channel\\>thought\\n(?P<thinking>.*?)\\<channel\\|\\>)?(?P<tool_calls>\\<\\|tool_call\\>.*\\<tool_call\\|\\>)?(?P<content>(?:(?!\\<turn\\|\\>)(?!\\<\\|tool_response\\>).)+)?(?:\\<turn\\|\\>|\\<\\|tool_response\\>)?"
+  },
+  "response_template": {
+    "defaults": {
+      "role": "assistant"
+    },
+    "fields": {
+      "content": {
+        "close": [
+          "<turn|>",
+          "<|tool_response>",
+          "<eos>"
+        ],
+        "content": "text"
+      },
+      "thinking": {
+        "close": "<channel|>",
+        "content": "text",
+        "open": "<|channel>thought\n"
+      },
+      "tool_calls": {
+        "close": "<tool_call|>",
+        "content": "json",
+        "content_args": {
+          "string_delims": [
+            [
+              "<|\"|>",
+              "<|\"|>"
+            ]
+          ],
+          "unquoted_keys": true
+        },
+        "open_pattern": "<\\|tool_call>call:(?P<name>\\w+)",
+        "repeats": true,
+        "transform": {
+          "function": {
+            "arguments": "{content}",
+            "name": "{name}"
+          },
+          "type": "function"
+        }
+      }
+    },
+    "start_anchor": [
+      "<|turn>model\n",
+      "<tool_response|>"
+    ]
+  },
+  "soc_token": "<|channel>",
+  "sot_token": "<|turn>",
+  "stc_token": "<|tool_call>",
+  "std_token": "<|tool>",
+  "str_token": "<|tool_response>",
+  "think_token": "<|think|>",
+  "tokenizer_class": "GemmaTokenizer",
+  "unk_token": "<unk>"
+}