Initial commit

Co-authored-by: Cuiunbo <Cuiunbo@users.noreply.huggingface.co>
Co-authored-by: cursor <cursor@users.noreply.huggingface.co>

.gitattributes

@@ -0,0 +1,43 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text
+example1.gif filter=lfs diff=lfs merge=lfs -text
+example2.gif filter=lfs diff=lfs merge=lfs -text
+example3.gif filter=lfs diff=lfs merge=lfs -text
+example4.gif filter=lfs diff=lfs merge=lfs -text
+example5.gif filter=lfs diff=lfs merge=lfs -text
+manip_benchmark.png filter=lfs diff=lfs merge=lfs -text
+manip_case_en.gif filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,239 @@
+---
+license: apache-2.0
+library_name: transformers
+pipeline_tag: robotics
+tags:
+  - vision-language-action
+  - robotics
+  - embodied-ai
+  - minicpm
+  - manipulation
+---
+
+<h1 align="center">
+  MiniCPM-RobotManip
+</h1>
+
+<p align="center">
+  <strong>A Smarter and Faster On-Device AI Brain for Robots</strong>
+</p>
+
+<p align="center">
+    <span style="display: inline-flex; align-items: center; margin-right: 2px;">
+    <img src="github_logo.png" alt="github" width="15" height="15" style="margin-right: 4px;">
+    <a href="https://github.com/OpenBMB/MiniCPM-Robot" target="_blank"> Github</a> &nbsp;|
+    </span>
+    <span style="display: inline-flex; align-items: center; margin-right: 2px;">
+    <img src="feishu_logo.png" alt="feishu" width="15" height="15" style="margin-right: 4px;">
+    <a href="feishu_qrcode.png" target="_blank"> Feishu</a> &nbsp;|
+    </span>
+    <span style="display: inline-flex; align-items: center; margin-right: 2px;">
+    <img src="X_logo.jpg" alt="X" width="15" height="15" style="margin-right: 4px;">
+    <a href="x.png" target="_blank"> X</a> &nbsp;|
+    </span>
+    <span style="display: inline-flex; align-items: center; margin-right: 2px;">
+    <img src="discord_logo.png" alt="Discord" width="15" height="15" style="margin-right: 4px;">
+    <a href="discord.jpeg" target="_blank"> Discord</a> &nbsp;|
+    </span>
+</p>
+
+<strong>MiniCPM-RobotManip</strong> is a 1.5B vision-language-action model for embodied manipulation with the following highlights:
+<ul>
+  <li><b>Generalist Manipulation:</b> A unified 1.5B generalist policy that <b>uses one set of weights across all downstream tasks</b> and <b>outperforms larger models such as π₀.₅ and Qwen-VLA across representative evaluations</b>.</li>
+  <li><b>Streaming Context:</b> Historical observations are continuously incorporated into the model context through streaming inference, <b>reducing per-step compute from 125 TFLOPs to 3.3 TFLOPs while retaining 60 frames of history</b> and supporting <b>up to one minute of visual memory</b>. This moves VLA beyond reactive action generation from single-frame observations toward continuous decision-making grounded in long-horizon visual context.</li>
+  <li><b>Efficient Inference:</b> Inherits MiniCPM-V 4.6's visual token compression, reducing each frame from 256 to 64 visual tokens for 4× compression. With H100, BF16, and single-frame input, model-forward latency per decision step is 120 ms, compared with 234 ms for π0.5. The measurement excludes task autoregressive decoding.</li>
+</ul>
+
+<p align="center">
+  <img src="manip_case_en.gif" width="800" alt="MiniCPM-RobotManip task demonstrations" />
+</p>
+
+## Benchmark Results
+<p align="center">
+  <img src="manip_benchmark.png" width="700" alt="manip_benchmark" />
+</p>
+
+## Inference Example
+
+Please Ensure transformers==5.7.0
+
+<pre><code class="language-python">from __future__ import annotations
+
+import argparse
+import json
+from pathlib import Path
+from typing import Sequence
+
+import numpy as np
+import torch
+from PIL import Image
+from transformers import AutoModel, AutoProcessor
+
+
+STATE_DIM = 80
+IMAGE_SIZE = (448, 448)
+
+
+class MiniCPMVLAInference:
+    """Processor and model wrapper for single-sample VLA inference."""
+
+    def __init__(
+        self,
+        checkpoint_path: str | Path = "openbmb/MiniCPM-RobotManip",
+        device: str | torch.device | None = None,
+    ):
+        if device is None:
+            device = "cuda" if torch.cuda.is_available() else "cpu"
+        self.device = torch.device(device)
+        checkpoint = str(checkpoint_path)
+        self.processor = AutoProcessor.from_pretrained(checkpoint, trust_remote_code=True)
+        self.model = AutoModel.from_pretrained(checkpoint, trust_remote_code=True)
+        self.model.to(self.device).eval()
+
+    @staticmethod
+    def _load_images(images: Sequence[str | Path | Image.Image | np.ndarray]) -> list[np.ndarray]:
+        if not images:
+            raise ValueError("At least one image is required")
+        loaded = []
+        for image in images:
+            if isinstance(image, (str, Path)):
+                with Image.open(image) as pil_image:
+                    array = np.asarray(pil_image.convert("RGB"))
+            elif isinstance(image, Image.Image):
+                array = np.asarray(image.convert("RGB"))
+            elif isinstance(image, np.ndarray):
+                array = image
+            else:
+                raise TypeError(f"Unsupported image type: {type(image)!r}")
+            if array.ndim != 3 or array.shape[-1] != 3:
+                raise ValueError(f"Expected an HxWx3 image, got shape {array.shape}")
+            # Match the training pipeline's ResizeImage(size=(448, 448)),
+            # including PIL's default resize interpolation.
+            resized = Image.fromarray(array).resize(IMAGE_SIZE)
+            loaded.append(np.asarray(resized).copy())
+        return loaded
+
+    def preprocess(self, images: Sequence, text: str) -> dict[str, torch.Tensor]:
+        """Apply the same MiniCPM-V chat template and processor as training."""
+        content = [
+            {"type": "image", "image": image}
+            for image in self._load_images(images)
+        ]
+        content.append({"type": "text", "text": text})
+        messages = [{"role": "user", "content": content}]
+        inputs = self.processor.apply_chat_template(
+            messages,
+            tokenize=True,
+            add_generation_prompt=True,
+            return_dict=True,
+            return_tensors="pt",
+            processor_kwargs={"padding": False},
+        )
+        return {
+            key: value.to(self.device)
+            for key, value in inputs.items()
+            if isinstance(value, torch.Tensor)
+        }
+
+    def _prepare_state(self, state: torch.Tensor | np.ndarray | Sequence[float]) -> torch.Tensor:
+        state = torch.as_tensor(state, dtype=torch.float32, device=self.device)
+        if state.ndim == 1:
+            state = state.unsqueeze(0).unsqueeze(0)
+        elif state.ndim == 2:
+            state = state.unsqueeze(1)
+        if state.shape != (1, 1, STATE_DIM):
+            raise ValueError(f"state must have shape (80,), (1, 80), or (1, 1, 80); got {tuple(state.shape)}")
+        return state
+
+    @torch.inference_mode()
+    def predict(
+        self,
+        images: Sequence[str | Path | Image.Image | np.ndarray],
+        text: str,
+        state: torch.Tensor | np.ndarray | Sequence[float] | None = None,
+        embodiment_id: int = 0,
+        seed: int | None = None,
+    ) -> torch.Tensor:
+        """Return one action chunk with shape ``(30, 80)`` on CPU."""
+        if not 0 <= embodiment_id < self.model.action_head.max_num_embodiments:
+            raise ValueError(
+                f"embodiment_id must be in [0, {self.model.action_head.max_num_embodiments - 1}]"
+            )
+        if state is None:
+            state = torch.zeros(STATE_DIM)
+        state_tensor = self._prepare_state(state)
+        embodiment = torch.tensor([embodiment_id], dtype=torch.long, device=self.device)
+        if seed is not None:
+            torch.manual_seed(seed)
+            if self.device.type == "cuda":
+                torch.cuda.manual_seed_all(seed)
+
+        vlm_inputs = self.preprocess(images, text)
+        actions = self.model.predict_action(
+            state=state_tensor,
+            embodiment_id=embodiment,
+            **vlm_inputs,
+        )
+        return actions[0].float().cpu()
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--image", action="append", required=True, help="Input image; repeat for multiple views")
+    parser.add_argument("--text", required=True, help="Robot instruction/prompt")
+    parser.add_argument("--device", default=None, help="Default: cuda if available, otherwise cpu")
+    state_group = parser.add_mutually_exclusive_group()
+    state_group.add_argument("--state-file", help="A .npy file containing 80 state values")
+    state_group.add_argument("--state", nargs=STATE_DIM, type=float, metavar="VALUE")
+    parser.add_argument("--embodiment-id", type=int, default=0)
+    parser.add_argument("--seed", type=int, default=None)
+    parser.add_argument("--output", help="Optional output .npy path; otherwise print JSON")
+    return parser.parse_args()
+
+
+if __name__ == "__main__":
+    args = parse_args()
+    if args.state_file:
+        state = np.load(args.state_file)
+    elif args.state is not None:
+        state = args.state
+    else:
+        state = np.zeros(STATE_DIM, dtype=np.float32)
+
+    infer_runner = MiniCPMVLAInference(
+        checkpoint_path="openbmb/MiniCPM-RobotManip",
+        device=args.device,
+    )
+    action = infer_runner.predict(
+        images=args.image,
+        text=args.text,
+        state=state,
+        embodiment_id=args.embodiment_id,
+        seed=args.seed,
+    )
+    if args.output:
+        output_path = Path(args.output)
+        output_path.parent.mkdir(parents=True, exist_ok=True)
+        np.save(output_path, action.numpy())
+        print(f"Saved action {tuple(action.shape)} to {output_path}")
+    else:
+        print(json.dumps(action.tolist()))
+
+</code></pre>
+
+
+
+## Acknowledgement
+
+<p>
+  This project builds on and references
+  <a href="https://github.com/starVLA/starVLA">starVLA</a>
+  and
+  <a href="https://github.com/huggingface/lerobot">LeRobot</a>.
+  We thank the authors for their open-source contributions.
+</p>
+
+## License
+
+Model weights and code are open-sourced under the [Apache-2.0](./LICENSE) license.
+

action_head.py

@@ -0,0 +1,387 @@
+from __future__ import annotations
+
+from typing import Optional
+
+import torch
+import torch.nn.functional as F
+from diffusers import ConfigMixin, ModelMixin
+from diffusers.configuration_utils import register_to_config
+from diffusers.models.attention import Attention, FeedForward
+from diffusers.models.embeddings import SinusoidalPositionalEmbedding, TimestepEmbedding, Timesteps
+from torch import nn
+
+
+class TimestepEncoder(nn.Module):
+    def __init__(self, embedding_dim: int):
+        super().__init__()
+        self.time_proj = Timesteps(num_channels=256, flip_sin_to_cos=True, downscale_freq_shift=1)
+        self.timestep_embedder = TimestepEmbedding(in_channels=256, time_embed_dim=embedding_dim)
+
+    def forward(self, timesteps: torch.Tensor) -> torch.Tensor:
+        return self.timestep_embedder(self.time_proj(timesteps).to(next(self.parameters()).dtype))
+
+
+class AdaLayerNorm(nn.Module):
+    def __init__(
+        self,
+        embedding_dim: int,
+        norm_elementwise_affine: bool = False,
+        norm_eps: float = 1e-5,
+    ):
+        super().__init__()
+        self.silu = nn.SiLU()
+        self.linear = nn.Linear(embedding_dim, embedding_dim * 2)
+        self.norm = nn.LayerNorm(embedding_dim, norm_eps, norm_elementwise_affine)
+
+    def forward(self, x: torch.Tensor, temb: torch.Tensor) -> torch.Tensor:
+        scale, shift = self.linear(self.silu(temb)).chunk(2, dim=1)
+        return self.norm(x) * (1 + scale[:, None]) + shift[:, None]
+
+
+class BasicTransformerBlock(nn.Module):
+    def __init__(
+        self,
+        dim: int,
+        num_attention_heads: int,
+        attention_head_dim: int,
+        dropout: float = 0.0,
+        cross_attention_dim: Optional[int] = None,
+        activation_fn: str = "geglu",
+        attention_bias: bool = False,
+        upcast_attention: bool = False,
+        norm_elementwise_affine: bool = True,
+        norm_type: str = "layer_norm",
+        norm_eps: float = 1e-5,
+        final_dropout: bool = False,
+        positional_embeddings: Optional[str] = None,
+        num_positional_embeddings: Optional[int] = None,
+        ff_inner_dim: Optional[int] = None,
+        ff_bias: bool = True,
+        attention_out_bias: bool = True,
+    ):
+        super().__init__()
+        if positional_embeddings and num_positional_embeddings is None:
+            raise ValueError("num_positional_embeddings is required for positional embeddings")
+        self.norm_type = norm_type
+        self.pos_embed = (
+            SinusoidalPositionalEmbedding(dim, max_seq_length=num_positional_embeddings)
+            if positional_embeddings == "sinusoidal"
+            else None
+        )
+        self.norm1 = (
+            AdaLayerNorm(dim)
+            if norm_type == "ada_norm"
+            else nn.LayerNorm(dim, elementwise_affine=norm_elementwise_affine, eps=norm_eps)
+        )
+        self.attn1 = Attention(
+            query_dim=dim,
+            heads=num_attention_heads,
+            dim_head=attention_head_dim,
+            dropout=dropout,
+            bias=attention_bias,
+            cross_attention_dim=cross_attention_dim,
+            upcast_attention=upcast_attention,
+            out_bias=attention_out_bias,
+        )
+        self.norm3 = nn.LayerNorm(dim, norm_eps, norm_elementwise_affine)
+        self.ff = FeedForward(
+            dim,
+            dropout=dropout,
+            activation_fn=activation_fn,
+            final_dropout=final_dropout,
+            inner_dim=ff_inner_dim,
+            bias=ff_bias,
+        )
+        self.final_dropout = nn.Dropout(dropout) if final_dropout else None
+
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        encoder_hidden_states: Optional[torch.Tensor] = None,
+        encoder_attention_mask: Optional[torch.Tensor] = None,
+        temb: Optional[torch.Tensor] = None,
+    ) -> torch.Tensor:
+        norm_hidden_states = (
+            self.norm1(hidden_states, temb) if self.norm_type == "ada_norm" else self.norm1(hidden_states)
+        )
+        if self.pos_embed is not None:
+            norm_hidden_states = self.pos_embed(norm_hidden_states)
+        attention_output = self.attn1(
+            norm_hidden_states,
+            encoder_hidden_states=encoder_hidden_states,
+            attention_mask=encoder_attention_mask,
+        )
+        if self.final_dropout is not None:
+            attention_output = self.final_dropout(attention_output)
+        hidden_states = attention_output + hidden_states
+        if hidden_states.ndim == 4:
+            hidden_states = hidden_states.squeeze(1)
+        hidden_states = self.ff(self.norm3(hidden_states)) + hidden_states
+        if hidden_states.ndim == 4:
+            hidden_states = hidden_states.squeeze(1)
+        return hidden_states
+
+
+class DiT(ModelMixin, ConfigMixin):
+    _supports_gradient_checkpointing = True
+
+    @register_to_config
+    def __init__(
+        self,
+        num_attention_heads: int = 8,
+        attention_head_dim: int = 64,
+        output_dim: int = 26,
+        num_layers: int = 12,
+        dropout: float = 0.1,
+        attention_bias: bool = True,
+        activation_fn: str = "gelu-approximate",
+        num_embeds_ada_norm: Optional[int] = 1000,
+        upcast_attention: bool = False,
+        norm_type: str = "ada_norm",
+        norm_elementwise_affine: bool = False,
+        norm_eps: float = 1e-5,
+        max_num_positional_embeddings: int = 512,
+        compute_dtype: torch.dtype = torch.float32,
+        final_dropout: bool = True,
+        positional_embeddings: Optional[str] = "sinusoidal",
+        interleave_self_attention: bool = False,
+        cross_attention_dim: Optional[int] = None,
+        **kwargs,
+    ):
+        super().__init__()
+        self.attention_head_dim = attention_head_dim
+        self.inner_dim = self.config.num_attention_heads * self.config.attention_head_dim
+        self.gradient_checkpointing = False
+        self.timestep_encoder = TimestepEncoder(self.inner_dim)
+        self.transformer_blocks = nn.ModuleList(
+            [
+                BasicTransformerBlock(
+                    self.inner_dim,
+                    self.config.num_attention_heads,
+                    self.config.attention_head_dim,
+                    dropout=self.config.dropout,
+                    activation_fn=self.config.activation_fn,
+                    attention_bias=self.config.attention_bias,
+                    upcast_attention=self.config.upcast_attention,
+                    norm_type=norm_type,
+                    norm_elementwise_affine=self.config.norm_elementwise_affine,
+                    norm_eps=self.config.norm_eps,
+                    positional_embeddings=positional_embeddings,
+                    num_positional_embeddings=self.config.max_num_positional_embeddings,
+                    final_dropout=final_dropout,
+                    cross_attention_dim=(
+                        None if index % 2 == 1 and interleave_self_attention else cross_attention_dim
+                    ),
+                )
+                for index in range(self.config.num_layers)
+            ]
+        )
+        self.norm_out = nn.LayerNorm(self.inner_dim, elementwise_affine=False, eps=1e-6)
+        self.proj_out_1 = nn.Linear(self.inner_dim, 2 * self.inner_dim)
+        self.proj_out_2 = nn.Linear(self.inner_dim, self.config.output_dim)
+
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        encoder_hidden_states: torch.Tensor,
+        timestep: Optional[torch.LongTensor] = None,
+        return_all_hidden_states: bool = False,
+        encoder_attention_mask: Optional[torch.Tensor] = None,
+    ):
+        time_embedding = self.timestep_encoder(timestep)
+        hidden_states = hidden_states.contiguous()
+        encoder_hidden_states = encoder_hidden_states.contiguous()
+        all_hidden_states = [hidden_states]
+        for index, block in enumerate(self.transformer_blocks):
+            self_attention = index % 2 == 1 and self.config.interleave_self_attention
+            hidden_states = block(
+                hidden_states,
+                encoder_hidden_states=None if self_attention else encoder_hidden_states,
+                encoder_attention_mask=None if self_attention else encoder_attention_mask,
+                temb=time_embedding,
+            )
+            all_hidden_states.append(hidden_states)
+        shift, scale = self.proj_out_1(F.silu(time_embedding)).chunk(2, dim=1)
+        hidden_states = self.norm_out(hidden_states) * (1 + scale[:, None]) + shift[:, None]
+        output = self.proj_out_2(hidden_states)
+        return (output, all_hidden_states) if return_all_hidden_states else output
+
+
+class SinusoidalPositionalEncoding(nn.Module):
+    def __init__(self, embedding_dim: int):
+        super().__init__()
+        self.embedding_dim = embedding_dim
+
+    def forward(self, timesteps: torch.Tensor) -> torch.Tensor:
+        timesteps = timesteps.float()
+        half_dim = self.embedding_dim // 2
+        exponent = -torch.arange(half_dim, dtype=torch.float, device=timesteps.device) * (
+            torch.log(torch.tensor(10000.0, device=timesteps.device)) / half_dim
+        )
+        frequencies = timesteps.unsqueeze(-1) * exponent.exp()
+        return torch.cat([torch.sin(frequencies), torch.cos(frequencies)], dim=-1)
+
+
+class CategorySpecificLinear(nn.Module):
+    def __init__(self, num_categories: int, input_dim: int, output_dim: int):
+        super().__init__()
+        self.num_categories = num_categories
+        self.W = nn.Parameter(0.02 * torch.randn(num_categories, input_dim, output_dim))
+        self.b = nn.Parameter(torch.zeros(num_categories, output_dim))
+
+    def forward(self, x: torch.Tensor, category_ids: torch.Tensor) -> torch.Tensor:
+        if category_ids is None:
+            raise ValueError("embodiment_id (B,) is required")
+        return torch.bmm(x, self.W[category_ids]) + self.b[category_ids].unsqueeze(1)
+
+
+class CategorySpecificMLP(nn.Module):
+    def __init__(self, num_categories: int, input_dim: int, hidden_dim: int, output_dim: int):
+        super().__init__()
+        self.layer1 = CategorySpecificLinear(num_categories, input_dim, hidden_dim)
+        self.layer2 = CategorySpecificLinear(num_categories, hidden_dim, output_dim)
+
+    def forward(self, x: torch.Tensor, category_ids: torch.Tensor) -> torch.Tensor:
+        return self.layer2(F.relu(self.layer1(x, category_ids)), category_ids)
+
+
+class MultiEmbodimentActionEncoder(nn.Module):
+    def __init__(self, action_dim: int, hidden_size: int, num_embodiments: int):
+        super().__init__()
+        self.W1 = CategorySpecificLinear(num_embodiments, action_dim, hidden_size)
+        self.W2 = CategorySpecificLinear(num_embodiments, 2 * hidden_size, hidden_size)
+        self.W3 = CategorySpecificLinear(num_embodiments, hidden_size, hidden_size)
+        self.pos_encoding = SinusoidalPositionalEncoding(hidden_size)
+
+    def forward(
+        self,
+        actions: torch.Tensor,
+        timesteps: torch.Tensor,
+        embodiment_id: torch.Tensor,
+    ) -> torch.Tensor:
+        batch_size, horizon, _ = actions.shape
+        if timesteps.dim() != 1 or timesteps.shape[0] != batch_size:
+            raise ValueError("timesteps must have shape (B,)")
+        timesteps = timesteps.unsqueeze(1).expand(-1, horizon)
+        action_embedding = self.W1(actions, embodiment_id)
+        time_embedding = self.pos_encoding(timesteps).to(dtype=action_embedding.dtype)
+        hidden = self.W2(torch.cat([action_embedding, time_embedding], dim=-1), embodiment_id)
+        return self.W3(hidden * torch.sigmoid(hidden), embodiment_id)
+
+
+class MiniCPMV_VLA_ActionHead(nn.Module):
+    """80-D, 32-embodiment action head for MiniCPM-VLA."""
+
+    def __init__(
+        self,
+        hidden_size: int = 1024,
+        action_dim: int = 80,
+        state_dim: int = 80,
+        action_horizon: int = 30,
+        num_inference_timesteps: int = 4,
+        num_target_vision_tokens: int = 32,
+        max_seq_len: int = 1024,
+        num_timestep_buckets: int = 1000,
+        max_num_embodiments: int = 32,
+    ):
+        super().__init__()
+        self.hidden_size = hidden_size
+        self.input_embedding_dim = 768
+        self.model = DiT(
+            input_embedding_dim=768,
+            attention_head_dim=64,
+            num_attention_heads=12,
+            cross_attention_dim=1024,
+            dropout=0.2,
+            final_dropout=True,
+            interleave_self_attention=True,
+            norm_type="ada_norm",
+            num_layers=16,
+            output_dim=1024,
+            positional_embeddings=None,
+        )
+        self.action_dim = action_dim
+        self.state_dim = state_dim
+        self.action_horizon = action_horizon
+        self.num_inference_timesteps = num_inference_timesteps
+        self.max_num_embodiments = max_num_embodiments
+        self.multi_embodiment = True
+        self.proprio_inject = "concat"
+        self.state_encoder = None
+        self.action_encoder = MultiEmbodimentActionEncoder(
+            action_dim=action_dim + state_dim,
+            hidden_size=self.input_embedding_dim,
+            num_embodiments=max_num_embodiments,
+        )
+        self.action_decoder = CategorySpecificMLP(
+            num_categories=max_num_embodiments,
+            input_dim=self.model.config.output_dim,
+            hidden_dim=hidden_size,
+            output_dim=action_dim,
+        )
+        self.future_tokens = nn.Embedding(num_target_vision_tokens, self.input_embedding_dim)
+        nn.init.normal_(self.future_tokens.weight, mean=0.0, std=0.02)
+        self.position_embedding = nn.Embedding(max_seq_len, self.input_embedding_dim)
+        nn.init.normal_(self.position_embedding.weight, mean=0.0, std=0.02)
+        self.num_timestep_buckets = num_timestep_buckets
+
+    def _encode_action_tokens(
+        self,
+        noisy_actions: torch.Tensor,
+        state: torch.Tensor,
+        timesteps: torch.Tensor,
+        embodiment_id: torch.Tensor,
+    ) -> torch.Tensor:
+        if state is None:
+            raise ValueError("state is required because PROPRIO_INJECT=concat")
+        state = state.expand(-1, noisy_actions.shape[1], -1)
+        inputs = torch.cat([noisy_actions, state], dim=-1)
+        return self.action_encoder(inputs, timesteps, embodiment_id)
+
+    def _build_sequence(self, action_features: torch.Tensor) -> torch.Tensor:
+        future_tokens = self.future_tokens.weight.unsqueeze(0).expand(action_features.shape[0], -1, -1)
+        return torch.cat((future_tokens, action_features), dim=1)
+
+    def _predict(
+        self,
+        noisy_actions: torch.Tensor,
+        vl_embs: torch.Tensor,
+        state: torch.Tensor,
+        timesteps: torch.Tensor,
+        embodiment_id: torch.Tensor,
+    ) -> torch.Tensor:
+        features = self._encode_action_tokens(noisy_actions, state, timesteps, embodiment_id)
+        position_ids = torch.arange(features.shape[1], dtype=torch.long, device=features.device)
+        features = features + self.position_embedding(position_ids).unsqueeze(0)
+        output = self.model(
+            hidden_states=self._build_sequence(features),
+            encoder_hidden_states=vl_embs,
+            timestep=timesteps,
+        )
+        return self.action_decoder(output, embodiment_id)[:, -self.action_horizon :]
+
+    @torch.no_grad()
+    def predict_action(
+        self,
+        vl_embs: torch.Tensor,
+        state: torch.Tensor,
+        embodiment_id: torch.Tensor,
+    ) -> torch.Tensor:
+        actions = torch.zeros(
+            (vl_embs.shape[0], self.action_horizon, self.action_dim),
+            dtype=vl_embs.dtype,
+            device=vl_embs.device,
+        )
+        noise = torch.randn_like(actions)
+        for step in range(self.num_inference_timesteps, 0, -1):
+            time = step / float(self.num_inference_timesteps)
+            timestep = min(int(time * self.num_timestep_buckets), self.num_timestep_buckets - 1)
+            timesteps = torch.full(
+                (vl_embs.shape[0],),
+                timestep,
+                device=vl_embs.device,
+                dtype=torch.long,
+            )
+            noisy_actions = time * noise + (1 - time) * actions
+            actions = self._predict(noisy_actions, vl_embs, state, timesteps, embodiment_id)
+        return actions

chat_template.jinja

@@ -0,0 +1,145 @@
+{%- if enable_thinking is not defined -%}
+    {%- set enable_thinking = false -%}
+{%- endif -%}
+{%- macro render_content(content, is_system_content=false) -%}
+    {%- if content is string -%}
+        {{- content -}}
+    {%- elif content is iterable and content is not mapping -%}
+        {%- set ns = namespace(parts=[]) -%}
+        {%- for item in content -%}
+            {%- if 'image' in item or 'image_url' in item or item.type == 'image' -%}
+                {%- if is_system_content -%}
+                    {{- raise_exception('System message cannot contain images.') -}}
+                {%- endif -%}
+                {%- set ns.parts = ns.parts + ['<|image_pad|>'] -%}
+            {%- elif 'video' in item or item.type == 'video' -%}
+                {%- if is_system_content -%}
+                    {{- raise_exception('System message cannot contain videos.') -}}
+                {%- endif -%}
+                {%- set ns.parts = ns.parts + ['<|video_pad|>'] -%}
+            {%- elif 'text' in item -%}
+                {%- set ns.parts = ns.parts + [item.text] -%}
+            {%- else -%}
+                {{- raise_exception('Unexpected item type in content.') -}}
+            {%- endif -%}
+        {%- endfor -%}
+        {{- ns.parts | join('\n') -}}
+    {%- elif content is none or content is undefined -%}
+        {{- '' -}}
+    {%- else -%}
+        {{- raise_exception('Unexpected content type.') -}}
+    {%- endif -%}
+{%- endmacro -%}
+{%- if not messages %}
+    {{- raise_exception('No messages provided.') }}
+{%- endif %}
+{%- if tools and tools is iterable and tools is not mapping %}
+    {{- '<|im_start|>system\n' }}
+    {{- "# Tools\n\nYou have access to the following functions:\n\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>" }}
+    {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n<tool_call>\n<function=example_function_name>\n<parameter=example_parameter_1>\nvalue_1\n</parameter>\n<parameter=example_parameter_2>\nThis is the value for the second parameter\nthat can span\nmultiple lines\n</parameter>\n</function>\n</tool_call>\n\n<IMPORTANT>\nReminder:\n- Function calls MUST follow the specified format: an inner <function=...></function> block must be nested within <tool_call></tool_call> XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n</IMPORTANT>' }}
+    {%- if messages[0].role == 'system' %}
+        {%- set content = render_content(messages[0].content, true)|trim %}
+        {%- if content %}
+            {{- '\n\n' + content }}
+        {%- endif %}
+    {%- endif %}
+    {{- '<|im_end|>\n' }}
+{%- else %}
+    {%- if messages[0].role == 'system' %}
+        {%- set content = render_content(messages[0].content, true)|trim %}
+        {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+    {%- set index = (messages|length - 1) - loop.index0 %}
+    {%- if ns.multi_step_tool and message.role == "user" %}
+        {%- set content = render_content(message.content)|trim %}
+        {%- if not(content.startswith('<tool_response>') and content.endswith('</tool_response>')) %}
+            {%- set ns.multi_step_tool = false %}
+            {%- set ns.last_query_index = index %}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if ns.multi_step_tool %}
+    {{- raise_exception('No user query found in messages.') }}
+{%- endif %}
+{%- for message in messages %}
+    {%- set content = render_content(message.content)|trim %}
+    {%- if message.role == "system" %}
+        {%- if not loop.first %}
+            {{- raise_exception('System message must be at the beginning.') }}
+        {%- endif %}
+    {%- elif message.role == "user" %}
+        {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {%- set reasoning_content = '' %}
+        {%- if message.reasoning_content is string %}
+            {%- set reasoning_content = message.reasoning_content %}
+        {%- else %}
+            {%- if '</think>' in content %}
+                {%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
+                {%- set content = content.split('</think>')[-1].lstrip('\n') %}
+            {%- endif %}
+        {%- endif %}
+        {%- set reasoning_content = reasoning_content|trim %}
+        {%- if loop.index0 > ns.last_query_index %}
+            {{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content + '\n</think>\n\n' + content }}
+        {%- else %}
+            {{- '<|im_start|>' + message.role + '\n' + content }}
+        {%- endif %}
+        {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %}
+            {%- for tool_call in message.tool_calls %}
+                {%- if tool_call.function is defined %}
+                    {%- set tool_call = tool_call.function %}
+                {%- endif %}
+                {%- if loop.first %}
+                    {%- if content|trim %}
+                        {{- '\n\n<tool_call>\n<function=' + tool_call.name + '>\n' }}
+                    {%- else %}
+                        {{- '<tool_call>\n<function=' + tool_call.name + '>\n' }}
+                    {%- endif %}
+                {%- else %}
+                    {{- '\n<tool_call>\n<function=' + tool_call.name + '>\n' }}
+                {%- endif %}
+                {%- if tool_call.arguments is defined %}
+                    {%- for args_name, args_value in tool_call.arguments|items %}
+                        {{- '<parameter=' + args_name + '>\n' }}
+                        {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %}
+                        {{- args_value }}
+                        {{- '\n</parameter>\n' }}
+                    {%- endfor %}
+                {%- endif %}
+                {{- '</function>\n</tool_call>' }}
+            {%- endfor %}
+        {%- endif %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if loop.previtem and loop.previtem.role != "tool" %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- content }}
+        {{- '\n</tool_response>' }}
+        {%- if not loop.last and loop.nextitem.role != "tool" %}
+            {{- '<|im_end|>\n' }}
+        {%- elif loop.last %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- else %}
+        {{- raise_exception('Unexpected message role.') }}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+    {%- if enable_thinking is defined and enable_thinking is false %}
+        {{- '<think>\n\n</think>\n\n' }}
+    {%- else %}
+        {{- '<think>\n' }}
+    {%- endif %}
+{%- endif %}

config.json

@@ -0,0 +1,170 @@
+{
+  "action_dim": 80,
+  "action_head_dtype": "float32",
+  "action_horizon": 30,
+  "architectures": [
+    "MiniCPMV_VLA"
+  ],
+  "dtype": "bfloat16",
+  "max_num_embodiments": 32,
+  "model_type": "minicpm_vla",
+  "num_inference_timesteps": 4,
+  "state_dim": 80,
+  "transformers_version": "5.7.0",
+  "vlm_config": {
+    "architectures": [
+      "MiniCPMV4_6ForConditionalGeneration"
+    ],
+    "bos_token_id": null,
+    "chunk_size_feed_forward": 0,
+    "downsample_mode": "16x",
+    "drop_vision_last_layer": false,
+    "dtype": null,
+    "eos_token_id": 248044,
+    "id2label": {
+      "0": "LABEL_0",
+      "1": "LABEL_1"
+    },
+    "image_size": 1120,
+    "image_token_id": 248056,
+    "insert_layer_id": 6,
+    "is_encoder_decoder": false,
+    "label2id": {
+      "LABEL_0": 0,
+      "LABEL_1": 1
+    },
+    "merge_kernel_size": [
+      2,
+      2
+    ],
+    "merger_times": 1,
+    "model_type": "minicpmv4_6",
+    "output_attentions": false,
+    "output_hidden_states": false,
+    "pad_token_id": null,
+    "patch_size": 14,
+    "problem_type": null,
+    "return_dict": true,
+    "text_config": {
+      "architectures": null,
+      "attention_bias": false,
+      "attention_dropout": 0.0,
+      "attn_output_gate": true,
+      "bos_token_id": null,
+      "chunk_size_feed_forward": 0,
+      "dtype": null,
+      "eos_token_id": null,
+      "full_attention_interval": 4,
+      "head_dim": 256,
+      "hidden_act": "silu",
+      "hidden_size": 1024,
+      "id2label": {
+        "0": "LABEL_0",
+        "1": "LABEL_1"
+      },
+      "initializer_range": 0.02,
+      "intermediate_size": 3584,
+      "is_encoder_decoder": false,
+      "label2id": {
+        "LABEL_0": 0,
+        "LABEL_1": 1
+      },
+      "layer_types": [
+        "linear_attention",
+        "linear_attention",
+        "linear_attention",
+        "full_attention",
+        "linear_attention",
+        "linear_attention",
+        "linear_attention",
+        "full_attention",
+        "linear_attention",
+        "linear_attention",
+        "linear_attention",
+        "full_attention",
+        "linear_attention",
+        "linear_attention",
+        "linear_attention",
+        "full_attention",
+        "linear_attention",
+        "linear_attention",
+        "linear_attention",
+        "full_attention",
+        "linear_attention",
+        "linear_attention",
+        "linear_attention",
+        "full_attention"
+      ],
+      "linear_conv_kernel_dim": 4,
+      "linear_key_head_dim": 128,
+      "linear_num_key_heads": 16,
+      "linear_num_value_heads": 16,
+      "linear_value_head_dim": 128,
+      "mamba_ssm_dtype": "float32",
+      "max_position_embeddings": 262144,
+      "mlp_only_layers": [],
+      "model_type": "qwen3_5_text",
+      "mtp_num_hidden_layers": 1,
+      "mtp_use_dedicated_embeddings": false,
+      "num_attention_heads": 8,
+      "num_hidden_layers": 24,
+      "num_key_value_heads": 2,
+      "output_attentions": false,
+      "output_hidden_states": false,
+      "pad_token_id": null,
+      "partial_rotary_factor": 0.25,
+      "problem_type": null,
+      "return_dict": true,
+      "rms_norm_eps": 1e-06,
+      "rope_parameters": {
+        "partial_rotary_factor": 0.25,
+        "rope_theta": 10000000,
+        "rope_type": "default"
+      },
+      "tie_word_embeddings": true,
+      "use_cache": true,
+      "vocab_size": 248094
+    },
+    "tie_word_embeddings": true,
+    "transformers_version": "5.7.0",
+    "video_token_id": 248057,
+    "vision_config": {
+      "architectures": null,
+      "attention_dropout": 0.0,
+      "chunk_size_feed_forward": 0,
+      "dtype": null,
+      "hidden_act": "gelu_pytorch_tanh",
+      "hidden_size": 1152,
+      "id2label": {
+        "0": "LABEL_0",
+        "1": "LABEL_1"
+      },
+      "image_size": 980,
+      "insert_layer_id": 6,
+      "intermediate_size": 4304,
+      "is_encoder_decoder": false,
+      "label2id": {
+        "LABEL_0": 0,
+        "LABEL_1": 1
+      },
+      "layer_norm_eps": 1e-06,
+      "num_attention_heads": 16,
+      "num_channels": 3,
+      "num_hidden_layers": 27,
+      "output_attentions": false,
+      "output_hidden_states": false,
+      "patch_size": 14,
+      "problem_type": null,
+      "return_dict": true,
+      "window_kernel_size": [
+        2,
+        2
+      ]
+    }
+  },
+  "vlm_dtype": "bfloat16",
+  "auto_map": {
+    "AutoConfig": "configuration_minicpm_vla.MiniCPMVLAConfig",
+    "AutoModel": "modeling_minicpm_vla.MiniCPMV_VLA"
+  }
+}

configuration_minicpm_vla.py

@@ -0,0 +1,35 @@
+from __future__ import annotations
+
+from transformers import PretrainedConfig
+
+
+class MiniCPMVLAConfig(PretrainedConfig):
+    """Configuration for the complete VLM and action-head VLA model."""
+
+    model_type = "minicpm_vla"
+    is_composition = True
+
+    def __init__(
+        self,
+        vlm_config: dict | PretrainedConfig | None = None,
+        action_dim: int = 80,
+        state_dim: int = 80,
+        action_horizon: int = 30,
+        max_num_embodiments: int = 32,
+        num_inference_timesteps: int = 4,
+        vlm_dtype: str = "bfloat16",
+        action_head_dtype: str = "float32",
+        **kwargs,
+    ):
+        super().__init__(**kwargs)
+        if isinstance(vlm_config, PretrainedConfig):
+            vlm_config = vlm_config.to_dict()
+        self.vlm_config = vlm_config
+        self.action_dim = action_dim
+        self.state_dim = state_dim
+        self.action_horizon = action_horizon
+        self.max_num_embodiments = max_num_embodiments
+        self.num_inference_timesteps = num_inference_timesteps
+        self.vlm_dtype = vlm_dtype
+        self.action_head_dtype = action_head_dtype
+        self.architectures = ["MiniCPMV_VLA"]

generation_config.json

@@ -0,0 +1,13 @@
+{
+  "bos_token_id": 248045,
+  "do_sample": true,
+  "eos_token_id": [
+    248044,
+    248046
+  ],
+  "temperature": 0.7,
+  "top_k": 0,
+  "top_p": 1.0,
+  "repetition_penalty": 1.0,
+  "transformers_version": "5.7.0"
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:404314e1f9599d3162fdaa46e4d0c40acf82c27bc7aee2e15e7fe675fb1635b7
+size 3544229208

modeling_minicpm_vla.py

@@ -0,0 +1,92 @@
+from __future__ import annotations
+
+from collections.abc import Mapping
+
+import torch
+from transformers import AutoConfig, AutoModel, AutoModelForImageTextToText, PretrainedConfig, PreTrainedModel
+
+try:
+    from .action_head import MiniCPMV_VLA_ActionHead
+    from .configuration_minicpm_vla import MiniCPMVLAConfig
+except ImportError:  # pragma: no cover - fallback for non-package loaders
+    from action_head import MiniCPMV_VLA_ActionHead
+    from configuration_minicpm_vla import MiniCPMVLAConfig
+
+
+def _torch_dtype(name: str) -> torch.dtype:
+    try:
+        return getattr(torch, name)
+    except AttributeError as exc:
+        raise ValueError(f"Unsupported torch dtype: {name}") from exc
+
+
+def _build_vlm_config(config_dict: dict) -> PretrainedConfig:
+    values = dict(config_dict)
+    model_type = values.pop("model_type")
+    return AutoConfig.for_model(model_type, **values)
+
+
+class MiniCPMV_VLA(PreTrainedModel):
+    """Complete VLA model loadable with ``AutoModel.from_pretrained``."""
+
+    config_class = MiniCPMVLAConfig
+    base_model_prefix = ""
+    _tied_weights_keys = {
+        "vlm.lm_head.weight": "vlm.model.language_model.embed_tokens.weight",
+    }
+    _no_split_modules = ["BasicTransformerBlock"]
+
+    def __init__(self, config: MiniCPMVLAConfig):
+        super().__init__(config)
+        if config.vlm_config is None:
+            raise ValueError("config.vlm_config is required")
+        vlm_config = _build_vlm_config(config.vlm_config)
+        self.vlm_dtype = _torch_dtype(config.vlm_dtype)
+        self.action_head_dtype = _torch_dtype(config.action_head_dtype)
+        self.vlm = AutoModelForImageTextToText.from_config(vlm_config).to(self.vlm_dtype)
+        self.vlm.config.hidden_size = self.vlm.config.text_config.hidden_size
+        self.action_head = MiniCPMV_VLA_ActionHead(
+            action_dim=config.action_dim,
+            state_dim=config.state_dim,
+            action_horizon=config.action_horizon,
+            num_inference_timesteps=config.num_inference_timesteps,
+            max_num_embodiments=config.max_num_embodiments,
+        ).to(self.action_head_dtype)
+        self.post_init()
+
+    def get_input_embeddings(self):
+        return self.vlm.get_input_embeddings()
+
+    def set_input_embeddings(self, value):
+        return self.vlm.set_input_embeddings(value)
+
+    def _vlm_forward(self, data: Mapping[str, object]):
+        inputs = {key: value for key, value in data.items() if value is not None}
+        if "language_attention_mask" in inputs and "attention_mask" not in inputs:
+            inputs["attention_mask"] = inputs.pop("language_attention_mask")
+        inputs.setdefault("output_hidden_states", True)
+        inputs.setdefault("return_dict", True)
+        inputs.setdefault("use_cache", False)
+        inputs.setdefault("logits_to_keep", 1)
+        with torch.autocast("cuda", dtype=self.vlm_dtype, enabled=torch.cuda.is_available()):
+            try:
+                return self.vlm(**inputs)
+            except TypeError as exc:
+                if "logits_to_keep" not in str(exc):
+                    raise
+                inputs.pop("logits_to_keep")
+                return self.vlm(**inputs)
+
+    @torch.no_grad()
+    def predict_action(
+        self,
+        state: torch.Tensor,
+        embodiment_id: torch.Tensor,
+        **vlm_inputs,
+    ) -> torch.Tensor:
+        vl_embs = self._vlm_forward(vlm_inputs).hidden_states[-1].to(self.action_head_dtype)
+        return self.action_head.predict_action(vl_embs, state, embodiment_id)
+
+
+AutoConfig.register("minicpm_vla", MiniCPMVLAConfig)
+AutoModel.register(MiniCPMVLAConfig, MiniCPMV_VLA)

preprocessor_config.json

@@ -0,0 +1,19 @@
+{
+  "image_processor_type": "MiniCPMV4_6ImageProcessor",
+  "processor_class": "MiniCPMV4_6Processor",
+  "max_slice_nums": 9,
+  "scale_resolution": 448,
+  "patch_size": 14,
+  "use_image_id": true,
+  "slice_mode": true,
+  "image_mean": [
+    0.5,
+    0.5,
+    0.5
+  ],
+  "image_std": [
+    0.5,
+    0.5,
+    0.5
+  ]
+}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:33861e37bb955af1e3f3061182b820f347eba2b9c2c1011c82794bf0d6e77b54
+size 19992481

tokenizer_config.json

@@ -0,0 +1,42 @@
+{
+  "add_prefix_space": false,
+  "audio_bos_token": "<|audio_start|>",
+  "audio_eos_token": "<|audio_end|>",
+  "audio_token": "<|audio_pad|>",
+  "backend": "tokenizers",
+  "bos_token": "<|im_start|>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": {
+    "image_token": "<|image_pad|>",
+    "video_token": "<|video_pad|>",
+    "image_start_token": "<image>",
+    "image_end_token": "</image>",
+    "slice_start_token": "<slice>",
+    "slice_end_token": "</slice>",
+    "image_id_start_token": "<image_id>",
+    "image_id_end_token": "</image_id>"
+  },
+  "image_token": "<|image_pad|>",
+  "is_local": true,
+  "model_max_length": 262144,
+  "model_specific_special_tokens": {
+    "audio_bos_token": "<|audio_start|>",
+    "audio_eos_token": "<|audio_end|>",
+    "audio_token": "<|audio_pad|>",
+    "image_token": "<|image_pad|>",
+    "video_token": "<|video_pad|>",
+    "vision_bos_token": "<|vision_start|>",
+    "vision_eos_token": "<|vision_end|>"
+  },
+  "pad_token": "<|endoftext|>",
+  "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+",
+  "split_special_tokens": false,
+  "unk_token": "<unk>",
+  "video_token": "<|video_pad|>",
+  "vision_bos_token": "<|vision_start|>",
+  "vision_eos_token": "<|vision_end|>",
+  "chat_template": "{%- if enable_thinking is not defined -%}\n    {%- set enable_thinking = false -%}\n{%- endif -%}\n{%- macro render_content(content, is_system_content=false) -%}\n    {%- if content is string -%}\n        {{- content -}}\n    {%- elif content is iterable and content is not mapping -%}\n        {%- set ns = namespace(parts=[]) -%}\n        {%- for item in content -%}\n            {%- if 'image' in item or 'image_url' in item or item.type == 'image' -%}\n                {%- if is_system_content -%}\n                    {{- raise_exception('System message cannot contain images.') -}}\n                {%- endif -%}\n                {%- set ns.parts = ns.parts + ['<|image_pad|>'] -%}\n            {%- elif 'video' in item or item.type == 'video' -%}\n                {%- if is_system_content -%}\n                    {{- raise_exception('System message cannot contain videos.') -}}\n                {%- endif -%}\n                {%- set ns.parts = ns.parts + ['<|video_pad|>'] -%}\n            {%- elif 'text' in item -%}\n                {%- set ns.parts = ns.parts + [item.text] -%}\n            {%- else -%}\n                {{- raise_exception('Unexpected item type in content.') -}}\n            {%- endif -%}\n        {%- endfor -%}\n        {{- ns.parts | join('\\n') -}}\n    {%- elif content is none or content is undefined -%}\n        {{- '' -}}\n    {%- else -%}\n        {{- raise_exception('Unexpected content type.') -}}\n    {%- endif -%}\n{%- endmacro -%}\n{%- if not messages %}\n    {{- raise_exception('No messages provided.') }}\n{%- endif %}\n{%- if tools and tools is iterable and tools is not mapping %}\n    {{- '<|im_start|>system\\n' }}\n    {{- \"# Tools\\n\\nYou have access to the following functions:\\n\\n<tools>\" }}\n    {%- for tool in tools %}\n        {{- \"\\n\" }}\n        {{- tool | tojson }}\n    {%- endfor %}\n    {{- \"\\n</tools>\" }}\n    {{- '\\n\\nIf you choose to call a function ONLY reply in the following format with NO suffix:\\n\\n<tool_call>\\n<function=example_function_name>\\n<parameter=example_parameter_1>\\nvalue_1\\n</parameter>\\n<parameter=example_parameter_2>\\nThis is the value for the second parameter\\nthat can span\\nmultiple lines\\n</parameter>\\n</function>\\n</tool_call>\\n\\n<IMPORTANT>\\nReminder:\\n- Function calls MUST follow the specified format: an inner <function=...></function> block must be nested within <tool_call></tool_call> XML tags\\n- Required parameters MUST be specified\\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\\n</IMPORTANT>' }}\n    {%- if messages[0].role == 'system' %}\n        {%- set content = render_content(messages[0].content, true)|trim %}\n        {%- if content %}\n            {{- '\\n\\n' + content }}\n        {%- endif %}\n    {%- endif %}\n    {{- '<|im_end|>\\n' }}\n{%- else %}\n    {%- if messages[0].role == 'system' %}\n        {%- set content = render_content(messages[0].content, true)|trim %}\n        {{- '<|im_start|>system\\n' + content + '<|im_end|>\\n' }}\n    {%- endif %}\n{%- endif %}\n{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}\n{%- for message in messages[::-1] %}\n    {%- set index = (messages|length - 1) - loop.index0 %}\n    {%- if ns.multi_step_tool and message.role == \"user\" %}\n        {%- set content = render_content(message.content)|trim %}\n        {%- if not(content.startswith('<tool_response>') and content.endswith('</tool_response>')) %}\n            {%- set ns.multi_step_tool = false %}\n            {%- set ns.last_query_index = index %}\n        {%- endif %}\n    {%- endif %}\n{%- endfor %}\n{%- if ns.multi_step_tool %}\n    {{- raise_exception('No user query found in messages.') }}\n{%- endif %}\n{%- for message in messages %}\n    {%- set content = render_content(message.content)|trim %}\n    {%- if message.role == \"system\" %}\n        {%- if not loop.first %}\n            {{- raise_exception('System message must be at the beginning.') }}\n        {%- endif %}\n    {%- elif message.role == \"user\" %}\n        {{- '<|im_start|>' + message.role + '\\n' + content + '<|im_end|>' + '\\n' }}\n    {%- elif message.role == \"assistant\" %}\n        {%- set reasoning_content = '' %}\n        {%- if message.reasoning_content is string %}\n            {%- set reasoning_content = message.reasoning_content %}\n        {%- else %}\n            {%- if '</think>' in content %}\n                {%- set reasoning_content = content.split('</think>')[0].rstrip('\\n').split('<think>')[-1].lstrip('\\n') %}\n                {%- set content = content.split('</think>')[-1].lstrip('\\n') %}\n            {%- endif %}\n        {%- endif %}\n        {%- set reasoning_content = reasoning_content|trim %}\n        {%- if loop.index0 > ns.last_query_index %}\n            {{- '<|im_start|>' + message.role + '\\n<think>\\n' + reasoning_content + '\\n</think>\\n\\n' + content }}\n        {%- else %}\n            {{- '<|im_start|>' + message.role + '\\n' + content }}\n        {%- endif %}\n        {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %}\n            {%- for tool_call in message.tool_calls %}\n                {%- if tool_call.function is defined %}\n                    {%- set tool_call = tool_call.function %}\n                {%- endif %}\n                {%- if loop.first %}\n                    {%- if content|trim %}\n                        {{- '\\n\\n<tool_call>\\n<function=' + tool_call.name + '>\\n' }}\n                    {%- else %}\n                        {{- '<tool_call>\\n<function=' + tool_call.name + '>\\n' }}\n                    {%- endif %}\n                {%- else %}\n                    {{- '\\n<tool_call>\\n<function=' + tool_call.name + '>\\n' }}\n                {%- endif %}\n                {%- if tool_call.arguments is defined %}\n                    {%- for args_name, args_value in tool_call.arguments|items %}\n                        {{- '<parameter=' + args_name + '>\\n' }}\n                        {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %}\n                        {{- args_value }}\n                        {{- '\\n</parameter>\\n' }}\n                    {%- endfor %}\n                {%- endif %}\n                {{- '</function>\\n</tool_call>' }}\n            {%- endfor %}\n        {%- endif %}\n        {{- '<|im_end|>\\n' }}\n    {%- elif message.role == \"tool\" %}\n        {%- if loop.previtem and loop.previtem.role != \"tool\" %}\n            {{- '<|im_start|>user' }}\n        {%- endif %}\n        {{- '\\n<tool_response>\\n' }}\n        {{- content }}\n        {{- '\\n</tool_response>' }}\n        {%- if not loop.last and loop.nextitem.role != \"tool\" %}\n            {{- '<|im_end|>\\n' }}\n        {%- elif loop.last %}\n            {{- '<|im_end|>\\n' }}\n        {%- endif %}\n    {%- else %}\n        {{- raise_exception('Unexpected message role.') }}\n    {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {{- '<|im_start|>assistant\\n' }}\n    {%- if enable_thinking is defined and enable_thinking is false %}\n        {{- '<think>\\n\\n</think>\\n\\n' }}\n    {%- else %}\n        {{- '<think>\\n' }}\n    {%- endif %}\n{%- endif %}\n",
+  "tokenizer_class": "Qwen2Tokenizer"
+}