Stage 1 checkpoint (projector warmup on LLaVA-CC3M-Pretrain-595K)

README.md

@@ -0,0 +1,98 @@
+---
+language:
+- en
+- fr
+- de
+- es
+- it
+- pl
+license: apache-2.0
+tags:
+- vision-language-model
+- multimodal
+- visual-question-answering
+- image-captioning
+- vlm
+
+base_model:
+- OpenGVLab/InternViT-300M-448px-V2_5
+- PleIAs/Baguettotron
+---
+
+# Baguettotron-VLM — Stage 1 (Projector Warmup)
+
+**Stage 1 checkpoint.** Only the MLP projector has been trained (on LLaVA-CC3M-Pretrain-595K); the ViT and the Baguettotron LLM are the unmodified base weights. Published for reproducibility — for actual use prefer the Stage 2 or Stage 3 checkpoints.
+
+Part of the [Baguettotron-VLM](https://github.com/andreagemelli/baguettotron-vlm)
+project: an open, reproducible, multilingual Vision-Language Model built by
+extending [PleIAs/Baguettotron](https://huggingface.co/PleIAs/Baguettotron)
+(321M reasoning SLM) with visual capabilities via
+[InternViT-300M-448px-V2.5](https://huggingface.co/OpenGVLab/InternViT-300M-448px-V2_5).
+
+Related checkpoints:
+- Stage 1 (projector warmup): [andreagemelli/Baguettotron-VLM-Stage1](https://huggingface.co/andreagemelli/Baguettotron-VLM-Stage1)
+- Stage 2 (instruction tuning): [andreagemelli/Baguettotron-VLM-Stage2](https://huggingface.co/andreagemelli/Baguettotron-VLM-Stage2)
+- Stage 3 (reasoning SFT, flagship): [andreagemelli/Baguettotron-VLM](https://huggingface.co/andreagemelli/Baguettotron-VLM)
+
+## Architecture
+
+```
+Image (448×448)
+  → InternViT-300M-448px-V2.5 (304M, frozen) → 1024 tokens × 1024d
+  → Pixel unshuffle (factor=2)                → 256 tokens × 4096d
+  → MLP projector (2-layer, ~2.7M)            → 256 tokens × 576d
+  → Interleave with text tokens
+  → Baguettotron (321M, Llama arch, 80L, h=576)
+  → Text output with <think> reasoning traces
+
+Total: ~628M parameters
+```
+
+## Usage
+
+```python
+import torch
+from transformers import AutoModelForImageTextToText, AutoProcessor
+from PIL import Image
+
+model = AutoModelForImageTextToText.from_pretrained(
+    "andreagemelli/Baguettotron-VLM-Stage1",
+    trust_remote_code=True,
+    torch_dtype=torch.bfloat16,
+    device_map="auto",
+)
+processor = AutoProcessor.from_pretrained(
+    "andreagemelli/Baguettotron-VLM-Stage1",
+    trust_remote_code=True,
+)
+
+image = Image.open("photo.jpg").convert("RGB")
+inputs = processor(
+    messages=[{"role": "user", "content": "<image>\nDescribe the image"}],
+    image=image,
+)
+inputs = {k: v.to(model.device) for k, v in inputs.items() if v is not None}
+
+print(model.chat(**inputs))
+```
+
+### Chat template
+
+Stage 1 was trained on short image captions with no `<think>` traces. The processor emits a bare assistant prefix (`<|im_start|>assistant\n`) and the model completes the caption directly. Keep prompts simple ("Describe the image").
+
+## Training details
+
+| | Stage 1 |
+|---|---|
+| Data | LLaVA-CC3M-Pretrain-595K (595K image-caption pairs) |
+| Trainable params | ~2.7M (projector only) |
+| Frozen | ViT + LLM |
+| Effective batch size | 256 |
+| Learning rate | 1e-3, cosine, 250-step warmup |
+| Precision | bf16 |
+| Hardware | 1× H100 SXM (RunPod) |
+| Duration | ~5h |
+
+## License
+
+Apache 2.0 — see the [GitHub repo](https://github.com/andreagemelli/baguettotron-vlm).

config.json

@@ -0,0 +1,19 @@
+{
+  "model_type": "baguettotron_vlm",
+  "vit_model_id": "OpenGVLab/InternViT-300M-448px-V2_5",
+  "llm_model_id": "PleIAs/Baguettotron",
+  "vit_hidden": 1024,
+  "vit_tokens": 1024,
+  "llm_hidden": 576,
+  "num_visual_tokens": 256,
+  "unshuffle_factor": 2,
+  "image_token": "<image>",
+  "chat_style": "base",
+  "stage": 1,
+  "torch_dtype": "bfloat16",
+  "auto_map": {
+    "AutoConfig": "configuration_baguettotron_vlm.BaguettotronVLMConfig",
+    "AutoModelForImageTextToText": "modeling_baguettotron_vlm.BaguettotronVLMForConditionalGeneration",
+    "AutoProcessor": "processing_baguettotron_vlm.BaguettotronVLMProcessor"
+  }
+}

configuration_baguettotron_vlm.py

@@ -0,0 +1,38 @@
+"""BaguettotronVLM configuration."""
+from __future__ import annotations
+from transformers import PretrainedConfig
+
+
+class BaguettotronVLMConfig(PretrainedConfig):
+    model_type = "baguettotron_vlm"
+
+    def __init__(
+        self,
+        vit_model_id: str = "OpenGVLab/InternViT-300M-448px-V2_5",
+        llm_model_id: str = "PleIAs/Baguettotron",
+        vit_hidden: int = 1024,
+        vit_tokens: int = 1024,
+        llm_hidden: int = 576,
+        num_visual_tokens: int = 256,
+        unshuffle_factor: int = 2,
+        image_token: str = "<image>",
+        chat_style: str = "answer",
+        stage: int = 2,
+        **kwargs,
+    ):
+        super().__init__(**kwargs)
+        self.vit_model_id = vit_model_id
+        self.llm_model_id = llm_model_id
+        self.vit_hidden = vit_hidden
+        self.vit_tokens = vit_tokens
+        self.llm_hidden = llm_hidden
+        self.num_visual_tokens = num_visual_tokens
+        self.unshuffle_factor = unshuffle_factor
+        self.image_token = image_token
+        # chat_style controls the assistant-turn prefix emitted by the
+        # processor when add_generation_prompt=True:
+        #   "base"   → <|im_start|>assistant\n        (stage 1, no think tokens)
+        #   "answer" → <|im_start|>assistant\n</think>\n (stage 2, answer-only)
+        #   "think"  → <|im_start|>assistant\n<think>\n  (stage 3, reasoning)
+        self.chat_style = chat_style
+        self.stage = stage

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:86537f6ef5d0e5e3d7e28cccea94ff515278b9defa5000971bb476f63934ebfa
+size 1336330920

modeling_baguettotron_vlm.py

@@ -0,0 +1,172 @@
+"""BaguettotronVLM model — self-contained for HuggingFace Hub."""
+from __future__ import annotations
+
+import torch
+import torch.nn as nn
+from transformers import (
+    AutoModel,
+    AutoModelForCausalLM,
+    AutoTokenizer,
+    PreTrainedModel,
+)
+from transformers.modeling_outputs import CausalLMOutputWithPast
+
+from .configuration_baguettotron_vlm import BaguettotronVLMConfig
+
+
+class PixelUnshuffleProjector(nn.Module):
+    """Reduces ViT tokens 4× via PixelUnshuffle then projects to LLM dim."""
+
+    def __init__(self, in_dim: int, out_dim: int, factor: int):
+        super().__init__()
+        self.factor = factor
+        self.unshuffle = nn.PixelUnshuffle(factor)
+        self.mlp = nn.Sequential(
+            nn.Linear(in_dim * factor * factor, out_dim),
+            nn.GELU(),
+            nn.Linear(out_dim, out_dim),
+        )
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        B, N, D = x.shape
+        spatial = int(N ** 0.5)
+        x = x.reshape(B, spatial, spatial, D).permute(0, 3, 1, 2)
+        x = self.unshuffle(x)
+        x = x.flatten(2).transpose(1, 2)
+        return self.mlp(x)
+
+
+class BaguettotronVLMForConditionalGeneration(PreTrainedModel):
+    """
+    BaguettotronVLM: InternViT-300M + PixelUnshuffle projector + Baguettotron-321M.
+
+    ~628M total parameters. The same architecture is shipped for all three
+    training stages; only the checkpoint weights and `config.chat_style`
+    differ between them.
+
+    Load with:
+        from transformers import AutoModelForImageTextToText
+        model = AutoModelForImageTextToText.from_pretrained(
+            "andreagemelli/Baguettotron-VLM",
+            trust_remote_code=True,
+            torch_dtype=torch.bfloat16,
+        )
+    """
+
+    config_class = BaguettotronVLMConfig
+    _no_split_modules = ["InternVisionEncoderLayer", "LlamaDecoderLayer"]
+    # Tell HF Trainer not to pass num_items_in_batch (loss handled internally)
+    model_accepts_loss_kwargs: bool = False
+
+    def __init__(self, config: BaguettotronVLMConfig):
+        super().__init__(config)
+
+        self.vit = AutoModel.from_pretrained(
+            config.vit_model_id,
+            dtype=torch.bfloat16,
+            low_cpu_mem_usage=True,
+            trust_remote_code=True,
+        )
+        self.projector = PixelUnshuffleProjector(
+            in_dim=config.vit_hidden,
+            out_dim=config.llm_hidden,
+            factor=config.unshuffle_factor,
+        )
+
+        tokenizer = AutoTokenizer.from_pretrained(config.llm_model_id)
+        tokenizer.add_special_tokens(
+            {"additional_special_tokens": [config.image_token, "</image>"]}
+        )
+        raw_id = tokenizer.convert_tokens_to_ids(config.image_token)
+        self.image_token_id: int = raw_id if isinstance(raw_id, int) else int(raw_id[0])
+
+        self.llm = AutoModelForCausalLM.from_pretrained(
+            config.llm_model_id, dtype=torch.bfloat16
+        )
+        self.llm.resize_token_embeddings(len(tokenizer))
+        # Break weight tying — safetensors rejects shared-storage tensors
+        self.llm.lm_head.weight = nn.Parameter(self.llm.lm_head.weight.data.clone())
+
+        self._tokenizer = tokenizer
+
+    def _init_weights(self, module: nn.Module) -> None:
+        # Pretrained components are initialised from their respective hubs;
+        # the projector weights come from the saved checkpoint — skip random init.
+        pass
+
+    # ------------------------------------------------------------------
+    # Training interface
+    # ------------------------------------------------------------------
+
+    def forward(
+        self,
+        input_ids: torch.Tensor,
+        attention_mask: torch.Tensor,
+        labels: torch.Tensor | None = None,
+        pixel_values: torch.Tensor | None = None,
+        **kwargs,
+    ) -> CausalLMOutputWithPast:
+        inputs_embeds = self.llm.get_input_embeddings()(input_ids)
+
+        if pixel_values is not None:
+            with torch.no_grad():
+                vit_out = self.vit(pixel_values)
+            image_features = vit_out.last_hidden_state
+            if image_features.shape[1] == self.config.vit_tokens + 1:
+                image_features = image_features[:, 1:, :]
+            visual_tokens = self.projector(image_features.float())
+            image_mask = input_ids == self.image_token_id
+            inputs_embeds[image_mask] = visual_tokens.reshape(
+                -1, self.config.llm_hidden
+            ).to(inputs_embeds.dtype)
+
+        return self.llm(
+            inputs_embeds=inputs_embeds,
+            attention_mask=attention_mask,
+            labels=labels,
+            return_dict=True,
+            use_cache=False,
+        )
+
+    # ------------------------------------------------------------------
+    # Inference interface
+    # ------------------------------------------------------------------
+
+    @torch.no_grad()
+    def chat(
+        self,
+        input_ids: torch.Tensor,
+        attention_mask: torch.Tensor,
+        pixel_values: torch.Tensor | None = None,
+        max_new_tokens: int = 256,
+        repetition_penalty: float = 1.3,
+        **generate_kwargs,
+    ) -> str:
+        """Inject visual tokens, generate autoregressively, return decoded string."""
+        inputs_embeds = self.llm.get_input_embeddings()(input_ids)
+
+        if pixel_values is not None:
+            vit_out = self.vit(pixel_values)
+            image_features = vit_out.last_hidden_state
+            if image_features.shape[1] == self.config.vit_tokens + 1:
+                image_features = image_features[:, 1:, :]
+            visual_tokens = self.projector(image_features.float())
+            image_mask = input_ids == self.image_token_id
+            inputs_embeds[image_mask] = visual_tokens.reshape(
+                -1, self.config.llm_hidden
+            ).to(inputs_embeds.dtype)
+
+        im_end_id = int(self._tokenizer.convert_tokens_to_ids("<|im_end|>"))
+        output_ids = self.llm.generate(
+            inputs_embeds=inputs_embeds,
+            attention_mask=attention_mask,
+            max_new_tokens=max_new_tokens,
+            do_sample=False,
+            repetition_penalty=repetition_penalty,
+            eos_token_id=[self._tokenizer.eos_token_id, im_end_id],
+            **generate_kwargs,
+        )
+        decoded = self._tokenizer.decode(output_ids[0], skip_special_tokens=False)
+        if "<|im_end|>" in decoded:
+            decoded = decoded[: decoded.index("<|im_end|>")]
+        return decoded.strip()

preprocessor_config.json

@@ -0,0 +1,27 @@
+{
+  "crop_size": {
+    "height": 448,
+    "width": 448
+  },
+  "do_center_crop": true,
+  "do_convert_rgb": true,
+  "do_normalize": true,
+  "do_rescale": true,
+  "do_resize": true,
+  "image_mean": [
+    0.485,
+    0.456,
+    0.406
+  ],
+  "image_processor_type": "CLIPImageProcessor",
+  "image_std": [
+    0.229,
+    0.224,
+    0.225
+  ],
+  "resample": 3,
+  "rescale_factor": 0.00392156862745098,
+  "size": {
+    "shortest_edge": 448
+  }
+}

processing_baguettotron_vlm.py

@@ -0,0 +1,165 @@
+"""BaguettotronVLM processor — self-contained for HuggingFace Hub."""
+from __future__ import annotations
+
+import json
+import os
+from pathlib import Path
+
+from PIL import Image
+from transformers import CLIPImageProcessor, ProcessorMixin
+from transformers import PreTrainedTokenizerFast
+
+NUM_VISUAL_TOKENS = 256
+IMAGE_TOKEN = "<image>"
+IMAGE_END_TOKEN = "</image>"
+
+
+def _assistant_prefix(chat_style: str, enable_thinking: bool | None) -> str:
+    """Return the assistant-turn content prefix for a generation prompt.
+
+    chat_style is the default baked into the repo at publish time:
+      - "base":   stage 1 — no think tokens
+      - "answer": stage 2 — pre-fill </think> so the model skips reasoning
+      - "think":  stage 3 — pre-fill <think> to trigger reasoning traces
+
+    enable_thinking overrides chat_style at call-time (stage 3 models can
+    toggle thinking on/off dynamically):
+      - None  → keep chat_style default
+      - True  → "<think>\n"
+      - False → "</think>\n"
+    """
+    if enable_thinking is True:
+        return "<think>\n"
+    if enable_thinking is False:
+        return "</think>\n"
+    if chat_style == "think":
+        return "<think>\n"
+    if chat_style == "answer":
+        return "</think>\n"
+    return ""
+
+
+class BaguettotronVLMProcessor(ProcessorMixin):
+    """
+    Wraps CLIPImageProcessor + Baguettotron tokenizer.
+
+    Expands a single <image> placeholder into NUM_VISUAL_TOKENS consecutive
+    <image> token IDs so the model's forward() can replace them with ViT
+    features. Builds Qwen-style chat prompts with stage-appropriate
+    assistant prefixes.
+
+    Load with::
+
+        from transformers import AutoProcessor
+        processor = AutoProcessor.from_pretrained(
+            "andreagemelli/Baguettotron-VLM",
+            trust_remote_code=True,
+        )
+    """
+
+    attributes = ["image_processor", "tokenizer"]
+    image_processor_class = "CLIPImageProcessor"
+    tokenizer_class = "AutoTokenizer"
+
+    def __init__(
+        self,
+        image_processor: CLIPImageProcessor,
+        tokenizer: PreTrainedTokenizerFast,
+        num_visual_tokens: int = NUM_VISUAL_TOKENS,
+        chat_style: str = "answer",
+    ):
+        super().__init__(image_processor, tokenizer)
+        self.num_visual_tokens = num_visual_tokens
+        self.chat_style = chat_style
+        raw_id = tokenizer.convert_tokens_to_ids(IMAGE_TOKEN)
+        self.image_token_id: int = raw_id if isinstance(raw_id, int) else int(raw_id[0])
+
+    @classmethod
+    def from_pretrained(cls, pretrained_model_name_or_path, **kwargs):  # type: ignore[override]
+        processor = super().from_pretrained(pretrained_model_name_or_path, **kwargs)
+        # Re-register special tokens (some tokenizers drop them on save/load)
+        processor.tokenizer.add_special_tokens(
+            {"additional_special_tokens": [IMAGE_TOKEN, IMAGE_END_TOKEN]}
+        )
+        raw_id = processor.tokenizer.convert_tokens_to_ids(IMAGE_TOKEN)
+        processor.image_token_id = raw_id if isinstance(raw_id, int) else int(raw_id[0])
+
+        # Load chat_style from config.json (written by push_to_hub per stage)
+        chat_style = "answer"
+        try:
+            if os.path.isdir(pretrained_model_name_or_path):
+                cfg_path = Path(pretrained_model_name_or_path) / "config.json"
+                if cfg_path.exists():
+                    chat_style = json.loads(cfg_path.read_text()).get(
+                        "chat_style", chat_style
+                    )
+            else:
+                from huggingface_hub import hf_hub_download
+                cfg_path = hf_hub_download(
+                    repo_id=pretrained_model_name_or_path, filename="config.json"
+                )
+                chat_style = json.loads(Path(cfg_path).read_text()).get(
+                    "chat_style", chat_style
+                )
+        except Exception:
+            pass
+        processor.chat_style = chat_style
+        return processor
+
+    def _format_messages(
+        self,
+        messages: list[dict],
+        add_generation_prompt: bool,
+        enable_thinking: bool | None,
+    ) -> str:
+        parts = [
+            f"<|im_start|>{m['role']}\n{m['content']}<|im_end|>" for m in messages
+        ]
+        text = "\n".join(parts)
+        if add_generation_prompt:
+            prefix = _assistant_prefix(self.chat_style, enable_thinking)
+            text = f"{text}\n<|im_start|>assistant\n{prefix}"
+        return text
+
+    def __call__(
+        self,
+        text: str | None = None,
+        messages: list[dict] | None = None,
+        image: Image.Image | None = None,
+        return_tensors: str = "pt",
+        add_generation_prompt: bool = True,
+        enable_thinking: bool | None = None,
+    ) -> dict:
+        """Tokenise text (or a messages list) and optionally preprocess an image.
+
+        Args:
+            text: raw prompt string (with a single <image> placeholder).
+            messages: alternative to text — list of chat dicts with "role"/"content".
+            image: PIL image to preprocess (optional).
+            add_generation_prompt: append an <|im_start|>assistant\n prefix.
+            enable_thinking: override chat_style for this call.
+                - None: keep the stage default (chat_style)
+                - True: pre-fill <think>\n (stage 3 reasoning mode)
+                - False: pre-fill </think>\n (stage 2 / stage 3 no-think mode)
+        """
+        if messages is not None:
+            text = self._format_messages(
+                messages, add_generation_prompt, enable_thinking
+            )
+        if text is None:
+            raise ValueError("Provide either text or messages.")
+
+        expanded = text.replace(IMAGE_TOKEN, IMAGE_TOKEN * self.num_visual_tokens, 1)
+        enc = self.tokenizer(
+            expanded, return_tensors=return_tensors, add_special_tokens=False
+        )
+        result = {
+            "input_ids": enc["input_ids"],
+            "attention_mask": enc["attention_mask"],
+        }
+        if image is not None:
+            pv = self.image_processor(images=image, return_tensors=return_tensors)
+            result["pixel_values"] = pv.pixel_values
+        else:
+            result["pixel_values"] = None
+        return result

processor_config.json

@@ -0,0 +1,8 @@
+{
+  "processor_class": "BaguettotronVLMProcessor",
+  "chat_style": "base",
+  "num_visual_tokens": 256,
+  "auto_map": {
+    "AutoProcessor": "processing_baguettotron_vlm.BaguettotronVLMProcessor"
+  }
+}

special_tokens_map.json

@@ -0,0 +1,46 @@
+{
+  "additional_special_tokens": [
+    {
+      "content": "<image>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "</image>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    }
+  ],
+  "bos_token": {
+    "content": "<|begin_of_text|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|end_of_text|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "[PAD]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "[UNK]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,424 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "[UNK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<|begin_of_text|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "<|end_of_text|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "[PAD]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "65491": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "65492": {
+      "content": "<|im_end>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "65493": {
+      "content": "<think>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "65494": {
+      "content": "</think>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "65495": {
+      "content": "source_1",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "65496": {
+      "content": "source_2",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "65497": {
+      "content": "source_3",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "65498": {
+      "content": "source_4",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "65499": {
+      "content": "source_5",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "65500": {
+      "content": "source_6",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "65501": {
+      "content": "source_7",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "65502": {
+      "content": "source_8",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "65503": {
+      "content": "source_9",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "65504": {
+      "content": "source_10",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "65505": {
+      "content": "<ref",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "65506": {
+      "content": "</ref>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "65507": {
+      "content": "→",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "65508": {
+      "content": "↺",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "65509": {
+      "content": "※",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "65510": {
+      "content": "?maybe?",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "65511": {
+      "content": "●",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "65512": {
+      "content": "◐",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "65513": {
+      "content": "○",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "65514": {
+      "content": "⚠",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "65515": {
+      "content": "☐",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "65516": {
+      "content": "☑",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "65517": {
+      "content": "✓",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "65518": {
+      "content": "⟨H≈0.1⟩",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "65519": {
+      "content": "⟨H≈0.2⟩",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "65520": {
+      "content": "⟨H≈0.3⟩",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "65521": {
+      "content": "⟨H≈0.4⟩",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "65522": {
+      "content": "⟨H≈0.5⟩",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "65523": {
+      "content": "⟨H≈0.6⟩",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "65524": {
+      "content": "⟨H≈0.7⟩",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "65525": {
+      "content": "⟨H≈0.8⟩",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "65526": {
+      "content": "⟨H≈0.9⟩",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "65527": {
+      "content": "⟨H≈1.0⟩",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "65528": {
+      "content": "⟨H≈1.1⟩",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "65529": {
+      "content": "⟨H≈1.2⟩",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "65530": {
+      "content": "⟨H≈1.3⟩",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "65531": {
+      "content": "⟨H≈1.4⟩",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "65532": {
+      "content": "⟨H≈1.5⟩",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "65533": {
+      "content": "⟨H≈1.6⟩",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "65534": {
+      "content": "⟨H≈1.7⟩",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "65535": {
+      "content": "⟨H≈1.8⟩",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "65536": {
+      "content": "<image>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "65537": {
+      "content": "</image>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<image>",
+    "</image>"
+  ],
+  "bos_token": "<|begin_of_text|>",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|end_of_text|>",
+  "extra_special_tokens": {},
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "[PAD]",
+  "tokenizer_class": "PreTrainedTokenizerFast",
+  "unk_token": "[UNK]"
+}