Duplicate from onnx-community/jina-embeddings-v5-omni-nano-ONNX

Co-authored-by: Shreyas Karnik <shreyask@users.noreply.huggingface.co>

.gitattributes

@@ -0,0 +1,45 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text
+onnx/text_model.onnx.data filter=lfs diff=lfs merge=lfs -text
+onnx/text_model_fp16.onnx_data filter=lfs diff=lfs merge=lfs -text
+onnx/text_model_q4f16.onnx_data filter=lfs diff=lfs merge=lfs -text
+onnx/vision_model.onnx.data filter=lfs diff=lfs merge=lfs -text
+onnx/vision_model_fp16.onnx_data filter=lfs diff=lfs merge=lfs -text
+onnx/vision_model_q4f16.onnx_data filter=lfs diff=lfs merge=lfs -text
+onnx/audio_model.onnx.data filter=lfs diff=lfs merge=lfs -text
+onnx/audio_model_fp16.onnx_data filter=lfs diff=lfs merge=lfs -text
+onnx/audio_model_q4f16.onnx_data filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,161 @@
+---
+license: cc-by-nc-4.0
+base_model: jinaai/jina-embeddings-v5-omni-nano
+base_model_relation: quantized
+library_name: transformers.js
+pipeline_tag: feature-extraction
+tags:
+- feature-extraction
+- sentence-similarity
+- multimodal
+- cross-modal-retrieval
+- onnx
+- webgpu
+- transformers.js
+- jina-embeddings
+- embeddings
+language:
+- multilingual
+---
+
+# jina-embeddings-v5-omni-nano · ONNX
+
+ONNX exports of [`jinaai/jina-embeddings-v5-omni-nano`](https://huggingface.co/jinaai/jina-embeddings-v5-omni-nano) for in-browser inference via [transformers.js](https://huggingface.co/docs/transformers.js) **v4** with WebGPU.
+
+`jina-embeddings-v5-omni-nano` is a ~1.04 B-parameter multimodal embedding model that maps **text, images, audio, and video** into a single shared 768-dimensional L2-normalized space, enabling cross-modal retrieval without reindexing.
+
+## Architecture
+
+The base model is a LLaVA-style composition of three frozen encoders plus small trainable projectors, with per-task LoRA adapters. This repo's exports have the **`retrieval`** task adapter merged into the static weights.
+
+| Tower | Backbone | Role |
+|---|---|---|
+| Text | EuroBERT-210m (loaded as a bidirectional Llama) | Text encoder |
+| Vision | Qwen3-VL vision tower + spatial merger | Image and video frames |
+| Audio | Whisper-large-v3 encoder + Qwen2.5-Omni audio adapter | Audio (16 kHz mono) |
+
+Each tower is bundled with its retrieval projector AND a copy of the language model (which performs the final cross-modal fusion + last-token pooling), so the three ONNX graphs are independently loadable.
+
+## Files
+
+Three ONNX graphs, three quantization variants each. Every variant ships as a small `.onnx` schema + a large `.onnx.data` or `.onnx_data` external-weight sidecar (HF Hub-friendly layout, no 2 GB protobuf limits).
+
+| Modality | fp32 | fp16 | q4f16 | Verified parity vs fp32 (fp16) |
+|---|---:|---:|---:|---|
+| Text   |  849 MB | 424 MB |  263 MB | cos = 1.000000 |
+| Vision | 1247 MB | 622 MB |  460 MB | cos = 0.999998 |
+| Audio  | 3400 MB | 1700 MB | 1465 MB | (fp16 doesn't load on CPU EP; verify on WebGPU) |
+
+**For desktop WebGPU, use `q4f16`** — it's the smallest and runs natively on shader-int4 hardware. Use `fp16` if you need higher numerical fidelity or your GPU lacks int4 paths.
+
+Every graph outputs a single tensor `sentence_embedding` of shape `[batch, 768]`, already L2-normalized — cosine similarity reduces to a dot product.
+
+## Input constraints
+
+**Text** (`text_model*.onnx`): inputs `input_ids` and `attention_mask`, both `[batch, seq]` with `seq` fully dynamic. Apply the asymmetric retrieval prefix convention before tokenizing: `Query: …` for queries, `Document: …` for corpus items.
+
+**Vision** (`vision_model*.onnx`): the graph is traced at a fixed image-patch layout. `image_grid_thw` is folded as a constant (it drives `torch.linspace` inside Qwen3-VL's `fast_pos_embed_interpolate`, which dynamo cannot symbolicate). Resize every image to **224×224** before passing through `LlavaEuroBertProcessor` — that yields the exact shapes the graph expects:
+
+```
+input_ids        [batch, 271]    int64
+attention_mask   [batch, 271]    int64
+pixel_values     [1024, 1536]    float32
+```
+
+`image_grid_thw` is NOT an ONNX input (it's a constant). Don't pass it.
+
+**Audio** (`audio_model*.onnx`): traced with a 5-second 16 kHz mono clip. The graph expects exactly:
+
+```
+input_ids                [batch, 125]      int64   (audio_token_id placeholders)
+attention_mask           [batch, 125]      int64
+input_features           [1, 128, 3000]    float32 (Whisper log-mel, 30s padded)
+feature_attention_mask   [1, 3000]         int64   (frame-level, 500 ones for 5s of real audio)
+```
+
+Pad or truncate every clip to 5 s. Longer-clip chunking (sliding 5 s windows, average pooled embeddings) is a v2 follow-up.
+
+## Use with transformers.js v4
+
+### Text
+
+```js
+import { AutoTokenizer } from "@huggingface/transformers";
+import * as ort from "onnxruntime-web";
+
+const REPO = "shreyask/jina-embeddings-v5-omni-nano-ONNX";
+
+const tok = await AutoTokenizer.from_pretrained(REPO);
+const sess = await ort.InferenceSession.create(
+  `https://huggingface.co/${REPO}/resolve/main/onnx/text_model_q4f16.onnx`,
+  { executionProviders: ["webgpu", "wasm"] },
+);
+
+const { input_ids, attention_mask } = await tok(
+  "Query: a saxophone solo",
+  { return_tensors: "ort" },
+);
+const { sentence_embedding } = await sess.run({ input_ids, attention_mask });
+// Float32Array of length 768, already L2-normalized.
+```
+
+### Vision
+
+```js
+import { AutoProcessor } from "@huggingface/transformers";
+import * as ort from "onnxruntime-web";
+
+const proc = await AutoProcessor.from_pretrained(REPO);
+const sess = await ort.InferenceSession.create(
+  `https://huggingface.co/${REPO}/resolve/main/onnx/vision_model_q4f16.onnx`,
+  { executionProviders: ["webgpu", "wasm"] },
+);
+
+// Resize to 224x224 before passing in.
+const inputs = await proc.apply_chat_template(
+  [{ role: "user", content: [{ type: "image", image: imageBlob }] }],
+  { add_generation_prompt: false, tokenize: true, return_dict: true, return_tensors: "ort" },
+);
+const { sentence_embedding } = await sess.run({
+  input_ids: inputs.input_ids,
+  attention_mask: inputs.attention_mask,
+  pixel_values: inputs.pixel_values,
+});
+```
+
+### Audio
+
+Audio preprocessing isn't bundled in `LlavaEuroBertProcessor` — use Whisper's feature extractor directly and stamp `audio_token_id` placeholders into `input_ids`. See the [reference implementation](https://huggingface.co/jinaai/jina-embeddings-v5-omni-nano/blob/main/modeling_jina_embeddings_v5_omni.py) for the exact mel-spec → placeholder count plumbing.
+
+## Cross-modal retrieval
+
+All three towers project into the same 768-dim space, so a text query can rank images / audio / video corpus items (and vice versa) without re-indexing. Embeddings are L2-normalized, so cosine similarity is a dot product:
+
+```js
+const score = textVec.reduce((s, v, i) => s + v * imageVec[i], 0);
+```
+
+## How these were exported
+
+- `torch` 2.11, `transformers` 5.8, `onnx` 1.21, `onnxruntime` 1.26
+- **Text**: `torch.onnx.export(..., dynamo=True)`. The LlamaModel-based encoder exports cleanly through the dynamo path
+- **Vision and audio**: `torch.onnx.export(..., dynamo=False)` (legacy TorchScript tracer). Dynamo refuses to specialize Qwen3-VL's data-dependent `torch.linspace`, and the GQA-aware SDPA is monkey-patched with a manual MatMul+Softmax for the trace's duration
+- PEFT LoRA fused via `merge_and_unload(safe_merge=True)` so the `retrieval` task adapter is baked in
+- fp16 cast via `onnxruntime.transformers.float16.convert_float_to_float16` (the `onnxconverter_common` path mishandles dynamo's `_to_copy` nodes)
+- 4-bit quant via `onnxruntime.quantization.matmul_nbits_quantizer.MatMulNBitsQuantizer` (`bits=4, block_size=32, accuracy_level=4`)
+- Two graph post-patches required for the audio tower's ORT-loadability: `Cast(to=int64)` inserted before every `Slice` index input (366 inserts), and `Unsqueeze(0)`/`Squeeze(0)` wrapped around the rank-2-input `AveragePool` (1 site)
+
+## License
+
+Inherited from the base model: **CC BY-NC 4.0**. Commercial use requires reaching out to `sales@jina.ai`.
+
+## Citation
+
+```bibtex
+@misc{jina-embeddings-v5-omni-nano,
+  author = {Jina AI},
+  title  = {jina-embeddings-v5-omni-nano},
+  year   = {2025},
+  url    = {https://huggingface.co/jinaai/jina-embeddings-v5-omni-nano}
+}
+```

chat_template.jinja

@@ -0,0 +1,154 @@
+{%- set image_count = namespace(value=0) %}
+{%- set video_count = namespace(value=0) %}
+{%- macro render_content(content, do_vision_count, is_system_content=false) %}
+    {%- if content is string %}
+        {{- content }}
+    {%- elif content is iterable and content is not mapping %}
+        {%- for item in content %}
+            {%- if 'image' in item or 'image_url' in item or item.type == 'image' %}
+                {%- if is_system_content %}
+                    {{- raise_exception('System message cannot contain images.') }}
+                {%- endif %}
+                {%- if do_vision_count %}
+                    {%- set image_count.value = image_count.value + 1 %}
+                {%- endif %}
+                {%- if add_vision_id %}
+                    {{- 'Picture ' ~ image_count.value ~ ': ' }}
+                {%- endif %}
+                {{- '<|vision_start|><|image_pad|><|vision_end|>' }}
+            {%- elif 'video' in item or item.type == 'video' %}
+                {%- if is_system_content %}
+                    {{- raise_exception('System message cannot contain videos.') }}
+                {%- endif %}
+                {%- if do_vision_count %}
+                    {%- set video_count.value = video_count.value + 1 %}
+                {%- endif %}
+                {%- if add_vision_id %}
+                    {{- 'Video ' ~ video_count.value ~ ': ' }}
+                {%- endif %}
+                {{- '<|vision_start|><|video_pad|><|vision_end|>' }}
+            {%- elif 'text' in item %}
+                {{- item.text }}
+            {%- else %}
+                {{- raise_exception('Unexpected item type in content.') }}
+            {%- endif %}
+        {%- endfor %}
+    {%- elif content is none or content is undefined %}
+        {{- '' }}
+    {%- else %}
+        {{- raise_exception('Unexpected content type.') }}
+    {%- endif %}
+{%- endmacro %}
+{%- if not messages %}
+    {{- raise_exception('No messages provided.') }}
+{%- endif %}
+{%- if tools and tools is iterable and tools is not mapping %}
+    {{- '<|im_start|>system\n' }}
+    {{- "# Tools\n\nYou have access to the following functions:\n\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>" }}
+    {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n<tool_call>\n<function=example_function_name>\n<parameter=example_parameter_1>\nvalue_1\n</parameter>\n<parameter=example_parameter_2>\nThis is the value for the second parameter\nthat can span\nmultiple lines\n</parameter>\n</function>\n</tool_call>\n\n<IMPORTANT>\nReminder:\n- Function calls MUST follow the specified format: an inner <function=...></function> block must be nested within <tool_call></tool_call> XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n</IMPORTANT>' }}
+    {%- if messages[0].role == 'system' %}
+        {%- set content = render_content(messages[0].content, false, true)|trim %}
+        {%- if content %}
+            {{- '\n\n' + content }}
+        {%- endif %}
+    {%- endif %}
+    {{- '<|im_end|>\n' }}
+{%- else %}
+    {%- if messages[0].role == 'system' %}
+        {%- set content = render_content(messages[0].content, false, true)|trim %}
+        {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+    {%- set index = (messages|length - 1) - loop.index0 %}
+    {%- if ns.multi_step_tool and message.role == "user" %}
+        {%- set content = render_content(message.content, false)|trim %}
+        {%- if not(content.startswith('<tool_response>') and content.endswith('</tool_response>')) %}
+            {%- set ns.multi_step_tool = false %}
+            {%- set ns.last_query_index = index %}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if ns.multi_step_tool %}
+    {{- raise_exception('No user query found in messages.') }}
+{%- endif %}
+{%- for message in messages %}
+    {%- set content = render_content(message.content, true)|trim %}
+    {%- if message.role == "system" %}
+        {%- if not loop.first %}
+            {{- raise_exception('System message must be at the beginning.') }}
+        {%- endif %}
+    {%- elif message.role == "user" %}
+        {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {%- set reasoning_content = '' %}
+        {%- if message.reasoning_content is string %}
+            {%- set reasoning_content = message.reasoning_content %}
+        {%- else %}
+            {%- if '</think>' in content %}
+                {%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
+                {%- set content = content.split('</think>')[-1].lstrip('\n') %}
+            {%- endif %}
+        {%- endif %}
+        {%- set reasoning_content = reasoning_content|trim %}
+        {%- if loop.index0 > ns.last_query_index %}
+            {{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content + '\n</think>\n\n' + content }}
+        {%- else %}
+            {{- '<|im_start|>' + message.role + '\n' + content }}
+        {%- endif %}
+        {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %}
+            {%- for tool_call in message.tool_calls %}
+                {%- if tool_call.function is defined %}
+                    {%- set tool_call = tool_call.function %}
+                {%- endif %}
+                {%- if loop.first %}
+                    {%- if content|trim %}
+                        {{- '\n\n<tool_call>\n<function=' + tool_call.name + '>\n' }}
+                    {%- else %}
+                        {{- '<tool_call>\n<function=' + tool_call.name + '>\n' }}
+                    {%- endif %}
+                {%- else %}
+                    {{- '\n<tool_call>\n<function=' + tool_call.name + '>\n' }}
+                {%- endif %}
+                {%- if tool_call.arguments is defined %}
+                    {%- for args_name, args_value in tool_call.arguments|items %}
+                        {{- '<parameter=' + args_name + '>\n' }}
+                        {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %}
+                        {{- args_value }}
+                        {{- '\n</parameter>\n' }}
+                    {%- endfor %}
+                {%- endif %}
+                {{- '</function>\n</tool_call>' }}
+            {%- endfor %}
+        {%- endif %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if loop.previtem and loop.previtem.role != "tool" %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- content }}
+        {{- '\n</tool_response>' }}
+        {%- if not loop.last and loop.nextitem.role != "tool" %}
+            {{- '<|im_end|>\n' }}
+        {%- elif loop.last %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- else %}
+        {{- raise_exception('Unexpected message role.') }}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+    {%- if enable_thinking is defined and enable_thinking is true %}
+        {{- '<think>\n' }}
+    {%- else %}
+        {{- '<think>\n\n</think>\n\n' }}
+    {%- endif %}
+{%- endif %}

config.json

@@ -0,0 +1,101 @@
+{
+  "architectures": [
+    "JinaEmbeddingsV5OmniModel"
+  ],
+  "auto_map": {
+    "AutoConfig": "modeling_jina_embeddings_v5_omni.JinaEmbeddingsV5OmniConfig",
+    "AutoModel": "modeling_jina_embeddings_v5_omni.JinaEmbeddingsV5OmniModel"
+  },
+  "model_type": "jina_embeddings_v5_omni",
+  "task_names": [
+    "retrieval",
+    "text-matching",
+    "clustering",
+    "classification"
+  ],
+  "special_token_ids": [
+    128256,
+    128257,
+    128258,
+    128259
+  ],
+  "vision_config": {
+    "deepstack_visual_indexes": [],
+    "depth": 12,
+    "dtype": "bfloat16",
+    "hidden_act": "gelu_pytorch_tanh",
+    "hidden_size": 768,
+    "in_channels": 3,
+    "initializer_range": 0.02,
+    "intermediate_size": 3072,
+    "model_type": "",
+    "num_heads": 12,
+    "num_position_embeddings": 2304,
+    "out_hidden_size": 1024,
+    "patch_size": 16,
+    "spatial_merge_size": 2,
+    "temporal_patch_size": 2
+  },
+  "text_config": {
+    "attention_bias": false,
+    "attention_dropout": 0.0,
+    "bos_token_id": 1,
+    "eos_token_id": 2,
+    "head_dim": 64,
+    "hidden_act": "silu",
+    "hidden_size": 768,
+    "initializer_range": 0.02,
+    "intermediate_size": 3072,
+    "is_causal": false,
+    "max_position_embeddings": 8192,
+    "mlp_bias": false,
+    "model_type": "",
+    "num_attention_heads": 12,
+    "num_hidden_layers": 12,
+    "num_key_value_heads": 12,
+    "pad_token_id": null,
+    "pretraining_tp": 1,
+    "rms_norm_eps": 1e-05,
+    "rope_parameters": {
+      "rope_theta": 1000000.0,
+      "rope_type": "default"
+    },
+    "tie_word_embeddings": false,
+    "vocab_size": 128260
+  },
+  "audio_config": {
+    "activation_dropout": 0.0,
+    "activation_function": "gelu",
+    "attention_dropout": 0.0,
+    "d_model": 1280,
+    "dropout": 0.0,
+    "dtype": "float32",
+    "encoder_attention_heads": 20,
+    "encoder_ffn_dim": 5120,
+    "encoder_layers": 32,
+    "initializer_range": 0.02,
+    "max_source_positions": 1500,
+    "num_mel_bins": 128,
+    "scale_embedding": false,
+    "n_window": 100,
+    "output_dim": 3584
+  },
+  "image_token_index": 128259,
+  "audio_token_id": 128256,
+  "audio_start_token_id": 128257,
+  "audio_end_token_id": 128258,
+  "projector_hidden_act": "gelu",
+  "tie_word_embeddings": false,
+  "dtype": "bfloat16",
+  "transformers_version": "5.4.0",
+  "torch_dtype": "bfloat16",
+  "is_matryoshka": true,
+  "matryoshka_dimensions": [
+    32,
+    64,
+    128,
+    256,
+    512,
+    768
+  ]
+}

modeling_jina_embeddings_v5_omni.py

@@ -0,0 +1,616 @@
+"""
+Unified jina-embeddings-v5-omni-nano: vision + audio + text with task-specific routing.
+
+Shared:  Qwen3VLVisionModel + Qwen2.5-Omni audio encoder + LlamaModel (EuroBERT, bidirectional)
+Per-task: vision merger, audio projector, special token embeddings, LoRA adapter
+
+Modality loading:
+    model = AutoModel.from_pretrained(path, trust_remote_code=True)                      # all components (default)
+    model = AutoModel.from_pretrained(path, trust_remote_code=True, modality="vision")    # no audio tower/projectors
+    model = AutoModel.from_pretrained(path, trust_remote_code=True, modality="audio")     # no vision tower/mergers
+
+Usage:
+    model = AutoModel.from_pretrained("jinaai/jina-embeddings-v5-omni-nano", trust_remote_code=True)
+    embeddings = model.encode(["hello world"], task="retrieval")
+"""
+
+from typing import List, Optional
+import os
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+from huggingface_hub import snapshot_download
+from transformers import AutoTokenizer, LlamaConfig, PreTrainedModel, PretrainedConfig
+from transformers.modeling_outputs import BaseModelOutputWithPast
+from transformers.models.llama.modeling_llama import LlamaModel
+from transformers.models.qwen3_vl.configuration_qwen3_vl import Qwen3VLVisionConfig
+from transformers.models.qwen3_vl.modeling_qwen3_vl import Qwen3VLVisionModel
+from transformers.models.qwen2_5_omni.configuration_qwen2_5_omni import Qwen2_5OmniAudioEncoderConfig
+from transformers.models.qwen2_5_omni.modeling_qwen2_5_omni import Qwen2_5OmniAudioEncoder
+from peft import PeftMixedModel, PeftConfig
+
+TASK_NAMES = ["retrieval", "text-matching", "clustering", "classification"]
+_VALID_MODALITIES = ("omni", "vision", "audio", "text")
+
+
+def _key(task):
+    return task.replace("-", "_")
+
+
+class PretrainedMerger(nn.Module):
+    def __init__(self, hidden_size, out_hidden_size, spatial_merge_size=2):
+        super().__init__()
+        self.hidden_size = hidden_size * (spatial_merge_size ** 2)
+        self.norm = nn.LayerNorm(hidden_size, eps=1e-6)
+        self.linear_fc1 = nn.Linear(self.hidden_size, self.hidden_size)
+        self.act = nn.GELU()
+        self.linear_fc2 = nn.Linear(self.hidden_size, out_hidden_size)
+
+    def forward(self, x):
+        x = self.norm(x)
+        x = x.view(-1, self.hidden_size)
+        x = self.linear_fc2(self.act(self.linear_fc1(x)))
+        return x
+
+
+class JinaEmbeddingsV5OmniConfig(PretrainedConfig):
+    model_type = "jina_embeddings_v5_omni"
+
+    def __init__(
+        self,
+        vision_config=None,
+        text_config=None,
+        audio_config=None,
+        task_names=None,
+        special_token_ids=None,
+        image_token_index=None,
+        audio_token_id=None,
+        audio_start_token_id=None,
+        audio_end_token_id=None,
+        projector_hidden_act="gelu",
+        tie_word_embeddings=False,
+        modality="omni",
+        **kwargs,
+    ):
+        if isinstance(vision_config, dict):
+            vision_config = PretrainedConfig(**vision_config)
+        self.vision_config = vision_config or PretrainedConfig()
+        if isinstance(text_config, dict):
+            text_config = PretrainedConfig(**text_config)
+        self.text_config = text_config or PretrainedConfig()
+        if isinstance(audio_config, dict):
+            audio_config = PretrainedConfig(**audio_config)
+        self.audio_config = audio_config or PretrainedConfig()
+        self.task_names = task_names or TASK_NAMES
+        self.special_token_ids = special_token_ids or []
+        self.image_token_index = image_token_index
+        self.audio_token_id = audio_token_id
+        self.audio_start_token_id = audio_start_token_id
+        self.audio_end_token_id = audio_end_token_id
+        self.projector_hidden_act = projector_hidden_act
+        if modality not in _VALID_MODALITIES:
+            raise ValueError(f"modality must be one of {_VALID_MODALITIES}, got '{modality}'")
+        self.modality = modality
+        super().__init__(tie_word_embeddings=tie_word_embeddings, **kwargs)
+
+    def get_text_config(self, **kwargs):
+        return self.text_config
+
+
+class JinaEmbeddingsV5OmniBase(PreTrainedModel):
+    config_class = JinaEmbeddingsV5OmniConfig
+    supports_gradient_checkpointing = True
+    _supports_sdpa = True
+    _supports_flash_attn_2 = True
+    _supports_attention_backend = True
+    _tied_weights_keys = []
+    _keys_to_ignore_on_load_missing = ["lm_head.weight"]
+    _keys_to_ignore_on_load_unexpected = []
+
+    def __init__(self, config: JinaEmbeddingsV5OmniConfig):
+        super().__init__(config)
+
+        modality = getattr(config, "modality", "omni")
+        if modality not in _VALID_MODALITIES:
+            raise ValueError(f"modality must be one of {_VALID_MODALITIES}, got '{modality}'")
+        self._modality = modality
+
+        vision_cfg = config.vision_config
+        if not isinstance(vision_cfg, Qwen3VLVisionConfig):
+            d = vision_cfg.to_dict() if hasattr(vision_cfg, "to_dict") else dict(vision_cfg)
+            d.pop("model_type", None)
+            d.pop("transformers_version", None)
+            vision_cfg = Qwen3VLVisionConfig(**d)
+        vision_cfg.deepstack_visual_indexes = []
+
+        spatial_merge_size = getattr(vision_cfg, "spatial_merge_size", 2)
+        self._spatial_merge_size = spatial_merge_size
+        self._vision_hidden_size = vision_cfg.hidden_size
+
+        text_cfg = config.text_config
+        txt_dict = text_cfg.to_dict() if hasattr(text_cfg, "to_dict") else text_cfg
+        if not isinstance(text_cfg, LlamaConfig):
+            text_cfg = LlamaConfig(**txt_dict)
+        text_hidden = text_cfg.hidden_size
+
+        if modality not in ("audio", "text"):
+            self.vision_tower = Qwen3VLVisionModel(vision_cfg)
+            self.vision_tower.merger = nn.Identity()
+            self.vision_tower.deepstack_merger_list = nn.ModuleList()
+            self.vision_tower.deepstack_visual_indexes = []
+            self.mergers = nn.ModuleDict({
+                _key(t): PretrainedMerger(vision_cfg.hidden_size, text_hidden, spatial_merge_size)
+                for t in config.task_names
+            })
+
+        self.language_model = LlamaModel(text_cfg)
+        for layer in self.language_model.layers:
+            layer.self_attn.is_causal = False
+
+        self.multi_modal_projector = nn.Identity()
+        self.lm_head = nn.Identity()
+
+        if modality not in ("vision", "text"):
+            aud_cfg = config.audio_config
+            aud_dict = aud_cfg.to_dict() if hasattr(aud_cfg, "to_dict") else aud_cfg
+            audio_encoder_config = Qwen2_5OmniAudioEncoderConfig(**aud_dict)
+            self.audio_tower = Qwen2_5OmniAudioEncoder(audio_encoder_config)
+            self.audio_tower.proj = nn.Identity()  # fused into audio_projector(s)
+            output_dim = aud_dict.get('d_model', 1280)  # fused: audio_projector(s) now take d_model
+            self.audio_projectors = nn.ModuleDict({
+                _key(t): nn.Linear(output_dim, text_hidden) for t in config.task_names
+            })
+
+        ignore = []
+        if modality in ("audio", "text"):
+            ignore.extend([r"^vision_tower\.", r"^mergers\."])
+        if modality in ("vision", "text"):
+            ignore.extend([r"^audio_tower\.", r"^audio_projectors\."])
+        if ignore:
+            self._keys_to_ignore_on_load_unexpected = ignore
+
+        n_special = len(config.special_token_ids)
+        self.task_token_embeddings = nn.ParameterDict({
+            _key(t): nn.Parameter(torch.zeros(n_special, text_hidden))
+            for t in config.task_names
+        })
+
+        self._active_task_key = _key(config.task_names[0])
+        self._special_token_ids = config.special_token_ids
+        self.post_init()
+
+    @property
+    def modality(self) -> str:
+        return self._modality
+
+    def set_task(self, task):
+        k = _key(task)
+        self._active_task_key = k
+        with torch.no_grad():
+            w = self.language_model.embed_tokens.weight.data
+            te = self.task_token_embeddings[k]
+            for i, tid in enumerate(self._special_token_ids):
+                w[tid] = te[i]
+
+    def get_input_embeddings(self):
+        return self.language_model.embed_tokens
+
+    def set_input_embeddings(self, value):
+        self.language_model.embed_tokens = value
+
+    def get_output_embeddings(self):
+        return None
+
+    def get_image_features(self, pixel_values, image_grid_thw, num_image_tokens=None):
+        if self._modality in ("audio", "text"):
+            raise ValueError(
+                f"Vision inputs are not available in {self._modality}-only mode. "
+                "Load with modality='omni' or modality='vision'."
+            )
+
+        out = self.vision_tower(hidden_states=pixel_values, grid_thw=image_grid_thw)
+        raw = out[0] if isinstance(out, tuple) else getattr(out, "last_hidden_state", out[0])
+        merged = self.mergers[self._active_task_key](raw)
+
+        merge = self._spatial_merge_size
+        sizes = []
+        for i in range(image_grid_thw.shape[0]):
+            t, h, w = image_grid_thw[i].tolist()
+            sizes.append(int(t) * (int(h) // merge) * (int(w) // merge))
+
+        # Default: return the un-padded per-image feature slices. Their
+        # concatenation has exactly sum(sizes) rows == number of <image>
+        # placeholder tokens in input_ids, which is what masked_scatter
+        # consumes. Padding is only meaningful when callers want a square
+        # [N, max_tok, dim] block (e.g. multi-sample batched forward where
+        # each row owns its own image), and that path passes
+        # num_image_tokens explicitly to opt in.
+        dim = merged.shape[-1]
+        features, offset = [], 0
+        if num_image_tokens is not None:
+            max_tok = num_image_tokens
+            for n in sizes:
+                feat = merged[offset:offset + n]
+                if n < max_tok:
+                    feat = torch.cat([feat, feat.new_zeros(max_tok - n, dim)], dim=0)
+                features.append(feat)
+                offset += n
+        else:
+            for n in sizes:
+                features.append(merged[offset:offset + n])
+                offset += n
+        return features
+
+    def get_audio_features(self, input_features, feature_attention_mask=None):
+        if self._modality in ("vision", "text"):
+            raise ValueError(
+                f"Audio inputs are not available in {self._modality}-only mode. "
+                "Load with modality='omni' or modality='audio'."
+            )
+
+        batch_size = input_features.shape[0]
+        if batch_size > 1:
+            # Serialize per-sample so the packed-frames GEMM shape stays invariant
+            # across batch sizes. Makes batched audio bit-exact to B=1 in bf16,
+            # and is substantially faster for B>=16 because B=1 hits a
+            # well-optimized kernel while the packed-B=N path thrashes on a
+            # (total_frames)^2 sdpa matrix.
+            outs = [
+                self.get_audio_features(
+                    input_features[i : i + 1],
+                    feature_attention_mask[i : i + 1] if feature_attention_mask is not None else None,
+                )
+                for i in range(batch_size)
+            ]
+            return torch.cat(outs, dim=0)
+        if feature_attention_mask is not None:
+            feature_lens = feature_attention_mask.sum(-1).long()
+            packed = input_features.permute(0, 2, 1)[feature_attention_mask.bool()].permute(1, 0)
+        else:
+            feature_lens = torch.full(
+                (batch_size,), input_features.shape[2],
+                device=input_features.device, dtype=torch.long,
+            )
+            packed = input_features.transpose(1, 2).reshape(-1, input_features.shape[1]).T
+        aftercnn_lens, _ = self.audio_tower._get_feat_extract_output_lengths(feature_lens)
+        audio_output = self.audio_tower(
+            packed, feature_lens=feature_lens, aftercnn_lens=aftercnn_lens,
+        )
+        return self.audio_projectors[self._active_task_key](audio_output.last_hidden_state)
+
+    def forward(
+        self,
+        input_ids=None,
+        pixel_values=None,
+        attention_mask=None,
+        position_ids=None,
+        past_key_values=None,
+        inputs_embeds=None,
+        input_features=None,
+        feature_attention_mask=None,
+        cache_position=None,
+        output_hidden_states=None,
+        **kwargs,
+    ):
+        image_grid_thw = kwargs.pop("image_grid_thw", None)
+        num_image_tokens = kwargs.pop("num_image_tokens", None)
+        pixel_values_videos = kwargs.pop("pixel_values_videos", None)
+        video_grid_thw = kwargs.pop("video_grid_thw", None)
+        num_video_tokens = kwargs.pop("num_video_tokens", None)
+        kwargs.pop("spatial_shapes", None)
+        kwargs.pop("pixel_attention_mask", None)
+
+        if pixel_values is not None and self._modality in ("audio", "text"):
+            raise ValueError(
+                f"Vision inputs are not available in {self._modality}-only mode. "
+                "Load with modality='omni' or modality='vision'."
+            )
+        if input_features is not None and self._modality in ("vision", "text"):
+            raise ValueError(
+                f"Audio inputs are not available in {self._modality}-only mode. "
+                "Load with modality='omni' or modality='audio'."
+            )
+
+        if (input_ids is None) ^ (inputs_embeds is not None):
+            raise ValueError("You must specify exactly one of input_ids or inputs_embeds")
+
+        if inputs_embeds is None:
+            inputs_embeds = self.get_input_embeddings()(input_ids)
+
+        # Image and video both use config.image_token_index (the processor
+        # remaps <|video_pad|> to <image>). When a single forward pass mixes
+        # both modalities, the mask matches both sets of placeholders, so we
+        # build one combined source with image features first then video
+        # features, matching the order of placeholders in input_ids.
+        all_feats = []
+        if pixel_values is not None and image_grid_thw is not None:
+            all_feats.extend(self.get_image_features(pixel_values, image_grid_thw, num_image_tokens))
+        if pixel_values_videos is not None and video_grid_thw is not None:
+            all_feats.extend(self.get_image_features(pixel_values_videos, video_grid_thw, num_video_tokens))
+        if all_feats:
+            feats = torch.cat(all_feats, dim=0).to(inputs_embeds.device, inputs_embeds.dtype)
+            mask = (input_ids == self.config.image_token_index).unsqueeze(-1).expand_as(inputs_embeds)
+            inputs_embeds = inputs_embeds.masked_scatter(mask, feats)
+
+        if input_features is not None:
+            aud = self.get_audio_features(input_features, feature_attention_mask)
+            aud_flat = aud.reshape(-1, aud.shape[-1]).to(inputs_embeds.device, inputs_embeds.dtype)
+            mask = (input_ids == self.config.audio_token_id).unsqueeze(-1).expand_as(inputs_embeds)
+            inputs_embeds = inputs_embeds.masked_scatter(mask, aud_flat)
+
+        if attention_mask is not None and attention_mask.dim() == 2:
+            dtype = inputs_embeds.dtype
+            seq_len = inputs_embeds.shape[1]
+            bidi = attention_mask[:, None, None, :].to(dtype=dtype)
+            bidi = (1.0 - bidi) * torch.finfo(dtype).min
+            attention_mask = bidi.expand(-1, -1, seq_len, -1)
+
+        out = self.language_model(
+            attention_mask=attention_mask,
+            position_ids=position_ids,
+            past_key_values=past_key_values,
+            inputs_embeds=inputs_embeds,
+            cache_position=cache_position,
+            output_hidden_states=output_hidden_states,
+        )
+
+        return BaseModelOutputWithPast(
+            last_hidden_state=self.lm_head(out[0]),
+            past_key_values=out.past_key_values,
+            hidden_states=out.hidden_states,
+            attentions=out.attentions,
+        )
+
+
+class JinaEmbeddingsV5OmniModel(PeftMixedModel):
+    config_class = JinaEmbeddingsV5OmniConfig
+
+    @classmethod
+    def register_for_auto_class(cls, auto_class="AutoModel"):
+        return PreTrainedModel.register_for_auto_class.__func__(cls, auto_class)
+
+    @classmethod
+    def from_pretrained(cls, pretrained_model_name_or_path, *args, **kwargs):
+        modality = kwargs.pop("modality", None)
+        task_kwarg = kwargs.pop("task", None)
+        config = kwargs.pop("config", None)
+        if config is None:
+            config = JinaEmbeddingsV5OmniConfig.from_pretrained(pretrained_model_name_or_path)
+        if modality is not None:
+            config.modality = modality
+        elif not hasattr(config, "modality") or config.modality is None:
+            config.modality = "omni"
+
+        default_dtype = getattr(config, "torch_dtype", None) or torch.float32
+        base_model = JinaEmbeddingsV5OmniBase.from_pretrained(
+            pretrained_model_name_or_path,
+            config=config,
+            torch_dtype=kwargs.pop("torch_dtype", kwargs.pop("dtype", default_dtype)),
+        )
+
+        if os.path.isdir(pretrained_model_name_or_path):
+            adapters_dir = os.path.join(pretrained_model_name_or_path, "adapters")
+        else:
+            cache = snapshot_download(
+                repo_id=pretrained_model_name_or_path,
+                allow_patterns=["adapters/*"],
+            )
+            adapters_dir = os.path.join(cache, "adapters")
+
+        adapter_paths = {
+            name: os.path.join(adapters_dir, name) for name in config.task_names
+        }
+
+        peft_config = PeftConfig.from_pretrained(adapter_paths["retrieval"], **kwargs)
+        model = cls(base_model, peft_config, adapter_name="retrieval")
+        model._pretrained_path = pretrained_model_name_or_path
+        for name in config.task_names:
+            model.load_adapter(adapter_paths[name], adapter_name=name, **kwargs)
+
+        model.tokenizer = AutoTokenizer.from_pretrained(
+            pretrained_model_name_or_path, trust_remote_code=True,
+        )
+        # Task precedence: kwarg > config.task (hf_overrides path) > env var > default.
+        task = task_kwarg
+        if task is None:
+            task = getattr(config, "task", None)
+        if task is None:
+            task = os.environ.get("JINA_V5_TASK")
+        if task is None:
+            task = config.task_names[0]
+        if task not in config.task_names:
+            raise ValueError(
+                f"task must be one of {config.task_names}, got '{task}'"
+            )
+        model.set_adapter(task)
+        return model
+
+    @property
+    def modality(self) -> str:
+        return self.base_model.model.modality
+
+    def set_adapter(self, adapters):
+        super().set_adapter(adapters)
+        task = adapters[0] if isinstance(adapters, list) else adapters
+        self.base_model.model.set_task(task)
+
+    def encode(
+        self,
+        texts: List[str],
+        task: str,
+        prompt_name: Optional[str] = "document",
+        truncate_dim: Optional[int] = None,
+        max_length: Optional[int] = None,
+    ) -> torch.Tensor:
+        cfg = self.base_model.model.config
+        if task not in cfg.task_names:
+            raise ValueError(f"Unknown task: {task}")
+        if prompt_name is None:
+            prompt_name = "document"
+        if prompt_name not in {"query", "document"}:
+            raise ValueError(f"Unknown prompt_name: {prompt_name}")
+
+        prefix = "Query: " if prompt_name == "query" else "Document: "
+        inputs = [f"{prefix}{t}" for t in texts]
+
+        max_length = max_length or cfg.text_config.max_position_embeddings
+        batch = self.tokenizer(
+            inputs, return_tensors="pt", padding=True, truncation=True, max_length=max_length,
+        )
+        device = next(self.parameters()).device
+        batch = {k: v.to(device) for k, v in batch.items()}
+        self.set_adapter([task])
+        self.eval()
+        with torch.no_grad():
+            hidden = self(**batch).last_hidden_state
+            mask = batch.get("attention_mask")
+            if mask is None:
+                pooled = hidden[:, -1]
+            else:
+                seq_lens = mask.sum(dim=1) - 1
+                pooled = hidden[torch.arange(hidden.shape[0], device=hidden.device), seq_lens]
+            if truncate_dim is not None:
+                pooled = pooled[:, :truncate_dim]
+            return F.normalize(pooled, p=2, dim=-1)
+
+    def embed(self, truncate_dim: Optional[int] = None, **inputs):
+        """Encode processor outputs into L2-normalized last-token embeddings.
+
+        Matryoshka: pass `truncate_dim=N` to get an N-dim unit-norm vector
+        (truncation is applied before L2-normalization).
+        """
+        attention_mask = inputs.get("attention_mask", None)
+        self.eval()
+        with torch.no_grad():
+            out = self(**inputs)
+        hidden = out.last_hidden_state
+        if attention_mask is not None and attention_mask.dim() == 2:
+            idx = attention_mask.sum(dim=1) - 1
+        else:
+            idx = torch.full(
+                (hidden.shape[0],), hidden.shape[1] - 1,
+                device=hidden.device, dtype=torch.long,
+            )
+        pooled = hidden[torch.arange(hidden.shape[0], device=hidden.device), idx]
+        if truncate_dim is not None:
+            pooled = pooled[:, :truncate_dim]
+        return torch.nn.functional.normalize(pooled, dim=-1)
+
+
+
+# ---------------------------------------------------------------------------
+# vLLM registration (side-effect on module import).
+#
+# Triggered via config.json "auto_map.AutoConfig" -> this module.
+# HF / sentence-transformers path unaffected: any failure is silently swallowed
+# so that pure transformers users never see a vLLM error.
+# ---------------------------------------------------------------------------
+
+def _register_vllm() -> None:
+    # All vLLM references are resolved via importlib so transformers'
+    # static check_imports does NOT flag vllm as a required dependency.
+    # Pure-HF / sentence-transformers usage is unaffected.
+    #
+    # When loaded via transformers' `trust_remote_code=True`, only the
+    # modeling_*.py referenced in auto_map is fetched into the
+    # transformers_modules cache — sibling vLLM adapter files are NOT.
+    # We pull them from HF Hub before registering; otherwise vLLM falls
+    # back to its transformers backend (wrong attention semantics) and
+    # multi-request batches collapse.
+    import importlib.util as _iu
+    if _iu.find_spec("vllm") is None:
+        return
+    try:
+        import os
+        import sys
+        import importlib
+        import inspect
+        import shutil
+
+        pkg = __package__ or ""
+        current_dir = os.path.dirname(os.path.abspath(__file__))
+        siblings = ("vllm_llava_eurobert_audio", "vllm_jina_v5_omni")
+
+        for sibling_name in siblings:
+            sibling_path = os.path.join(current_dir, sibling_name + ".py")
+            if os.path.exists(sibling_path):
+                continue
+            parts = pkg.split(".")
+            if len(parts) < 4 or parts[0] != "transformers_modules":
+                continue
+            from huggingface_hub import hf_hub_download
+            repo_name = parts[2].replace("_hyphen_", "-").replace("_dot_", ".")
+            repo_id = f"{parts[1]}/{repo_name}"
+            downloaded = hf_hub_download(
+                repo_id=repo_id,
+                filename=sibling_name + ".py",
+                revision=parts[3],
+            )
+            shutil.copy(downloaded, sibling_path)
+
+        os.environ.setdefault("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
+
+        _kvc = importlib.import_module("vllm.v1.core.kv_cache_coordinator")
+        _orig = _kvc.get_kv_cache_coordinator
+        _NoPrefix = _kvc.KVCacheCoordinatorNoPrefixCache
+
+        _orig_sig = inspect.signature(_orig)
+        _noprefix_sig = inspect.signature(_NoPrefix)
+
+        def _patched(kv_cache_config, max_model_len, *args, **kwargs):
+            if len(kv_cache_config.kv_cache_groups) == 0:
+                bound = _orig_sig.bind(kv_cache_config, max_model_len, *args, **kwargs)
+                return _NoPrefix(**{
+                    name: bound.arguments[name]
+                    for name in _noprefix_sig.parameters
+                    if name in bound.arguments
+                })
+            return _orig(kv_cache_config, max_model_len, *args, **kwargs)
+
+        _kvc.get_kv_cache_coordinator = _patched
+
+        # Make sibling-dir importable from a fresh subprocess too — vLLM's
+        # inspect_model_cls runs in a child Python process that doesn't
+        # inherit our sys.modules. Without this on PYTHONPATH the
+        # string-spec model registration below can't be resolved.
+        if current_dir not in sys.path:
+            sys.path.insert(0, current_dir)
+        existing = os.environ.get("PYTHONPATH", "")
+        if current_dir not in existing.split(os.pathsep):
+            os.environ["PYTHONPATH"] = (
+                current_dir if not existing else current_dir + os.pathsep + existing
+            )
+
+        if pkg:
+            _lla = importlib.import_module(".vllm_llava_eurobert_audio", package=pkg)
+            _omni = importlib.import_module(".vllm_jina_v5_omni", package=pkg)
+        else:
+            _lla = importlib.import_module("vllm_llava_eurobert_audio")
+            _omni = importlib.import_module("vllm_jina_v5_omni")
+        _ = _lla.LlavaEuroBertAudioForVLLMEmbedding  # keep reference
+
+        ModelRegistry = importlib.import_module(
+            "vllm.model_executor.models"
+        ).ModelRegistry
+        # String spec ("module:Class") — survives vLLM's cloudpickle-into-
+        # subprocess flow because the child re-imports by name. Passing the
+        # class object directly registers __module__ as the qualified
+        # transformers_modules.jinaai.<...> path, which the subprocess
+        # can't resolve without HF's dynamic-module setup.
+        ModelRegistry.register_model(
+            "JinaEmbeddingsV5OmniModel",
+            "vllm_jina_v5_omni:JinaV5OmniForVLLMEmbedding",
+        )
+    except Exception as e:
+        import warnings
+        warnings.warn(
+            f"jina-embeddings-v5-omni base: vLLM registration failed "
+            f"({type(e).__name__}: {e}); embeddings will fall back to "
+            f"vLLM's generic transformers backend (wrong tensor layout).",
+            stacklevel=2,
+        )
+
+
+_register_vllm()

modeling_llava_eurobert_audio.py

@@ -0,0 +1,400 @@
+"""
+LlavaEuroBertAudioForEmbedding: Qwen3VL vision + Qwen2.5-Omni audio + EuroBERT text.
+
+Architecture:
+  - Vision: Qwen3VLVisionModel (with RoPE, 3D Conv3d patch embed, all layers)
+  - Merger: PretrainedMerger (top-level, NOT inside vision_tower)
+  - Audio: Qwen2_5OmniAudioEncoder (Qwen2.5-Omni) + Linear projector
+  - Text: LlamaModel (EuroBERT, bidirectional)
+  - LM head: Identity (embedding model, no vocab projection)
+
+Modality loading:
+  model = AutoModel.from_pretrained(path, trust_remote_code=True, modality="omni")   # all components (default)
+  model = AutoModel.from_pretrained(path, trust_remote_code=True, modality="vision") # no audio tower/projector
+  model = AutoModel.from_pretrained(path, trust_remote_code=True, modality="audio")  # no vision tower/merger
+"""
+
+from typing import List, Optional, Union
+
+import torch
+import torch.nn as nn
+from transformers import LlamaConfig, PreTrainedModel, PretrainedConfig
+from transformers.modeling_outputs import BaseModelOutputWithPast
+from transformers.models.llama.modeling_llama import LlamaModel
+from transformers.models.qwen3_vl.configuration_qwen3_vl import Qwen3VLVisionConfig
+from transformers.models.qwen3_vl.modeling_qwen3_vl import Qwen3VLVisionModel
+from transformers.models.qwen2_5_omni.configuration_qwen2_5_omni import Qwen2_5OmniAudioEncoderConfig
+from transformers.models.qwen2_5_omni.modeling_qwen2_5_omni import Qwen2_5OmniAudioEncoder
+
+
+_VALID_MODALITIES = ("omni", "vision", "audio", "text")
+
+
+class PretrainedMerger(nn.Module):
+    def __init__(self, hidden_size, out_hidden_size, spatial_merge_size=2):
+        super().__init__()
+        self.hidden_size = hidden_size * (spatial_merge_size**2)
+        self.norm = nn.LayerNorm(hidden_size, eps=1e-6)
+        self.linear_fc1 = nn.Linear(self.hidden_size, self.hidden_size)
+        self.act = nn.GELU()
+        self.linear_fc2 = nn.Linear(self.hidden_size, out_hidden_size)
+
+    def forward(self, x):
+        x = self.norm(x)
+        x = x.view(-1, self.hidden_size)
+        x = self.linear_fc2(self.act(self.linear_fc1(x)))
+        return x
+
+
+class LlavaEuroBertAudioConfig(PretrainedConfig):
+    model_type = "llava_eurobert_audio"
+
+    def __init__(
+        self,
+        vision_config=None,
+        text_config=None,
+        audio_config=None,
+        image_token_index=None,
+        audio_token_id=None,
+        audio_start_token_id=None,
+        audio_end_token_id=None,
+        projector_hidden_act="gelu",
+        tie_word_embeddings=False,
+        modality="omni",
+        **kwargs,
+    ):
+        if isinstance(vision_config, dict):
+            vision_config = PretrainedConfig(**vision_config)
+        self.vision_config = vision_config or PretrainedConfig()
+        if isinstance(text_config, dict):
+            text_config = PretrainedConfig(**text_config)
+        self.text_config = text_config or PretrainedConfig()
+        if isinstance(audio_config, dict):
+            audio_config = PretrainedConfig(**audio_config)
+        self.audio_config = audio_config or PretrainedConfig()
+        self.image_token_index = image_token_index
+        self.audio_token_id = audio_token_id
+        self.audio_start_token_id = audio_start_token_id
+        self.audio_end_token_id = audio_end_token_id
+        self.projector_hidden_act = projector_hidden_act
+        if modality not in _VALID_MODALITIES:
+            raise ValueError(f"modality must be one of {_VALID_MODALITIES}, got '{modality}'")
+        self.modality = modality
+        super().__init__(tie_word_embeddings=tie_word_embeddings, **kwargs)
+
+    def get_text_config(self, **kwargs):
+        return self.text_config
+
+
+class LlavaEuroBertAudioForEmbedding(PreTrainedModel):
+    config_class = LlavaEuroBertAudioConfig
+    supports_gradient_checkpointing = True
+    _supports_sdpa = True
+    _supports_flash_attn_2 = True
+    _supports_attention_backend = True
+    _tied_weights_keys = []
+    _keys_to_ignore_on_load_missing = ["lm_head.weight"]
+    _keys_to_ignore_on_load_unexpected = []
+
+    def __init__(self, config: LlavaEuroBertAudioConfig):
+        super().__init__(config)
+
+        modality = getattr(config, "modality", "omni")
+        if modality not in _VALID_MODALITIES:
+            raise ValueError(f"modality must be one of {_VALID_MODALITIES}, got '{modality}'")
+        self._modality = modality
+
+        vision_cfg = config.vision_config
+        if not isinstance(vision_cfg, Qwen3VLVisionConfig):
+            if hasattr(vision_cfg, "to_dict"):
+                d = vision_cfg.to_dict()
+            else:
+                d = dict(vision_cfg)
+            d.pop("model_type", None)
+            d.pop("transformers_version", None)
+            vision_cfg = Qwen3VLVisionConfig(**d)
+
+        vision_cfg.deepstack_visual_indexes = []
+        spatial_merge_size = getattr(vision_cfg, "spatial_merge_size", 2)
+
+        text_cfg = config.text_config
+        if not isinstance(text_cfg, LlamaConfig):
+            txt_dict = text_cfg.to_dict() if hasattr(text_cfg, 'to_dict') else dict(text_cfg)
+            _saved_attn_impl = getattr(text_cfg, "_attn_implementation", None)
+            text_cfg = LlamaConfig(**txt_dict)
+            if _saved_attn_impl is not None:
+                text_cfg._attn_implementation = _saved_attn_impl
+        text_hidden = text_cfg.hidden_size
+
+        self._spatial_merge_size = spatial_merge_size
+        self._vision_hidden_size = getattr(vision_cfg, "hidden_size", 768)
+
+        if modality not in ("audio", "text"):
+            self.vision_tower = Qwen3VLVisionModel(vision_cfg)
+            self.vision_tower.merger = nn.Identity()
+            self.vision_tower.deepstack_merger_list = nn.ModuleList()
+            self.vision_tower.deepstack_visual_indexes = []
+            self.merger = PretrainedMerger(
+                vision_cfg.hidden_size, text_hidden, spatial_merge_size
+            )
+
+        self.multi_modal_projector = nn.Identity()
+        self.language_model = LlamaModel(text_cfg)
+        self.lm_head = nn.Identity()
+
+        for layer in self.language_model.layers:
+            layer.self_attn.is_causal = False
+
+        if modality not in ("vision", "text"):
+            aud_cfg = config.audio_config
+            aud_dict = aud_cfg.to_dict() if hasattr(aud_cfg, 'to_dict') else aud_cfg
+            audio_encoder_config = Qwen2_5OmniAudioEncoderConfig(**aud_dict)
+            self.audio_tower = Qwen2_5OmniAudioEncoder(audio_encoder_config)
+            output_dim = aud_dict.get('output_dim', 3584)
+            self.audio_projector = nn.Linear(output_dim, text_hidden)
+
+        ignore = []
+        if modality in ("audio", "text"):
+            ignore.extend([r"^vision_tower\.", r"^merger\."])
+        if modality in ("vision", "text"):
+            ignore.extend([r"^audio_tower\.", r"^audio_projector\."])
+        if ignore:
+            self._keys_to_ignore_on_load_unexpected = ignore
+
+        self.post_init()
+
+    @property
+    def modality(self) -> str:
+        return self._modality
+
+    def get_input_embeddings(self):
+        return self.language_model.embed_tokens
+
+    def set_input_embeddings(self, value):
+        self.language_model.embed_tokens = value
+
+    def get_output_embeddings(self):
+        return None
+
+    def get_image_features(
+        self,
+        pixel_values: torch.FloatTensor,
+        image_grid_thw: torch.LongTensor,
+        num_image_tokens: Optional[int] = None,
+    ) -> List[torch.Tensor]:
+        if self._modality in ("audio", "text"):
+            raise ValueError(
+                f"Vision inputs are not available in {self._modality}-only mode. "
+                "Load with modality='omni' or modality='vision'."
+            )
+
+        vision_output = self.vision_tower(
+            hidden_states=pixel_values, grid_thw=image_grid_thw
+        )
+        if isinstance(vision_output, tuple):
+            raw_hidden = vision_output[0]
+        elif hasattr(vision_output, "pooler_output") and vision_output.pooler_output is not None:
+            raw_hidden = vision_output.pooler_output
+        else:
+            raw_hidden = vision_output[0]
+
+        image_features = self.merger(raw_hidden)
+
+        merge_sq = self._spatial_merge_size ** 2
+        split_sizes = (image_grid_thw.prod(-1) // merge_sq).tolist()
+        return list(torch.split(image_features, split_sizes))
+
+    def get_audio_features(
+        self,
+        input_features: torch.FloatTensor,
+        feature_attention_mask: Optional[torch.LongTensor] = None,
+    ) -> torch.Tensor:
+        if self._modality in ("vision", "text"):
+            raise ValueError(
+                f"Audio inputs are not available in {self._modality}-only mode. "
+                "Load with modality='omni' or modality='audio'."
+            )
+
+        batch_size = input_features.shape[0]
+        if batch_size > 1:
+            # Serialize per-sample so the packed-frames GEMM shape stays invariant
+            # across batch sizes. Makes batched audio bit-exact to B=1 in bf16,
+            # and is substantially faster for B>=16 because B=1 hits a
+            # well-optimized kernel while the packed-B=N path thrashes on a
+            # (total_frames)^2 sdpa matrix.
+            outs = [
+                self.get_audio_features(
+                    input_features[i : i + 1],
+                    feature_attention_mask[i : i + 1] if feature_attention_mask is not None else None,
+                )
+                for i in range(batch_size)
+            ]
+            return torch.cat(outs, dim=0)
+        if feature_attention_mask is not None:
+            feature_lens = feature_attention_mask.sum(-1).long()
+            packed = input_features.permute(0, 2, 1)[feature_attention_mask.bool()].permute(1, 0)
+        else:
+            feature_lens = torch.full(
+                (batch_size,), input_features.shape[2],
+                device=input_features.device, dtype=torch.long,
+            )
+            packed = input_features.transpose(1, 2).reshape(-1, input_features.shape[1]).T
+        aftercnn_lens, _ = self.audio_tower._get_feat_extract_output_lengths(feature_lens)
+        audio_output = self.audio_tower(
+            packed, feature_lens=feature_lens, aftercnn_lens=aftercnn_lens,
+        )
+        return self.audio_projector(audio_output.last_hidden_state)
+
+    def forward(
+        self,
+        input_ids: Optional[torch.LongTensor] = None,
+        pixel_values: Optional[torch.FloatTensor] = None,
+        attention_mask: Optional[torch.Tensor] = None,
+        position_ids: Optional[torch.LongTensor] = None,
+        past_key_values=None,
+        inputs_embeds: Optional[torch.FloatTensor] = None,
+        input_features: Optional[torch.FloatTensor] = None,
+        feature_attention_mask: Optional[torch.LongTensor] = None,
+        cache_position: Optional[torch.LongTensor] = None,
+        output_hidden_states: Optional[bool] = None,
+        **kwargs,
+    ):
+        image_grid_thw = kwargs.pop("image_grid_thw", None)
+        num_image_tokens = kwargs.pop("num_image_tokens", None)
+        kwargs.pop("spatial_shapes", None)
+        kwargs.pop("pixel_attention_mask", None)
+
+        if pixel_values is not None and self._modality in ("audio", "text"):
+            raise ValueError(
+                f"Vision inputs are not available in {self._modality}-only mode. "
+                "Load with modality='omni' or modality='vision'."
+            )
+        if input_features is not None and self._modality in ("vision", "text"):
+            raise ValueError(
+                f"Audio inputs are not available in {self._modality}-only mode. "
+                "Load with modality='omni' or modality='audio'."
+            )
+
+        if (input_ids is None) ^ (inputs_embeds is not None):
+            raise ValueError(
+                "You must specify exactly one of input_ids or inputs_embeds"
+            )
+
+        if inputs_embeds is None:
+            inputs_embeds = self.get_input_embeddings()(input_ids)
+
+        if pixel_values is not None and image_grid_thw is not None:
+            image_features = self.get_image_features(
+                pixel_values=pixel_values,
+                image_grid_thw=image_grid_thw,
+                num_image_tokens=num_image_tokens,
+            )
+            image_features = torch.cat(image_features, dim=0).to(
+                inputs_embeds.device, inputs_embeds.dtype
+            )
+            special_image_mask = (
+                (input_ids == self.config.image_token_index)
+                .unsqueeze(-1)
+                .expand_as(inputs_embeds)
+            )
+            inputs_embeds = inputs_embeds.masked_scatter(
+                special_image_mask, image_features
+            )
+
+        if input_features is not None:
+            audio_embeds = self.get_audio_features(
+                input_features, feature_attention_mask
+            )
+            audio_embeds_flat = audio_embeds.reshape(
+                -1, audio_embeds.shape[-1]
+            ).to(inputs_embeds.device, inputs_embeds.dtype)
+            audio_mask = (
+                (input_ids == self.config.audio_token_id)
+                .unsqueeze(-1)
+                .expand_as(inputs_embeds)
+            )
+            inputs_embeds = inputs_embeds.masked_scatter(
+                audio_mask, audio_embeds_flat
+            )
+
+        if attention_mask is not None and attention_mask.dim() == 2:
+            dtype = inputs_embeds.dtype
+            seq_len = inputs_embeds.shape[1]
+            bidi_mask = attention_mask[:, None, None, :].to(dtype=dtype)
+            bidi_mask = (1.0 - bidi_mask) * torch.finfo(dtype).min
+            attention_mask = bidi_mask.expand(-1, -1, seq_len, -1)
+
+        # vLLM's transformers backend passes `return_dict=False` + `attention_instances`.
+        # Force dict-style output internally, and forward remaining kwargs so the
+        # vllm attention hook receives its `attention_instances` dict.
+        kwargs.pop("return_dict", None)
+        outputs = self.language_model(
+            attention_mask=attention_mask,
+            position_ids=position_ids,
+            past_key_values=past_key_values,
+            inputs_embeds=inputs_embeds,
+            cache_position=cache_position,
+            output_hidden_states=output_hidden_states,
+            return_dict=True,
+            **kwargs,
+        )
+
+        hidden_states = outputs[0]
+        logits = self.lm_head(hidden_states)
+
+        return BaseModelOutputWithPast(
+            last_hidden_state=logits,
+            past_key_values=outputs.past_key_values,
+            hidden_states=outputs.hidden_states,
+            attentions=outputs.attentions,
+        )
+
+
+def _register_vllm() -> None:
+    import importlib.util as _iu
+    if _iu.find_spec("vllm") is None:
+        return
+    try:
+        import os, sys, importlib, shutil
+        pkg = __package__ or ""
+        current_dir = os.path.dirname(os.path.abspath(__file__))
+        sibling_name = "vllm_llava_eurobert_audio"
+        sibling_path = os.path.join(current_dir, sibling_name + ".py")
+        if not os.path.exists(sibling_path):
+            parts = pkg.split(".")
+            if len(parts) >= 4 and parts[0] == "transformers_modules":
+                from huggingface_hub import hf_hub_download
+                repo_name = parts[2].replace("_hyphen_", "-").replace("_dot_", ".")
+                repo_id = f"{parts[1]}/{repo_name}"
+                downloaded = hf_hub_download(
+                    repo_id=repo_id,
+                    filename=sibling_name + ".py",
+                    revision=parts[3],
+                )
+                shutil.copy(downloaded, sibling_path)
+        if current_dir not in sys.path:
+            sys.path.insert(0, current_dir)
+        existing = os.environ.get("PYTHONPATH", "")
+        if current_dir not in existing.split(os.pathsep):
+            os.environ["PYTHONPATH"] = (
+                current_dir if not existing else current_dir + os.pathsep + existing
+            )
+        if pkg:
+            _lla = importlib.import_module("." + sibling_name, package=pkg)
+        else:
+            _lla = importlib.import_module(sibling_name)
+        from vllm import ModelRegistry
+        ModelRegistry.register_model(
+            "LlavaEuroBertAudioForEmbedding",
+            _lla.LlavaEuroBertAudioForVLLMEmbedding,
+        )
+    except Exception as e:
+        import warnings
+        warnings.warn(
+            f"jina-embeddings-v5-omni nano: vLLM registration failed "
+            f"({type(e).__name__}: {e}); falling back to Transformers backend.",
+            stacklevel=2,
+        )
+
+
+_register_vllm()

onnx/audio_model.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9a39f40bf85201779d7af2a03b9c1467a6d998261ed976d27e2d4f955e4c1cca
+size 8820515

onnx/audio_model.onnx.data

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:48d0d710789d9d84b13e676a4302260f1b83cd968692fedd1d3c47d41e0dbb9e
+size 3391207424

onnx/audio_model_fp16.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:957dd6285e916c9027095c662a7a06c6d18b6a53be3cdf0c8b9399f5897f2406
+size 4983121

onnx/audio_model_fp16.onnx_data

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:59cbc12ea47bfa69f72b0b2546d6a04e5c7b1434efd2a6258d42417f3f605900
+size 1695603712

onnx/audio_model_q4f16.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:70f8898f6ff52e89b03385ce82f6ce393d157e8bdfa03f5af521ae12fbebd82e
+size 5030426

onnx/audio_model_q4f16.onnx_data

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8f769529695b56378267238c2300525d8a4e6476710ecf626f9afc50df70bf5d
+size 1460034560

onnx/text_model.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f47feec56eea10e323e1faeb8ab83a0feaacaf6256670bc851187aa4a2a68752
+size 2008672

onnx/text_model.onnx.data

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cde38ab39a4d39d7133b28a6989fdaf21dd717711028e65cd79b1ece50c503a5
+size 847130624

onnx/text_model_fp16.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f59419cb1d33e7bf66e11b336c618b61f1430349bdbd09180c49f7c934956cfe
+size 125318

onnx/text_model_fp16.onnx_data

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c02e2e7503b898e3c3f5f7e37aeee96415a6e33f9fd1d3813c69f95ea2172f5b
+size 423538176

onnx/text_model_q4f16.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1eec254d70f37bef242263514190ec76ea792546c520aa83b005639b5a36b817
+size 155687

onnx/text_model_q4f16.onnx_data

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cb70fcc7788a81338fce0cfde62ff2c81559e0e6ddd01004f2d73d8488869aab
+size 262516224

onnx/vision_model.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5af7a25cbad96b514d8b6f94bb6d56f28cb23ca5a8dffefc491c5e1a2388edd9
+size 696459

onnx/vision_model.onnx.data

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ceb559e9e7ff81fa03dee7028fc28c7df18f3f4cdf284b7014960272d9b3f393
+size 1246301184

onnx/vision_model_fp16.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7ef6d300523d024a7fa764685ef7e975ff9b1e291bdf0cf4bcde84ca322fd851
+size 496172

onnx/vision_model_fp16.onnx_data

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:77c4b4e8000e1548fa7a2268360c855d00871b5bbed92b86b364be4af901b48e
+size 621516032

onnx/vision_model_q4f16.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:05cdfefc21ebe04cf5af7a60939f51cab8d1e8250b4853d940b818f1f8e9e4f0
+size 530266

onnx/vision_model_q4f16.onnx_data

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eb4b0d512bc7e92ed5e9f59fca75fd37d51e5b21b59d0e0e85b9462178063ef1
+size 460494080

preprocessor_config.json

@@ -0,0 +1,24 @@
+{
+  "do_convert_rgb": true,
+  "do_normalize": true,
+  "do_rescale": true,
+  "do_resize": true,
+  "image_mean": [
+    0.5,
+    0.5,
+    0.5
+  ],
+  "image_processor_type": "Qwen2VLImageProcessor",
+  "image_std": [
+    0.5,
+    0.5,
+    0.5
+  ],
+  "merge_size": 2,
+  "patch_size": 16,
+  "resample": 3,
+  "rescale_factor": 0.00392156862745098,
+  "temporal_patch_size": 2,
+  "min_pixels": 262144,
+  "max_pixels": 1310720
+}

processor_config.json

@@ -0,0 +1,68 @@
+{
+  "processor_class": "LlavaEuroBertProcessor",
+  "auto_map": {
+    "AutoProcessor": "processing_llava_eurobert.LlavaEuroBertProcessor"
+  },
+  "image_processor": {
+    "image_processor_type": "Qwen2VLImageProcessorFast",
+    "do_convert_rgb": true,
+    "do_normalize": true,
+    "do_rescale": true,
+    "do_resize": true,
+    "image_mean": [
+      0.5,
+      0.5,
+      0.5
+    ],
+    "image_std": [
+      0.5,
+      0.5,
+      0.5
+    ],
+    "min_pixels": 262144,
+    "max_pixels": 1310720,
+    "size": {
+      "longest_edge": 16777216,
+      "shortest_edge": 65536
+    },
+    "merge_size": 2,
+    "patch_size": 16,
+    "resample": 3,
+    "rescale_factor": 0.00392156862745098,
+    "temporal_patch_size": 2
+  },
+  "video_processor": {
+    "video_processor_type": "Qwen3VLVideoProcessor",
+    "do_convert_rgb": true,
+    "do_normalize": true,
+    "do_rescale": true,
+    "do_resize": true,
+    "do_sample_frames": true,
+    "fps": 2,
+    "image_mean": [
+      0.5,
+      0.5,
+      0.5
+    ],
+    "image_std": [
+      0.5,
+      0.5,
+      0.5
+    ],
+    "max_frames": 768,
+    "min_frames": 4,
+    "merge_size": 2,
+    "patch_size": 16,
+    "resample": 3,
+    "rescale_factor": 0.00392156862745098,
+    "size": {
+      "longest_edge": 25165824,
+      "shortest_edge": 4096
+    },
+    "temporal_patch_size": 2
+  },
+  "image_token": "<image>",
+  "num_additional_image_tokens": 0,
+  "patch_size": null,
+  "vision_feature_select_strategy": null
+}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b8135ff0f019acbce7c4b93d1fb24cb8325b5fb0d76c57bbde8101a73cba7fa9
+size 17211089

tokenizer_config.json

@@ -0,0 +1,23 @@
+{
+  "backend": "tokenizers",
+  "bos_token": "<|begin_of_text|>",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|end_of_text|>",
+  "is_local": false,
+  "mask_token": "<|mask|>",
+  "max_length": null,
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_to_multiple_of": null,
+  "pad_token": "<|pad|>",
+  "pad_token_type_id": 0,
+  "padding_side": "right",
+  "processor_class": "LlavaEuroBertProcessor",
+  "tokenizer_class": "TokenizersBackend",
+  "auto_map": {
+    "AutoProcessor": "processing_llava_eurobert.LlavaEuroBertProcessor"
+  }
+}

video_preprocessor_config.json

@@ -0,0 +1,29 @@
+{
+  "do_sample_frames": false,
+  "fps": 2,
+  "min_frames": 4,
+  "max_frames": 32,
+  "size": {
+    "longest_edge": 12845056,
+    "shortest_edge": 262144
+  },
+  "patch_size": 16,
+  "merge_size": 2,
+  "temporal_patch_size": 2,
+  "do_convert_rgb": true,
+  "do_normalize": true,
+  "do_rescale": true,
+  "do_resize": true,
+  "image_mean": [
+    0.5,
+    0.5,
+    0.5
+  ],
+  "image_std": [
+    0.5,
+    0.5,
+    0.5
+  ],
+  "rescale_factor": 0.00392156862745098,
+  "resample": 3
+}