Text Generation
Safetensors
Transformers
English
Russian
Ukrainian
vllm
qwen3_5
image-text-to-text
long-context
1m-context
million-token-context
context-extension
needle-in-a-haystack
retrieval
retrieval-heads
consumer-gpu
single-gpu
rtx-5090
rtx-4090
quantization
nvfp4
3-bit
fp8
int8
kv-cache-quantization
turboquant
3-bit-kv-cache
hybrid-architecture
linear-attention
gated-deltanet
state-space
gqa
multimodal
vision-language
conversational
agentic
coding
roleplay
russian
ukrainian
custom_code
measured-benchmarks
Eval Results (legacy)
8-bit precision
compressed-tensors
| { | |
| "altay_architecture_id": "LOMONOSOV_ZENIT_ALTAY_72M_SKV_V1", | |
| "altay_extra_recurrent_slots": 6, | |
| "altay_kv_layout": { | |
| "actual_cache_units": 70, | |
| "allocated_cache_slots": 72, | |
| "attention_padding_cache_slots": 0, | |
| "enabled": true, | |
| "full_attention_cache_units": 16, | |
| "gdn_cache_units": 54, | |
| "gdn_padding_cache_slots": 2, | |
| "group_size": 8, | |
| "kv_content_modified": false, | |
| "legacy_allocated_cache_slots": 80, | |
| "padding_cache_slots": 2, | |
| "repair_id": "ALTAY72_KV_ATTENTION_EXACT_G8_V2", | |
| "schema": "lomonosov_zenit_altay_kv_layout_v1", | |
| "weights_modified": false | |
| }, | |
| "altay_lm_head_fp8_postprocess": { | |
| "format": "compressed-tensors float-quantized", | |
| "input_activation_strategy": "dynamic token FP8", | |
| "source_weight_dtype": "torch.bfloat16", | |
| "weight_dtype": "torch.float8_e4m3fn", | |
| "weight_scale_dtype": "torch.bfloat16", | |
| "weight_strategy": "channel" | |
| }, | |
| "altay_logical_layers": 72, | |
| "altay_physical_kv_units": 64, | |
| "architectures": [ | |
| "LomonosovZenitAltayForConditionalGeneration" | |
| ], | |
| "auto_map": { | |
| "AutoModel": "modeling_altay.LomonosovZenitAltayModel", | |
| "AutoModelForImageTextToText": "modeling_altay.LomonosovZenitAltayForConditionalGeneration", | |
| "AutoModelForCausalLM": "modeling_altay.LomonosovZenitAltayForConditionalGeneration" | |
| }, | |
| "dtype": "bfloat16", | |
| "image_token_id": 248056, | |
| "language_model_only": false, | |
| "lomonosov_serving_profiles": { | |
| "default_profile": "raw_1010k", | |
| "note": "The declared 1,010,000-token context is the default. When the GPU cannot hold it, the widest profile that fits is selected and the substitution is logged. An explicit --max-model-len, or ZENIT_SERVING_PROFILE, always wins.", | |
| "profiles": { | |
| "long_262k": { | |
| "description": "Compatibility window for cards that cannot hold the million, and for callers who ask for a shorter context explicitly.", | |
| "enforce_eager": true, | |
| "kv_cache_dtype": "turboquant_3bit_nc", | |
| "kv_cache_memory_bytes": 3600000000, | |
| "max_model_len": 262144, | |
| "max_num_batched_tokens": 4096, | |
| "max_num_seqs": 1, | |
| "noosphere_mode": "raw", | |
| "provenance": "prefill measured on RTX 5090: 147.9 s for 262,144 tokens (1772 tok/s) with a pinned arena, GPU at 99%. async_scheduling deliberately left unset: measured 168.9 s off against 172.1 s on at this length, a difference inside run-to-run noise. Арена задана явно: 3,600,000,000 байт, то есть 281,579 токенов ёмкости при окне 262,144 - запас есть, как и у миллиона. Это не только про запас. Без явной арены vLLM гоняет свой профилирующий проход, и на нём запуск падал в humming_gemm с CUDA_ERROR_INVALID_VALUE: ядро просило больше разделяемой памяти, чем разрешает карта. Все наши замеры этого не видели ровно потому, что всегда задавали арену и профилирование пропускалось.", | |
| "requires_free_bytes": 23972315586, | |
| "skip_mm_profiling": true, | |
| "gpu_memory_utilization": 0.95, | |
| "position_plan": { | |
| "stages": [ | |
| { | |
| "from": 65536, | |
| "group": 2, | |
| "first_pair": 15, | |
| "last_pair": 21 | |
| } | |
| ], | |
| "measured": "Замерено 26.07.2026 на 263 140 токенах, 48 закладок, локальная RTX 5090: релизный план 30/48, эта полоса 35/48. Перебор старта полосы: 65 536 → 35/48 лучше, чем 166 867 → 29/48. Ступень начинается с 65 536, поэтому промпты короче этого не затронуты вовсе." | |
| } | |
| }, | |
| "long_393k": { | |
| "description": "Window for 24 GiB cards. Measured on an RTX 4090: with the engine at 0.95 the KV arena holds 452,522 tokens, so this window fits with room to spare.", | |
| "enforce_eager": true, | |
| "gpu_memory_utilization": 0.95, | |
| "kv_cache_dtype": "turboquant_3bit_nc", | |
| "kv_cache_memory_bytes": 5200000000, | |
| "max_model_len": 393216, | |
| "max_num_batched_tokens": 4096, | |
| "max_num_seqs": 1, | |
| "noosphere_mode": "raw", | |
| "provenance": "sized from the RTX 4090 run, where the engine reported a KV arena of 452,522 tokens at gpu_memory_utilization 0.95 with 480 MB still free. The window is set below that capacity on purpose, and the stated requirement uses the more conservative of two overhead estimates (2.04 GiB against the 1.72 GiB the same run implies). Арена задана явно: 5,200,000,000 байт, то есть 406,726 токенов ёмкости при окне 393,216 - запас есть, как и у миллиона. Это не только про запас. Без явной арены vLLM гоняет свой профилирующий проход, и на нём запуск падал в humming_gemm с CUDA_ERROR_INVALID_VALUE: ядро просило больше разделяемой памяти, чем разрешает карта. Все наши замеры этого не видели ровно потому, что всегда задавали арену и профилирование пропускалось.", | |
| "requires_free_bytes": 25572315586, | |
| "skip_mm_profiling": true, | |
| "position_plan": { | |
| "stages": [ | |
| { | |
| "from": 131072, | |
| "group": 2, | |
| "first_pair": 15, | |
| "last_pair": 21 | |
| }, | |
| { | |
| "from": 196608, | |
| "group": 4, | |
| "first_pair": 0, | |
| "last_pair": 31 | |
| } | |
| ], | |
| "measured": "Замерено 26.07.2026 на 401 140 токенах, 48 закладок, арендованная RTX 5090: релизный план 28/48, эта карта 37/48 (снято дважды). Перебор старта полосы дал внутренний оптимум: 65 536 → 31/48, 98 304 → 34/48, 131 072 → 37/48, 163 840 → 33/48. Вторая ступень нужна, чтобы отображённая позиция последнего токена окна осталась внутри обученных 262 144 (f = 247 741)." | |
| } | |
| }, | |
| "raw_1010k": { | |
| "cuda_alloc_conf": "expandable_segments:True", | |
| "description": "The full declared 1,010,000-token context: one literal history, no eviction, no summary, no retrieval substitution.", | |
| "enable_prefix_caching": false, | |
| "enforce_eager": true, | |
| "gpu_memory_utilization": 0.95, | |
| "kv_cache_dtype": "turboquant_3bit_nc", | |
| "kv_cache_memory_bytes": 13000000000, | |
| "max_model_len": 1010001, | |
| "max_num_batched_tokens": 4096, | |
| "max_num_seqs": 1, | |
| "noosphere_mode": "raw", | |
| "position_plan": { | |
| "stages": [ | |
| { | |
| "from": 118700, | |
| "group": 2, | |
| "first_pair": 15, | |
| "last_pair": 21 | |
| }, | |
| { | |
| "from": 212000, | |
| "group": 16, | |
| "first_pair": 0, | |
| "last_pair": 31 | |
| } | |
| ], | |
| "measured": "Замерено 26.07.2026 на 1 006 140 токенах, 48 закладок, арендованная RTX 5090: прежний план 1/48 = 2.1%, этот 9/48 = 18.8%. Прежний промахивался мимо обученного окна на 71% (последний токен уезжал в 447 858). Условие «влезть» P + (L-P)//g <= 262 144 при этой длине допускает только группу 8 или 16; группа 4 требовала бы верного префикса не больше 14 525. Здесь f = 261 633. ВАЖНО: 18.8% — это не окно поиска. Одна закладка из пяти. Миллион остаётся возможностью префилла, и в карточке это сказано прямо." | |
| }, | |
| "provenance": "The arena is deliberately larger than the window. Pinning it to exactly 1,010,001 tokens - which is what the earlier profile did - leaves the scheduler no free blocks, and vLLM's engine loop then spends 69 to 72 per cent of its wall clock in an idle branch with the GPU at 13 to 21 per cent. The trigger was isolated by a controlled pair at 262,144 tokens: a roomy arena gave 147.9 s at 99 per cent GPU, an arena sized exactly to the window was still running after 385 s at 0 per cent. With spare blocks the full million-token prefill runs at 54 to 99 per cent utilisation. Chunk 1024 is the largest that fits beside this arena; 8192 and 16384 run out of memory during vLLM's profiling forward pass. Чанк 4096, не 1024: измерено 1318 с против 1558 на миллионе (на 15% быстрее) и это минимум, при котором vLLM разрешает prefix caching — при меньшем чанке срабатывает ассерт выравнивания mamba-кэша (block_size 2064). Карта позиций: точные до 262 144, floor(p/4) за ними. Замерено на 1 005 334 токенах, четырнадцать закладок, одна машина: YaRN 0/14, floor(p/4) по всем парам 3/14, по медленным 2/14, floor(p/8) 2/14, floor(p/16) 1/14, зажим пар 22-31 0/14. Внутри обученного окна карта тождественна, поэтому короткий и средний запрос не теряет ничего по построению. Температура проверена и отвергнута: tau=1.2890 дала 2/14 против 3/14 без неё.", | |
| "requires_free_bytes": 32344621711, | |
| "skip_mm_profiling": true, | |
| "swap_space": 0 | |
| } | |
| }, | |
| "schema": "lomonosov_zenit_serving_profiles_v2" | |
| }, | |
| "max_position_embeddings": 1010001, | |
| "model_type": "qwen3_5", | |
| "mtp_num_hidden_layers": 0, | |
| "noosphere": { | |
| "architecture_owned": true, | |
| "boundary_margin": 0.01, | |
| "context_tokens": 1010000, | |
| "decode_sparse_bias": true, | |
| "enabled": true, | |
| "exact_reread": true, | |
| "execution_contract": "raw_plus_additive", | |
| "hashes_per_feature": 2, | |
| "hot_tokens": 65536, | |
| "minimum_score": 0.1, | |
| "mode": "auto", | |
| "page_tokens": 128, | |
| "physical_window_tokens": 98304, | |
| "pinned_prefix_pages": 8, | |
| "quantization_independent": true, | |
| "query_tokens": 256, | |
| "raw_tokens_preserved": true, | |
| "routed_pages": 64, | |
| "routing_replaces_raw": false, | |
| "schema": "lomonosov_zenit_altay_model_owned_noosphere_v1", | |
| "semantic_bits": 1024, | |
| "vernadsky_surface": true, | |
| "vernadsky_weave": true, | |
| "widened_pages": 128 | |
| }, | |
| "quantization_config": { | |
| "config_groups": { | |
| "embed_tokens_w8a16_g128": { | |
| "format": "pack-quantized", | |
| "input_activations": null, | |
| "output_activations": null, | |
| "targets": [ | |
| "re:.*embed_tokens$" | |
| ], | |
| "weights": { | |
| "actorder": null, | |
| "block_structure": null, | |
| "dynamic": false, | |
| "group_size": 128, | |
| "num_bits": 8, | |
| "observer": "minmax", | |
| "observer_kwargs": {}, | |
| "scale_dtype": "torch.bfloat16", | |
| "strategy": "group", | |
| "symmetric": true, | |
| "type": "int", | |
| "zp_dtype": null | |
| } | |
| }, | |
| "group_0": { | |
| "format": "nvfp4-pack-quantized", | |
| "input_activations": { | |
| "actorder": null, | |
| "block_structure": null, | |
| "dynamic": "local", | |
| "group_size": 16, | |
| "num_bits": 4, | |
| "observer": "static_minmax", | |
| "observer_kwargs": {}, | |
| "scale_dtype": "torch.float8_e4m3fn", | |
| "strategy": "tensor_group", | |
| "symmetric": true, | |
| "type": "float", | |
| "zp_dtype": null | |
| }, | |
| "output_activations": null, | |
| "targets": [ | |
| "re:^model\\.language_model\\.layers\\.\\d+\\.linear_attn\\.(?:in_proj_qkv|in_proj_z|out_proj)$", | |
| "re:^language_model\\.model\\.layers\\.\\d+\\.linear_attn\\.(?:in_proj_qkv|in_proj_z|out_proj)$", | |
| "re:^model\\.language_model\\.layers\\.\\d+\\.mlp\\.(?:gate_proj|up_proj|down_proj)$", | |
| "re:^language_model\\.model\\.layers\\.\\d+\\.mlp\\.(?:gate_proj|up_proj|down_proj)$", | |
| "re:.*lm_head" | |
| ], | |
| "weights": { | |
| "actorder": null, | |
| "block_structure": null, | |
| "dynamic": false, | |
| "group_size": 16, | |
| "num_bits": 4, | |
| "observer": "memoryless_minmax", | |
| "observer_kwargs": {}, | |
| "scale_dtype": "torch.float8_e4m3fn", | |
| "strategy": "tensor_group", | |
| "symmetric": true, | |
| "type": "float", | |
| "zp_dtype": null | |
| } | |
| }, | |
| "group_1": { | |
| "format": "float-quantized", | |
| "input_activations": { | |
| "actorder": null, | |
| "block_structure": null, | |
| "dynamic": true, | |
| "group_size": null, | |
| "num_bits": 8, | |
| "observer": null, | |
| "observer_kwargs": {}, | |
| "scale_dtype": null, | |
| "strategy": "token", | |
| "symmetric": true, | |
| "type": "float", | |
| "zp_dtype": null | |
| }, | |
| "output_activations": null, | |
| "targets": [ | |
| "re:^model\\.language_model\\.layers\\.\\d+\\.self_attn\\.(?:q_proj|k_proj|v_proj|o_proj)$", | |
| "re:^language_model\\.model\\.layers\\.\\d+\\.self_attn\\.(?:q_proj|k_proj|v_proj|o_proj)$" | |
| ], | |
| "weights": { | |
| "actorder": null, | |
| "block_structure": null, | |
| "dynamic": false, | |
| "group_size": null, | |
| "num_bits": 8, | |
| "observer": "memoryless_minmax", | |
| "observer_kwargs": {}, | |
| "scale_dtype": null, | |
| "strategy": "channel", | |
| "symmetric": true, | |
| "type": "float", | |
| "zp_dtype": null | |
| } | |
| }, | |
| "vision_fp8": { | |
| "format": "float-quantized", | |
| "targets": [ | |
| "re:^model\\.visual\\.(?:blocks\\.\\d+\\.mlp\\.linear_fc1|(?:merger|deepstack_merger_list\\.\\d+)\\.linear_fc[12])$", | |
| "re:^model\\.visual\\.blocks\\.\\d+\\.attn\\.(?:qkv|proj)$", | |
| "re:^model\\.visual\\.blocks\\.\\d+\\.mlp\\.linear_fc2$", | |
| "re:^visual\\.(?:blocks\\.\\d+\\.mlp\\.linear_fc1|(?:merger|deepstack_merger_list\\.\\d+)\\.linear_fc[12])$", | |
| "re:^visual\\.blocks\\.\\d+\\.attn\\.(?:qkv|proj)$", | |
| "re:^visual\\.blocks\\.\\d+\\.mlp\\.linear_fc2$" | |
| ], | |
| "weights": { | |
| "num_bits": 8, | |
| "type": "float", | |
| "symmetric": true, | |
| "strategy": "channel", | |
| "group_size": null, | |
| "dynamic": false, | |
| "observer": "minmax", | |
| "observer_kwargs": {}, | |
| "actorder": null, | |
| "block_structure": null | |
| }, | |
| "input_activations": { | |
| "num_bits": 8, | |
| "type": "float", | |
| "symmetric": true, | |
| "strategy": "token", | |
| "group_size": null, | |
| "dynamic": true, | |
| "observer": null, | |
| "observer_kwargs": {}, | |
| "actorder": null, | |
| "block_structure": null | |
| }, | |
| "output_activations": null | |
| } | |
| }, | |
| "format": "mixed-precision", | |
| "global_compression_ratio": null, | |
| "ignore": [ | |
| "model.language_model.layers.0.linear_attn", | |
| "model.language_model.layers.0.linear_attn.norm", | |
| "model.language_model.layers.0.linear_attn.in_proj_b", | |
| "model.language_model.layers.0.linear_attn.in_proj_a", | |
| "model.language_model.layers.1.linear_attn", | |
| "model.language_model.layers.1.linear_attn.norm", | |
| "model.language_model.layers.1.linear_attn.in_proj_b", | |
| "model.language_model.layers.1.linear_attn.in_proj_a", | |
| "model.language_model.layers.2.linear_attn", | |
| "model.language_model.layers.2.linear_attn.norm", | |
| "model.language_model.layers.2.linear_attn.in_proj_b", | |
| "model.language_model.layers.2.linear_attn.in_proj_a", | |
| "model.language_model.layers.4.linear_attn", | |
| "model.language_model.layers.4.linear_attn.norm", | |
| "model.language_model.layers.4.linear_attn.in_proj_b", | |
| "model.language_model.layers.4.linear_attn.in_proj_a", | |
| "model.language_model.layers.5.linear_attn", | |
| "model.language_model.layers.5.linear_attn.norm", | |
| "model.language_model.layers.5.linear_attn.in_proj_b", | |
| "model.language_model.layers.5.linear_attn.in_proj_a", | |
| "model.language_model.layers.6.linear_attn", | |
| "model.language_model.layers.6.linear_attn.norm", | |
| "model.language_model.layers.6.linear_attn.in_proj_b", | |
| "model.language_model.layers.6.linear_attn.in_proj_a", | |
| "model.language_model.layers.8.linear_attn", | |
| "model.language_model.layers.8.linear_attn.norm", | |
| "model.language_model.layers.8.linear_attn.in_proj_b", | |
| "model.language_model.layers.8.linear_attn.in_proj_a", | |
| "model.language_model.layers.9.linear_attn", | |
| "model.language_model.layers.9.linear_attn.norm", | |
| "model.language_model.layers.9.linear_attn.in_proj_b", | |
| "model.language_model.layers.9.linear_attn.in_proj_a", | |
| "model.language_model.layers.10.linear_attn", | |
| "model.language_model.layers.10.linear_attn.norm", | |
| "model.language_model.layers.10.linear_attn.in_proj_b", | |
| "model.language_model.layers.10.linear_attn.in_proj_a", | |
| "model.language_model.layers.12.linear_attn", | |
| "model.language_model.layers.12.linear_attn.norm", | |
| "model.language_model.layers.12.linear_attn.in_proj_b", | |
| "model.language_model.layers.12.linear_attn.in_proj_a", | |
| "model.language_model.layers.13.linear_attn", | |
| "model.language_model.layers.13.linear_attn.norm", | |
| "model.language_model.layers.13.linear_attn.in_proj_b", | |
| "model.language_model.layers.13.linear_attn.in_proj_a", | |
| "model.language_model.layers.14.linear_attn", | |
| "model.language_model.layers.14.linear_attn.norm", | |
| "model.language_model.layers.14.linear_attn.in_proj_b", | |
| "model.language_model.layers.14.linear_attn.in_proj_a", | |
| "model.language_model.layers.16.linear_attn", | |
| "model.language_model.layers.16.linear_attn.norm", | |
| "model.language_model.layers.16.linear_attn.in_proj_b", | |
| "model.language_model.layers.16.linear_attn.in_proj_a", | |
| "model.language_model.layers.17.linear_attn", | |
| "model.language_model.layers.17.linear_attn.norm", | |
| "model.language_model.layers.17.linear_attn.in_proj_b", | |
| "model.language_model.layers.17.linear_attn.in_proj_a", | |
| "model.language_model.layers.18.linear_attn", | |
| "model.language_model.layers.18.linear_attn.norm", | |
| "model.language_model.layers.18.linear_attn.in_proj_b", | |
| "model.language_model.layers.18.linear_attn.in_proj_a", | |
| "model.language_model.layers.20.linear_attn", | |
| "model.language_model.layers.20.linear_attn.norm", | |
| "model.language_model.layers.20.linear_attn.in_proj_b", | |
| "model.language_model.layers.20.linear_attn.in_proj_a", | |
| "model.language_model.layers.21.linear_attn", | |
| "model.language_model.layers.21.linear_attn.norm", | |
| "model.language_model.layers.21.linear_attn.in_proj_b", | |
| "model.language_model.layers.21.linear_attn.in_proj_a", | |
| "model.language_model.layers.22.linear_attn", | |
| "model.language_model.layers.22.linear_attn.norm", | |
| "model.language_model.layers.22.linear_attn.in_proj_b", | |
| "model.language_model.layers.22.linear_attn.in_proj_a", | |
| "model.language_model.layers.24.linear_attn", | |
| "model.language_model.layers.24.linear_attn.norm", | |
| "model.language_model.layers.24.linear_attn.in_proj_b", | |
| "model.language_model.layers.24.linear_attn.in_proj_a", | |
| "model.language_model.layers.25.linear_attn", | |
| "model.language_model.layers.25.linear_attn.norm", | |
| "model.language_model.layers.25.linear_attn.in_proj_b", | |
| "model.language_model.layers.25.linear_attn.in_proj_a", | |
| "model.language_model.layers.26.linear_attn", | |
| "model.language_model.layers.26.linear_attn.norm", | |
| "model.language_model.layers.26.linear_attn.in_proj_b", | |
| "model.language_model.layers.26.linear_attn.in_proj_a", | |
| "model.language_model.layers.28.linear_attn", | |
| "model.language_model.layers.28.linear_attn.norm", | |
| "model.language_model.layers.28.linear_attn.in_proj_b", | |
| "model.language_model.layers.28.linear_attn.in_proj_a", | |
| "model.language_model.layers.29.linear_attn", | |
| "model.language_model.layers.29.linear_attn.norm", | |
| "model.language_model.layers.29.linear_attn.in_proj_b", | |
| "model.language_model.layers.29.linear_attn.in_proj_a", | |
| "model.language_model.layers.30.linear_attn", | |
| "model.language_model.layers.30.linear_attn.norm", | |
| "model.language_model.layers.30.linear_attn.in_proj_b", | |
| "model.language_model.layers.30.linear_attn.in_proj_a", | |
| "model.language_model.layers.32.linear_attn", | |
| "model.language_model.layers.32.linear_attn.norm", | |
| "model.language_model.layers.32.linear_attn.in_proj_b", | |
| "model.language_model.layers.32.linear_attn.in_proj_a", | |
| "model.language_model.layers.33.linear_attn", | |
| "model.language_model.layers.33.linear_attn.norm", | |
| "model.language_model.layers.33.linear_attn.in_proj_b", | |
| "model.language_model.layers.33.linear_attn.in_proj_a", | |
| "model.language_model.layers.34.linear_attn", | |
| "model.language_model.layers.34.linear_attn.norm", | |
| "model.language_model.layers.34.linear_attn.in_proj_b", | |
| "model.language_model.layers.34.linear_attn.in_proj_a", | |
| "model.language_model.layers.36.linear_attn", | |
| "model.language_model.layers.36.linear_attn.norm", | |
| "model.language_model.layers.36.linear_attn.in_proj_b", | |
| "model.language_model.layers.36.linear_attn.in_proj_a", | |
| "model.language_model.layers.37.linear_attn", | |
| "model.language_model.layers.37.linear_attn.norm", | |
| "model.language_model.layers.37.linear_attn.in_proj_b", | |
| "model.language_model.layers.37.linear_attn.in_proj_a", | |
| "model.language_model.layers.38.linear_attn", | |
| "model.language_model.layers.38.linear_attn.norm", | |
| "model.language_model.layers.38.linear_attn.in_proj_b", | |
| "model.language_model.layers.38.linear_attn.in_proj_a", | |
| "model.language_model.layers.40.linear_attn", | |
| "model.language_model.layers.40.linear_attn.norm", | |
| "model.language_model.layers.40.linear_attn.in_proj_b", | |
| "model.language_model.layers.40.linear_attn.in_proj_a", | |
| "model.language_model.layers.41.linear_attn", | |
| "model.language_model.layers.41.linear_attn.norm", | |
| "model.language_model.layers.41.linear_attn.in_proj_b", | |
| "model.language_model.layers.41.linear_attn.in_proj_a", | |
| "model.language_model.layers.42.linear_attn", | |
| "model.language_model.layers.42.linear_attn.norm", | |
| "model.language_model.layers.42.linear_attn.in_proj_b", | |
| "model.language_model.layers.42.linear_attn.in_proj_a", | |
| "model.language_model.layers.44.linear_attn", | |
| "model.language_model.layers.44.linear_attn.norm", | |
| "model.language_model.layers.44.linear_attn.in_proj_b", | |
| "model.language_model.layers.44.linear_attn.in_proj_a", | |
| "model.language_model.layers.45.linear_attn", | |
| "model.language_model.layers.45.linear_attn.norm", | |
| "model.language_model.layers.45.linear_attn.in_proj_b", | |
| "model.language_model.layers.45.linear_attn.in_proj_a", | |
| "model.language_model.layers.46.linear_attn", | |
| "model.language_model.layers.46.linear_attn.norm", | |
| "model.language_model.layers.46.linear_attn.in_proj_b", | |
| "model.language_model.layers.46.linear_attn.in_proj_a", | |
| "model.language_model.layers.48.linear_attn", | |
| "model.language_model.layers.48.linear_attn.norm", | |
| "model.language_model.layers.48.linear_attn.in_proj_b", | |
| "model.language_model.layers.48.linear_attn.in_proj_a", | |
| "model.language_model.layers.49.linear_attn", | |
| "model.language_model.layers.49.linear_attn.norm", | |
| "model.language_model.layers.49.linear_attn.in_proj_b", | |
| "model.language_model.layers.49.linear_attn.in_proj_a", | |
| "model.language_model.layers.50.linear_attn", | |
| "model.language_model.layers.50.linear_attn.norm", | |
| "model.language_model.layers.50.linear_attn.in_proj_b", | |
| "model.language_model.layers.50.linear_attn.in_proj_a", | |
| "model.language_model.layers.52.linear_attn", | |
| "model.language_model.layers.52.linear_attn.norm", | |
| "model.language_model.layers.52.linear_attn.in_proj_b", | |
| "model.language_model.layers.52.linear_attn.in_proj_a", | |
| "model.language_model.layers.53.linear_attn", | |
| "model.language_model.layers.53.linear_attn.norm", | |
| "model.language_model.layers.53.linear_attn.in_proj_b", | |
| "model.language_model.layers.53.linear_attn.in_proj_a", | |
| "model.language_model.layers.54.linear_attn", | |
| "model.language_model.layers.54.linear_attn.norm", | |
| "model.language_model.layers.54.linear_attn.in_proj_b", | |
| "model.language_model.layers.54.linear_attn.in_proj_a", | |
| "model.language_model.layers.56.linear_attn", | |
| "model.language_model.layers.56.linear_attn.norm", | |
| "model.language_model.layers.56.linear_attn.in_proj_b", | |
| "model.language_model.layers.56.linear_attn.in_proj_a", | |
| "model.language_model.layers.57.linear_attn", | |
| "model.language_model.layers.57.linear_attn.norm", | |
| "model.language_model.layers.57.linear_attn.in_proj_b", | |
| "model.language_model.layers.57.linear_attn.in_proj_a", | |
| "model.language_model.layers.58.linear_attn", | |
| "model.language_model.layers.58.linear_attn.norm", | |
| "model.language_model.layers.58.linear_attn.in_proj_b", | |
| "model.language_model.layers.58.linear_attn.in_proj_a", | |
| "model.language_model.layers.60.linear_attn", | |
| "model.language_model.layers.60.linear_attn.norm", | |
| "model.language_model.layers.60.linear_attn.in_proj_b", | |
| "model.language_model.layers.60.linear_attn.in_proj_a", | |
| "model.language_model.layers.61.linear_attn", | |
| "model.language_model.layers.61.linear_attn.norm", | |
| "model.language_model.layers.61.linear_attn.in_proj_b", | |
| "model.language_model.layers.61.linear_attn.in_proj_a", | |
| "model.language_model.layers.62.linear_attn", | |
| "model.language_model.layers.62.linear_attn.norm", | |
| "model.language_model.layers.62.linear_attn.in_proj_b", | |
| "model.language_model.layers.62.linear_attn.in_proj_a" | |
| ], | |
| "kv_cache_scheme": null, | |
| "quant_method": "compressed-tensors", | |
| "quantization_status": "compressed", | |
| "sparsity_config": {}, | |
| "transform_config": {}, | |
| "version": "0.17.2.a20260724" | |
| }, | |
| "rope_parameters": { | |
| "mrope_interleaved": true, | |
| "mrope_section": [ | |
| 11, | |
| 11, | |
| 10 | |
| ], | |
| "partial_rotary_factor": 0.25, | |
| "rope_theta": 10000000, | |
| "rope_type": "default" | |
| }, | |
| "text_config": { | |
| "altay_architecture_id": "LOMONOSOV_ZENIT_ALTAY_72M_SKV_V1", | |
| "altay_extra_recurrent_slots": 6, | |
| "altay_logical_layers": 72, | |
| "altay_physical_kv_units": 64, | |
| "architectures": [ | |
| "Qwen3_5ForCausalLM" | |
| ], | |
| "attention_bias": false, | |
| "attention_dropout": 0.0, | |
| "attn_output_gate": true, | |
| "bos_token_id": 248044, | |
| "dtype": "bfloat16", | |
| "eos_token_id": 248044, | |
| "full_attention_interval": 4, | |
| "head_dim": 256, | |
| "hidden_act": "silu", | |
| "hidden_size": 5120, | |
| "initializer_range": 0.02, | |
| "intermediate_size": 17408, | |
| "layer_types": [ | |
| "linear_attention", | |
| "linear_attention", | |
| "linear_attention", | |
| "full_attention", | |
| "linear_attention", | |
| "linear_attention", | |
| "linear_attention", | |
| "full_attention", | |
| "linear_attention", | |
| "linear_attention", | |
| "linear_attention", | |
| "full_attention", | |
| "linear_attention", | |
| "linear_attention", | |
| "linear_attention", | |
| "full_attention", | |
| "linear_attention", | |
| "linear_attention", | |
| "linear_attention", | |
| "full_attention", | |
| "linear_attention", | |
| "linear_attention", | |
| "linear_attention", | |
| "full_attention", | |
| "linear_attention", | |
| "linear_attention", | |
| "linear_attention", | |
| "full_attention", | |
| "linear_attention", | |
| "linear_attention", | |
| "linear_attention", | |
| "full_attention", | |
| "linear_attention", | |
| "linear_attention", | |
| "linear_attention", | |
| "full_attention", | |
| "linear_attention", | |
| "linear_attention", | |
| "linear_attention", | |
| "full_attention", | |
| "linear_attention", | |
| "linear_attention", | |
| "linear_attention", | |
| "full_attention", | |
| "linear_attention", | |
| "linear_attention", | |
| "linear_attention", | |
| "full_attention", | |
| "linear_attention", | |
| "linear_attention", | |
| "linear_attention", | |
| "full_attention", | |
| "linear_attention", | |
| "linear_attention", | |
| "linear_attention", | |
| "full_attention", | |
| "linear_attention", | |
| "linear_attention", | |
| "linear_attention", | |
| "full_attention", | |
| "linear_attention", | |
| "linear_attention", | |
| "linear_attention", | |
| "full_attention" | |
| ], | |
| "linear_conv_kernel_dim": 4, | |
| "linear_key_head_dim": 128, | |
| "linear_num_key_heads": 16, | |
| "linear_num_value_heads": 48, | |
| "linear_value_head_dim": 128, | |
| "mamba_ssm_dtype": "float32", | |
| "max_position_embeddings": 1010001, | |
| "model_name": "/mnt/MainDisk/Project Sirius Experimental/ZVEZDA/runtime-models/ZVEZDA-1M-TRAINABLE-NF4", | |
| "model_type": "qwen3_5_text", | |
| "mtp_num_hidden_layers": 0, | |
| "mtp_use_dedicated_embeddings": false, | |
| "num_attention_heads": 24, | |
| "num_hidden_layers": 64, | |
| "num_key_value_heads": 4, | |
| "output_gate_type": "swish", | |
| "pad_token_id": 248044, | |
| "partial_rotary_factor": 0.25, | |
| "rms_norm_eps": 1e-06, | |
| "rope_parameters": { | |
| "mrope_interleaved": true, | |
| "mrope_section": [ | |
| 11, | |
| 11, | |
| 10 | |
| ], | |
| "partial_rotary_factor": 0.25, | |
| "rope_theta": 10000000, | |
| "rope_type": "default" | |
| }, | |
| "tie_word_embeddings": false, | |
| "transformers_version": "5.14.1", | |
| "unsloth_version": "2026.7.2", | |
| "use_cache": false, | |
| "vocab_size": 248320 | |
| }, | |
| "tie_word_embeddings": false, | |
| "transformers_version": "5.13.0", | |
| "video_token_id": 248057, | |
| "vision_config": { | |
| "deepstack_visual_indexes": [], | |
| "depth": 27, | |
| "dtype": "bfloat16", | |
| "hidden_act": "gelu_pytorch_tanh", | |
| "hidden_size": 1152, | |
| "in_channels": 3, | |
| "initializer_range": 0.02, | |
| "intermediate_size": 4304, | |
| "model_type": "qwen3_5_vision", | |
| "num_heads": 16, | |
| "num_position_embeddings": 2304, | |
| "out_hidden_size": 5120, | |
| "patch_size": 16, | |
| "spatial_merge_size": 2, | |
| "temporal_patch_size": 2 | |
| }, | |
| "vision_end_token_id": 248054, | |
| "vision_quantization": { | |
| "activations": "BF16", | |
| "group_size": "adaptive_128_16", | |
| "promotion": "forbidden_until_multimodal_A_B_and_raw1010k_PASS", | |
| "scheme": "SELECTIVE_W8_W4_A16", | |
| "source_precision": "BF16", | |
| "targets": [ | |
| "re:^model\\.visual\\.blocks\\.\\d+\\.attn\\.(?:qkv|proj)$", | |
| "re:^visual\\.blocks\\.\\d+\\.attn\\.(?:qkv|proj)$", | |
| "re:^model\\.visual\\.(?:blocks\\.\\d+\\.mlp\\.linear_fc1|(?:merger|deepstack_merger_list\\.\\d+)\\.linear_fc[12])$", | |
| "re:^visual\\.(?:blocks\\.\\d+\\.mlp\\.linear_fc1|(?:merger|deepstack_merger_list\\.\\d+)\\.linear_fc[12])$", | |
| "re:^model\\.visual\\.blocks\\.\\d+\\.mlp\\.linear_fc2$", | |
| "re:^visual\\.blocks\\.\\d+\\.mlp\\.linear_fc2$" | |
| ], | |
| "weights": "INT8_attention_INT4_mlp" | |
| }, | |
| "vision_start_token_id": 248053, | |
| "_altay_embedding_group": { | |
| "group_size": 128, | |
| "num_bits": 8, | |
| "strategy": "group", | |
| "symmetric": true, | |
| "type": "int" | |
| } | |
| } | |