Updated files via hf-cli upload after local modifications.

.gitattributes

@@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+granite-4.0-h-350m-unsloth-mxfp4_moe-EKUD:B16-O:Q6K-Q:Q8_0.gguf filter=lfs diff=lfs merge=lfs -text
+granite-4.0-h-350m-unsloth-mxfp4_moe-O:Q6K-EQKUD:Q8_0.gguf filter=lfs diff=lfs merge=lfs -text

Benchmarks/DataCollection/granite-4.0-h-350m-unsloth-iq4_nl-attn_kv_Q5_K-attn_output_Q5_K-attn_q_Q5_K-embeddings_Q5_K-ffn_down_Q5_K-ffn_up_gate_Q5_K/bench_metrics.json

@@ -0,0 +1,44 @@
+{
+  "raw_metrics": {
+    "llamabench": {
+      "backend": "CUDA",
+      "log_path": "granite-4.0-h-350m-unsloth-iq4_nl-attn_kv_Q5_K-attn_output_Q5_K-attn_q_Q5_K-embeddings_Q5_K-ffn_down_Q5_K-ffn_up_gate_Q5_K/llamabench.md",
+      "ngl": "35",
+      "raw_row": {
+        "backend": "CUDA",
+        "model": "granitehybrid 350M IQ4_NL - 4.5 bpw",
+        "ngl": "35",
+        "params": "340.33 M",
+        "size": "211.65 MiB",
+        "t/s": "1548.90 \u00b1 70.54",
+        "test": "pp8",
+        "tps_value": 1548.9
+      },
+      "test": "pp8",
+      "tps": 1548.9
+    },
+    "perplexity": {
+      "code": {
+        "log_path": "granite-4.0-h-350m-unsloth-iq4_nl-attn_kv_Q5_K-attn_output_Q5_K-attn_q_Q5_K-embeddings_Q5_K-ffn_down_Q5_K-ffn_up_gate_Q5_K/perplexity_code.log",
+        "ppl": 2.041,
+        "ppl_error": 0.01914
+      },
+      "general": {
+        "log_path": "granite-4.0-h-350m-unsloth-iq4_nl-attn_kv_Q5_K-attn_output_Q5_K-attn_q_Q5_K-embeddings_Q5_K-ffn_down_Q5_K-ffn_up_gate_Q5_K/perplexity_general.log",
+        "ppl": 22.0027,
+        "ppl_error": 0.58551
+      },
+      "math": {
+        "log_path": "granite-4.0-h-350m-unsloth-iq4_nl-attn_kv_Q5_K-attn_output_Q5_K-attn_q_Q5_K-embeddings_Q5_K-ffn_down_Q5_K-ffn_up_gate_Q5_K/perplexity_math.log",
+        "ppl": 11.6501,
+        "ppl_error": 0.27272
+      }
+    }
+  },
+  "summary": {
+    "avg_prec_loss_pct": 12.9989,
+    "bench_tps": 1548.9,
+    "file_size_bytes": 225498048,
+    "file_size_gb": 0.21
+  }
+}

Benchmarks/DataCollection/granite-4.0-h-350m-unsloth-iq4_nl-attn_kv_Q5_K-attn_output_Q5_K-attn_q_Q5_K-embeddings_Q5_K-ffn_down_Q5_K-ffn_up_gate_Q5_K/llamabench.md

@@ -0,0 +1,11 @@
+ggml_cuda_init: GGML_CUDA_FORCE_MMQ:    no
+ggml_cuda_init: GGML_CUDA_FORCE_CUBLAS: no
+ggml_cuda_init: found 2 CUDA devices:
+  Device 0: NVIDIA GeForce RTX 3090, compute capability 8.6, VMM: yes
+  Device 1: NVIDIA GeForce RTX 3090, compute capability 8.6, VMM: yes
+| model                          |       size |     params | backend    | ngl |            test |                  t/s |
+| ------------------------------ | ---------: | ---------: | ---------- | --: | --------------: | -------------------: |
+| granitehybrid 350M IQ4_NL - 4.5 bpw | 211.65 MiB |   340.33 M | CUDA       |  35 |             pp8 |      1548.90 ± 70.54 |
+| granitehybrid 350M IQ4_NL - 4.5 bpw | 211.65 MiB |   340.33 M | CUDA       |  35 |           tg128 |       321.30 ± 12.55 |
+
+build: 92bb442ad (7040)

Benchmarks/DataCollection/granite-4.0-h-350m-unsloth-iq4_nl-attn_kv_Q5_K-attn_output_Q5_K-attn_q_Q5_K-embeddings_Q5_K-ffn_down_Q5_K-ffn_up_gate_Q5_K/perplexity_code.log

@@ -0,0 +1,189 @@
+ggml_cuda_init: GGML_CUDA_FORCE_MMQ:    no
+ggml_cuda_init: GGML_CUDA_FORCE_CUBLAS: no
+ggml_cuda_init: found 2 CUDA devices:
+  Device 0: NVIDIA GeForce RTX 3090, compute capability 8.6, VMM: yes
+  Device 1: NVIDIA GeForce RTX 3090, compute capability 8.6, VMM: yes
+build: 7040 (92bb442ad) with cc (Ubuntu 13.3.0-6ubuntu2~24.04) 13.3.0 for x86_64-linux-gnu
+llama_model_load_from_file_impl: using device CUDA0 (NVIDIA GeForce RTX 3090) (0000:01:00.0) - 20938 MiB free
+llama_model_load_from_file_impl: using device CUDA1 (NVIDIA GeForce RTX 3090) (0000:03:00.0) - 23581 MiB free
+llama_model_loader: loaded meta data with 48 key-value pairs and 402 tensors from /mnt/world8/AI/ToBench/granite-4.0-h-350m-unsloth/Magic_Quant/GGUF/dc_round0_granite-4.0-h-350m-unsloth-iq4_nl-attn_kv_Q5_K-attn_output_Q5_K-attn_q_Q5_K-embeddings_Q5_K-ffn_down_Q5_K-ffn_up_gate_Q5_K.gguf (version GGUF V3 (latest))
+llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output.
+llama_model_loader: - kv   0:                       general.architecture str              = granitehybrid
+llama_model_loader: - kv   1:                               general.type str              = model
+llama_model_loader: - kv   2:                               general.name str              = Granite 4.0 H 350m Unsloth
+llama_model_loader: - kv   3:                           general.finetune str              = unsloth
+llama_model_loader: - kv   4:                           general.basename str              = granite-4.0-h
+llama_model_loader: - kv   5:                         general.size_label str              = 350M
+llama_model_loader: - kv   6:                            general.license str              = apache-2.0
+llama_model_loader: - kv   7:                   general.base_model.count u32              = 1
+llama_model_loader: - kv   8:                  general.base_model.0.name str              = Granite 4.0 H 350m
+llama_model_loader: - kv   9:          general.base_model.0.organization str              = Ibm Granite
+llama_model_loader: - kv  10:              general.base_model.0.repo_url str              = https://huggingface.co/ibm-granite/gr...
+llama_model_loader: - kv  11:                               general.tags arr[str,3]       = ["language", "unsloth", "granite-4.0"]
+llama_model_loader: - kv  12:                  granitehybrid.block_count u32              = 32
+llama_model_loader: - kv  13:               granitehybrid.context_length u32              = 1048576
+llama_model_loader: - kv  14:             granitehybrid.embedding_length u32              = 768
+llama_model_loader: - kv  15:          granitehybrid.feed_forward_length u32              = 2048
+llama_model_loader: - kv  16:         granitehybrid.attention.head_count u32              = 12
+llama_model_loader: - kv  17:      granitehybrid.attention.head_count_kv arr[i32,32]      = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, ...
+llama_model_loader: - kv  18:               granitehybrid.rope.freq_base f32              = 10000.000000
+llama_model_loader: - kv  19: granitehybrid.attention.layer_norm_rms_epsilon f32              = 0.000010
+llama_model_loader: - kv  20:                 granitehybrid.expert_count u32              = 0
+llama_model_loader: - kv  21:            granitehybrid.expert_used_count u32              = 0
+llama_model_loader: - kv  22:                   granitehybrid.vocab_size u32              = 100352
+llama_model_loader: - kv  23:         granitehybrid.rope.dimension_count u32              = 64
+llama_model_loader: - kv  24:              granitehybrid.attention.scale f32              = 0.015625
+llama_model_loader: - kv  25:              granitehybrid.embedding_scale f32              = 12.000000
+llama_model_loader: - kv  26:               granitehybrid.residual_scale f32              = 0.246000
+llama_model_loader: - kv  27:                  granitehybrid.logit_scale f32              = 3.000000
+llama_model_loader: - kv  28: granitehybrid.expert_shared_feed_forward_length u32              = 2048
+llama_model_loader: - kv  29:              granitehybrid.ssm.conv_kernel u32              = 4
+llama_model_loader: - kv  30:               granitehybrid.ssm.state_size u32              = 128
+llama_model_loader: - kv  31:              granitehybrid.ssm.group_count u32              = 1
+llama_model_loader: - kv  32:               granitehybrid.ssm.inner_size u32              = 1536
+llama_model_loader: - kv  33:           granitehybrid.ssm.time_step_rank u32              = 48
+llama_model_loader: - kv  34:       granitehybrid.rope.scaling.finetuned bool             = false
+llama_model_loader: - kv  35:                       tokenizer.ggml.model str              = gpt2
+llama_model_loader: - kv  36:                         tokenizer.ggml.pre str              = dbrx
+llama_model_loader: - kv  37:                      tokenizer.ggml.tokens arr[str,100352]  = ["!", "\"", "#", "$", "%", "&", "'", ...
+llama_model_loader: - kv  38:                  tokenizer.ggml.token_type arr[i32,100352]  = [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...
+llama_model_loader: - kv  39:                      tokenizer.ggml.merges arr[str,100000]  = ["Ġ Ġ", "ĠĠ ĠĠ", "i n", "Ġ t",...
+llama_model_loader: - kv  40:                tokenizer.ggml.bos_token_id u32              = 100257
+llama_model_loader: - kv  41:                tokenizer.ggml.eos_token_id u32              = 100257
+llama_model_loader: - kv  42:            tokenizer.ggml.unknown_token_id u32              = 100269
+llama_model_loader: - kv  43:            tokenizer.ggml.padding_token_id u32              = 100256
+llama_model_loader: - kv  44:               tokenizer.ggml.add_bos_token bool             = false
+llama_model_loader: - kv  45:                    tokenizer.chat_template str              = {%- set tools_system_message_prefix =...
+llama_model_loader: - kv  46:               general.quantization_version u32              = 2
+llama_model_loader: - kv  47:                          general.file_type u32              = 25
+llama_model_loader: - type  f32:  233 tensors
+llama_model_loader: - type q5_K:  113 tensors
+llama_model_loader: - type iq4_nl:   56 tensors
+print_info: file format = GGUF V3 (latest)
+print_info: file type   = IQ4_NL - 4.5 bpw
+print_info: file size   = 211.65 MiB (5.22 BPW) 
+load: printing all EOG tokens:
+load:   - 100257 ('<|end_of_text|>')
+load:   - 100261 ('<|fim_pad|>')
+load: special tokens cache size = 96
+load: token to piece cache size = 0.6152 MB
+print_info: arch             = granitehybrid
+print_info: vocab_only       = 0
+print_info: n_ctx_train      = 1048576
+print_info: n_embd           = 768
+print_info: n_embd_inp       = 768
+print_info: n_layer          = 32
+print_info: n_head           = 12
+print_info: n_head_kv        = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0]
+print_info: n_rot            = 64
+print_info: n_swa            = 0
+print_info: is_swa_any       = 0
+print_info: n_embd_head_k    = 64
+print_info: n_embd_head_v    = 64
+print_info: n_gqa            = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 3, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0]
+print_info: n_embd_k_gqa     = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 256, 0, 0, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 0, 0]
+print_info: n_embd_v_gqa     = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 256, 0, 0, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 0, 0]
+print_info: f_norm_eps       = 0.0e+00
+print_info: f_norm_rms_eps   = 1.0e-05
+print_info: f_clamp_kqv      = 0.0e+00
+print_info: f_max_alibi_bias = 0.0e+00
+print_info: f_logit_scale    = 3.0e+00
+print_info: f_attn_scale     = 1.6e-02
+print_info: n_ff             = 2048
+print_info: n_expert         = 0
+print_info: n_expert_used    = 0
+print_info: n_expert_groups  = 0
+print_info: n_group_used     = 0
+print_info: causal attn      = 1
+print_info: pooling type     = 0
+print_info: rope type        = 0
+print_info: rope scaling     = linear
+print_info: freq_base_train  = 10000.0
+print_info: freq_scale_train = 1
+print_info: n_ctx_orig_yarn  = 1048576
+print_info: rope_finetuned   = unknown
+print_info: ssm_d_conv       = 4
+print_info: ssm_d_inner      = 1536
+print_info: ssm_d_state      = 128
+print_info: ssm_dt_rank      = 48
+print_info: ssm_n_group      = 1
+print_info: ssm_dt_b_c_rms   = 0
+print_info: model type       = 350M
+print_info: model params     = 340.33 M
+print_info: general.name     = Granite 4.0 H 350m Unsloth
+print_info: f_embedding_scale = 12.000000
+print_info: f_residual_scale  = 0.246000
+print_info: f_attention_scale = 0.015625
+print_info: n_ff_shexp        = 2048
+print_info: vocab type       = BPE
+print_info: n_vocab          = 100352
+print_info: n_merges         = 100000
+print_info: BOS token        = 100257 '<|end_of_text|>'
+print_info: EOS token        = 100257 '<|end_of_text|>'
+print_info: EOT token        = 100257 '<|end_of_text|>'
+print_info: UNK token        = 100269 '<|unk|>'
+print_info: PAD token        = 100256 '<|pad|>'
+print_info: LF token         = 198 'Ċ'
+print_info: FIM PRE token    = 100258 '<|fim_prefix|>'
+print_info: FIM SUF token    = 100260 '<|fim_suffix|>'
+print_info: FIM MID token    = 100259 '<|fim_middle|>'
+print_info: FIM PAD token    = 100261 '<|fim_pad|>'
+print_info: EOG token        = 100257 '<|end_of_text|>'
+print_info: EOG token        = 100261 '<|fim_pad|>'
+print_info: max token length = 256
+load_tensors: loading model tensors, this can take a while... (mmap = true)
+load_tensors: offloading 20 repeating layers to GPU
+load_tensors: offloaded 20/33 layers to GPU
+load_tensors:   CPU_Mapped model buffer size =   111.47 MiB
+load_tensors:        CUDA0 model buffer size =    49.58 MiB
+load_tensors:        CUDA1 model buffer size =    50.62 MiB
+..............................................................................
+llama_context: constructing llama_context
+llama_context: n_seq_max     = 1
+llama_context: n_ctx         = 2048
+llama_context: n_ctx_seq     = 2048
+llama_context: n_batch       = 2048
+llama_context: n_ubatch      = 512
+llama_context: causal_attn   = 1
+llama_context: flash_attn    = auto
+llama_context: kv_unified    = false
+llama_context: freq_base     = 10000.0
+llama_context: freq_scale    = 1
+llama_context: n_ctx_seq (2048) < n_ctx_train (1048576) -- the full capacity of the model will not be utilized
+llama_context:        CPU  output buffer size =     0.38 MiB
+llama_kv_cache:        CPU KV buffer size =     2.00 MiB
+llama_kv_cache:      CUDA0 KV buffer size =     4.00 MiB
+llama_kv_cache:      CUDA1 KV buffer size =     2.00 MiB
+llama_kv_cache: size =    8.00 MiB (  2048 cells,   4 layers,  1/1 seqs), K (f16):    4.00 MiB, V (f16):    4.00 MiB
+llama_memory_recurrent:        CPU RS buffer size =     8.48 MiB
+llama_memory_recurrent:      CUDA0 RS buffer size =     6.16 MiB
+llama_memory_recurrent:      CUDA1 RS buffer size =     6.93 MiB
+llama_memory_recurrent: size =   21.57 MiB (     1 cells,  32 layers,  1 seqs), R (f32):    0.57 MiB, S (f32):   21.00 MiB
+llama_context: Flash Attention was auto, set to enabled
+llama_context:      CUDA0 compute buffer size =   248.03 MiB
+llama_context:      CUDA1 compute buffer size =    22.39 MiB
+llama_context:  CUDA_Host compute buffer size =    18.34 MiB
+llama_context: graph nodes  = 1815
+llama_context: graph splits = 182 (with bs=512), 41 (with bs=1)
+common_init_from_params: added <|end_of_text|> logit bias = -inf
+common_init_from_params: added <|fim_pad|> logit bias = -inf
+common_init_from_params: setting dry_penalty_last_n to ctx_size = 2048
+common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable)
+
+system_info: n_threads = 16 (n_threads_batch = 16) / 32 | CUDA : ARCHS = 860 | USE_GRAPHS = 1 | PEER_MAX_BATCH_SIZE = 128 | CPU : SSE3 = 1 | SSSE3 = 1 | AVX = 1 | AVX2 = 1 | F16C = 1 | FMA = 1 | BMI2 = 1 | AVX512 = 1 | AVX512_VBMI = 1 | AVX512_VNNI = 1 | AVX512_BF16 = 1 | LLAMAFILE = 1 | OPENMP = 1 | REPACK = 1 | 
+perplexity: tokenizing the input ..
+perplexity: tokenization took 96.565 ms
+perplexity: calculating perplexity over 44 chunks, n_ctx=2048, batch_size=2048, n_seq=1
+perplexity: 0.50 seconds per pass - ETA 0.35 minutes
+[1]4.7656,[2]4.4086,[3]2.7608,[4]2.5367,[5]2.8492,[6]3.1148,[7]2.9353,[8]2.7074,[9]2.4708,[10]2.2756,[11]2.2511,[12]2.2794,[13]2.1779,[14]2.1493,[15]2.1910,[16]2.1168,[17]2.0873,[18]2.1076,[19]2.0655,[20]2.0261,[21]1.9892,[22]1.9726,[23]2.0014,[24]1.9722,[25]1.9911,[26]1.9567,[27]1.9417,[28]1.9324,[29]1.9820,[30]1.9982,[31]1.9985,[32]1.9713,[33]1.9966,[34]1.9876,[35]1.9670,[36]2.0002,[37]2.0065,[38]2.0038,[39]2.0274,[40]2.0250,[41]2.0163,[42]2.0434,[43]2.0526,[44]2.0410,
+Final estimate: PPL = 2.0410 +/- 0.01914
+
+llama_perf_context_print:        load time =     199.89 ms
+llama_perf_context_print: prompt eval time =   14562.78 ms / 90112 tokens (    0.16 ms per token,  6187.83 tokens per second)
+llama_perf_context_print:        eval time =       0.00 ms /     1 runs   (    0.00 ms per token,      inf tokens per second)
+llama_perf_context_print:       total time =   15350.15 ms / 90113 tokens
+llama_perf_context_print:    graphs reused =          0
+llama_memory_breakdown_print: | memory breakdown [MiB] | total    free    self   model   context   compute    unaccounted |
+llama_memory_breakdown_print: |   - CUDA0 (RTX 3090)   | 24115 = 20558 + ( 315 =    49 +      10 +     256) +        3240 |
+llama_memory_breakdown_print: |   - CUDA1 (RTX 3090)   | 24124 = 23431 + (  81 =    50 +       8 +      22) +         610 |
+llama_memory_breakdown_print: |   - Host               |                   140 =   111 +      10 +      18                |

Benchmarks/DataCollection/granite-4.0-h-350m-unsloth-iq4_nl-attn_kv_Q5_K-attn_output_Q5_K-attn_q_Q5_K-embeddings_Q5_K-ffn_down_Q5_K-ffn_up_gate_Q5_K/perplexity_general.log

@@ -0,0 +1,189 @@
+ggml_cuda_init: GGML_CUDA_FORCE_MMQ:    no
+ggml_cuda_init: GGML_CUDA_FORCE_CUBLAS: no
+ggml_cuda_init: found 2 CUDA devices:
+  Device 0: NVIDIA GeForce RTX 3090, compute capability 8.6, VMM: yes
+  Device 1: NVIDIA GeForce RTX 3090, compute capability 8.6, VMM: yes
+build: 7040 (92bb442ad) with cc (Ubuntu 13.3.0-6ubuntu2~24.04) 13.3.0 for x86_64-linux-gnu
+llama_model_load_from_file_impl: using device CUDA0 (NVIDIA GeForce RTX 3090) (0000:01:00.0) - 20932 MiB free
+llama_model_load_from_file_impl: using device CUDA1 (NVIDIA GeForce RTX 3090) (0000:03:00.0) - 23581 MiB free
+llama_model_loader: loaded meta data with 48 key-value pairs and 402 tensors from /mnt/world8/AI/ToBench/granite-4.0-h-350m-unsloth/Magic_Quant/GGUF/dc_round0_granite-4.0-h-350m-unsloth-iq4_nl-attn_kv_Q5_K-attn_output_Q5_K-attn_q_Q5_K-embeddings_Q5_K-ffn_down_Q5_K-ffn_up_gate_Q5_K.gguf (version GGUF V3 (latest))
+llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output.
+llama_model_loader: - kv   0:                       general.architecture str              = granitehybrid
+llama_model_loader: - kv   1:                               general.type str              = model
+llama_model_loader: - kv   2:                               general.name str              = Granite 4.0 H 350m Unsloth
+llama_model_loader: - kv   3:                           general.finetune str              = unsloth
+llama_model_loader: - kv   4:                           general.basename str              = granite-4.0-h
+llama_model_loader: - kv   5:                         general.size_label str              = 350M
+llama_model_loader: - kv   6:                            general.license str              = apache-2.0
+llama_model_loader: - kv   7:                   general.base_model.count u32              = 1
+llama_model_loader: - kv   8:                  general.base_model.0.name str              = Granite 4.0 H 350m
+llama_model_loader: - kv   9:          general.base_model.0.organization str              = Ibm Granite
+llama_model_loader: - kv  10:              general.base_model.0.repo_url str              = https://huggingface.co/ibm-granite/gr...
+llama_model_loader: - kv  11:                               general.tags arr[str,3]       = ["language", "unsloth", "granite-4.0"]
+llama_model_loader: - kv  12:                  granitehybrid.block_count u32              = 32
+llama_model_loader: - kv  13:               granitehybrid.context_length u32              = 1048576
+llama_model_loader: - kv  14:             granitehybrid.embedding_length u32              = 768
+llama_model_loader: - kv  15:          granitehybrid.feed_forward_length u32              = 2048
+llama_model_loader: - kv  16:         granitehybrid.attention.head_count u32              = 12
+llama_model_loader: - kv  17:      granitehybrid.attention.head_count_kv arr[i32,32]      = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, ...
+llama_model_loader: - kv  18:               granitehybrid.rope.freq_base f32              = 10000.000000
+llama_model_loader: - kv  19: granitehybrid.attention.layer_norm_rms_epsilon f32              = 0.000010
+llama_model_loader: - kv  20:                 granitehybrid.expert_count u32              = 0
+llama_model_loader: - kv  21:            granitehybrid.expert_used_count u32              = 0
+llama_model_loader: - kv  22:                   granitehybrid.vocab_size u32              = 100352
+llama_model_loader: - kv  23:         granitehybrid.rope.dimension_count u32              = 64
+llama_model_loader: - kv  24:              granitehybrid.attention.scale f32              = 0.015625
+llama_model_loader: - kv  25:              granitehybrid.embedding_scale f32              = 12.000000
+llama_model_loader: - kv  26:               granitehybrid.residual_scale f32              = 0.246000
+llama_model_loader: - kv  27:                  granitehybrid.logit_scale f32              = 3.000000
+llama_model_loader: - kv  28: granitehybrid.expert_shared_feed_forward_length u32              = 2048
+llama_model_loader: - kv  29:              granitehybrid.ssm.conv_kernel u32              = 4
+llama_model_loader: - kv  30:               granitehybrid.ssm.state_size u32              = 128
+llama_model_loader: - kv  31:              granitehybrid.ssm.group_count u32              = 1
+llama_model_loader: - kv  32:               granitehybrid.ssm.inner_size u32              = 1536
+llama_model_loader: - kv  33:           granitehybrid.ssm.time_step_rank u32              = 48
+llama_model_loader: - kv  34:       granitehybrid.rope.scaling.finetuned bool             = false
+llama_model_loader: - kv  35:                       tokenizer.ggml.model str              = gpt2
+llama_model_loader: - kv  36:                         tokenizer.ggml.pre str              = dbrx
+llama_model_loader: - kv  37:                      tokenizer.ggml.tokens arr[str,100352]  = ["!", "\"", "#", "$", "%", "&", "'", ...
+llama_model_loader: - kv  38:                  tokenizer.ggml.token_type arr[i32,100352]  = [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...
+llama_model_loader: - kv  39:                      tokenizer.ggml.merges arr[str,100000]  = ["Ġ Ġ", "ĠĠ ĠĠ", "i n", "Ġ t",...
+llama_model_loader: - kv  40:                tokenizer.ggml.bos_token_id u32              = 100257
+llama_model_loader: - kv  41:                tokenizer.ggml.eos_token_id u32              = 100257
+llama_model_loader: - kv  42:            tokenizer.ggml.unknown_token_id u32              = 100269
+llama_model_loader: - kv  43:            tokenizer.ggml.padding_token_id u32              = 100256
+llama_model_loader: - kv  44:               tokenizer.ggml.add_bos_token bool             = false
+llama_model_loader: - kv  45:                    tokenizer.chat_template str              = {%- set tools_system_message_prefix =...
+llama_model_loader: - kv  46:               general.quantization_version u32              = 2
+llama_model_loader: - kv  47:                          general.file_type u32              = 25
+llama_model_loader: - type  f32:  233 tensors
+llama_model_loader: - type q5_K:  113 tensors
+llama_model_loader: - type iq4_nl:   56 tensors
+print_info: file format = GGUF V3 (latest)
+print_info: file type   = IQ4_NL - 4.5 bpw
+print_info: file size   = 211.65 MiB (5.22 BPW) 
+load: printing all EOG tokens:
+load:   - 100257 ('<|end_of_text|>')
+load:   - 100261 ('<|fim_pad|>')
+load: special tokens cache size = 96
+load: token to piece cache size = 0.6152 MB
+print_info: arch             = granitehybrid
+print_info: vocab_only       = 0
+print_info: n_ctx_train      = 1048576
+print_info: n_embd           = 768
+print_info: n_embd_inp       = 768
+print_info: n_layer          = 32
+print_info: n_head           = 12
+print_info: n_head_kv        = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0]
+print_info: n_rot            = 64
+print_info: n_swa            = 0
+print_info: is_swa_any       = 0
+print_info: n_embd_head_k    = 64
+print_info: n_embd_head_v    = 64
+print_info: n_gqa            = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 3, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0]
+print_info: n_embd_k_gqa     = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 256, 0, 0, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 0, 0]
+print_info: n_embd_v_gqa     = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 256, 0, 0, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 0, 0]
+print_info: f_norm_eps       = 0.0e+00
+print_info: f_norm_rms_eps   = 1.0e-05
+print_info: f_clamp_kqv      = 0.0e+00
+print_info: f_max_alibi_bias = 0.0e+00
+print_info: f_logit_scale    = 3.0e+00
+print_info: f_attn_scale     = 1.6e-02
+print_info: n_ff             = 2048
+print_info: n_expert         = 0
+print_info: n_expert_used    = 0
+print_info: n_expert_groups  = 0
+print_info: n_group_used     = 0
+print_info: causal attn      = 1
+print_info: pooling type     = 0
+print_info: rope type        = 0
+print_info: rope scaling     = linear
+print_info: freq_base_train  = 10000.0
+print_info: freq_scale_train = 1
+print_info: n_ctx_orig_yarn  = 1048576
+print_info: rope_finetuned   = unknown
+print_info: ssm_d_conv       = 4
+print_info: ssm_d_inner      = 1536
+print_info: ssm_d_state      = 128
+print_info: ssm_dt_rank      = 48
+print_info: ssm_n_group      = 1
+print_info: ssm_dt_b_c_rms   = 0
+print_info: model type       = 350M
+print_info: model params     = 340.33 M
+print_info: general.name     = Granite 4.0 H 350m Unsloth
+print_info: f_embedding_scale = 12.000000
+print_info: f_residual_scale  = 0.246000
+print_info: f_attention_scale = 0.015625
+print_info: n_ff_shexp        = 2048
+print_info: vocab type       = BPE
+print_info: n_vocab          = 100352
+print_info: n_merges         = 100000
+print_info: BOS token        = 100257 '<|end_of_text|>'
+print_info: EOS token        = 100257 '<|end_of_text|>'
+print_info: EOT token        = 100257 '<|end_of_text|>'
+print_info: UNK token        = 100269 '<|unk|>'
+print_info: PAD token        = 100256 '<|pad|>'
+print_info: LF token         = 198 'Ċ'
+print_info: FIM PRE token    = 100258 '<|fim_prefix|>'
+print_info: FIM SUF token    = 100260 '<|fim_suffix|>'
+print_info: FIM MID token    = 100259 '<|fim_middle|>'
+print_info: FIM PAD token    = 100261 '<|fim_pad|>'
+print_info: EOG token        = 100257 '<|end_of_text|>'
+print_info: EOG token        = 100261 '<|fim_pad|>'
+print_info: max token length = 256
+load_tensors: loading model tensors, this can take a while... (mmap = true)
+load_tensors: offloading 20 repeating layers to GPU
+load_tensors: offloaded 20/33 layers to GPU
+load_tensors:   CPU_Mapped model buffer size =   111.47 MiB
+load_tensors:        CUDA0 model buffer size =    49.58 MiB
+load_tensors:        CUDA1 model buffer size =    50.62 MiB
+..............................................................................
+llama_context: constructing llama_context
+llama_context: n_seq_max     = 1
+llama_context: n_ctx         = 2048
+llama_context: n_ctx_seq     = 2048
+llama_context: n_batch       = 2048
+llama_context: n_ubatch      = 512
+llama_context: causal_attn   = 1
+llama_context: flash_attn    = auto
+llama_context: kv_unified    = false
+llama_context: freq_base     = 10000.0
+llama_context: freq_scale    = 1
+llama_context: n_ctx_seq (2048) < n_ctx_train (1048576) -- the full capacity of the model will not be utilized
+llama_context:        CPU  output buffer size =     0.38 MiB
+llama_kv_cache:        CPU KV buffer size =     2.00 MiB
+llama_kv_cache:      CUDA0 KV buffer size =     4.00 MiB
+llama_kv_cache:      CUDA1 KV buffer size =     2.00 MiB
+llama_kv_cache: size =    8.00 MiB (  2048 cells,   4 layers,  1/1 seqs), K (f16):    4.00 MiB, V (f16):    4.00 MiB
+llama_memory_recurrent:        CPU RS buffer size =     8.48 MiB
+llama_memory_recurrent:      CUDA0 RS buffer size =     6.16 MiB
+llama_memory_recurrent:      CUDA1 RS buffer size =     6.93 MiB
+llama_memory_recurrent: size =   21.57 MiB (     1 cells,  32 layers,  1 seqs), R (f32):    0.57 MiB, S (f32):   21.00 MiB
+llama_context: Flash Attention was auto, set to enabled
+llama_context:      CUDA0 compute buffer size =   248.03 MiB
+llama_context:      CUDA1 compute buffer size =    22.39 MiB
+llama_context:  CUDA_Host compute buffer size =    18.34 MiB
+llama_context: graph nodes  = 1815
+llama_context: graph splits = 182 (with bs=512), 41 (with bs=1)
+common_init_from_params: added <|end_of_text|> logit bias = -inf
+common_init_from_params: added <|fim_pad|> logit bias = -inf
+common_init_from_params: setting dry_penalty_last_n to ctx_size = 2048
+common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable)
+
+system_info: n_threads = 16 (n_threads_batch = 16) / 32 | CUDA : ARCHS = 860 | USE_GRAPHS = 1 | PEER_MAX_BATCH_SIZE = 128 | CPU : SSE3 = 1 | SSSE3 = 1 | AVX = 1 | AVX2 = 1 | F16C = 1 | FMA = 1 | BMI2 = 1 | AVX512 = 1 | AVX512_VBMI = 1 | AVX512_VNNI = 1 | AVX512_BF16 = 1 | LLAMAFILE = 1 | OPENMP = 1 | REPACK = 1 | 
+perplexity: tokenizing the input ..
+perplexity: tokenization took 39.998 ms
+perplexity: calculating perplexity over 14 chunks, n_ctx=2048, batch_size=2048, n_seq=1
+perplexity: 0.51 seconds per pass - ETA 0.12 minutes
+[1]21.4253,[2]26.4285,[3]26.9843,[4]24.3892,[5]24.3760,[6]21.7415,[7]21.2597,[8]21.1013,[9]21.7698,[10]21.7031,[11]21.6025,[12]21.8384,[13]21.9628,[14]22.0027,
+Final estimate: PPL = 22.0027 +/- 0.58551
+
+llama_perf_context_print:        load time =     197.73 ms
+llama_perf_context_print: prompt eval time =    4847.06 ms / 28672 tokens (    0.17 ms per token,  5915.33 tokens per second)
+llama_perf_context_print:        eval time =       0.00 ms /     1 runs   (    0.00 ms per token,      inf tokens per second)
+llama_perf_context_print:       total time =    5108.94 ms / 28673 tokens
+llama_perf_context_print:    graphs reused =          0
+llama_memory_breakdown_print: | memory breakdown [MiB] | total    free    self   model   context   compute    unaccounted |
+llama_memory_breakdown_print: |   - CUDA0 (RTX 3090)   | 24115 = 20554 + ( 315 =    49 +      10 +     256) +        3244 |
+llama_memory_breakdown_print: |   - CUDA1 (RTX 3090)   | 24124 = 23431 + (  81 =    50 +       8 +      22) +         610 |
+llama_memory_breakdown_print: |   - Host               |                   140 =   111 +      10 +      18                |

Benchmarks/DataCollection/granite-4.0-h-350m-unsloth-iq4_nl-attn_kv_Q5_K-attn_output_Q5_K-attn_q_Q5_K-embeddings_Q5_K-ffn_down_Q5_K-ffn_up_gate_Q5_K/perplexity_math.log

@@ -0,0 +1,189 @@
+ggml_cuda_init: GGML_CUDA_FORCE_MMQ:    no
+ggml_cuda_init: GGML_CUDA_FORCE_CUBLAS: no
+ggml_cuda_init: found 2 CUDA devices:
+  Device 0: NVIDIA GeForce RTX 3090, compute capability 8.6, VMM: yes
+  Device 1: NVIDIA GeForce RTX 3090, compute capability 8.6, VMM: yes
+build: 7040 (92bb442ad) with cc (Ubuntu 13.3.0-6ubuntu2~24.04) 13.3.0 for x86_64-linux-gnu
+llama_model_load_from_file_impl: using device CUDA0 (NVIDIA GeForce RTX 3090) (0000:01:00.0) - 20943 MiB free
+llama_model_load_from_file_impl: using device CUDA1 (NVIDIA GeForce RTX 3090) (0000:03:00.0) - 23581 MiB free
+llama_model_loader: loaded meta data with 48 key-value pairs and 402 tensors from /mnt/world8/AI/ToBench/granite-4.0-h-350m-unsloth/Magic_Quant/GGUF/dc_round0_granite-4.0-h-350m-unsloth-iq4_nl-attn_kv_Q5_K-attn_output_Q5_K-attn_q_Q5_K-embeddings_Q5_K-ffn_down_Q5_K-ffn_up_gate_Q5_K.gguf (version GGUF V3 (latest))
+llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output.
+llama_model_loader: - kv   0:                       general.architecture str              = granitehybrid
+llama_model_loader: - kv   1:                               general.type str              = model
+llama_model_loader: - kv   2:                               general.name str              = Granite 4.0 H 350m Unsloth
+llama_model_loader: - kv   3:                           general.finetune str              = unsloth
+llama_model_loader: - kv   4:                           general.basename str              = granite-4.0-h
+llama_model_loader: - kv   5:                         general.size_label str              = 350M
+llama_model_loader: - kv   6:                            general.license str              = apache-2.0
+llama_model_loader: - kv   7:                   general.base_model.count u32              = 1
+llama_model_loader: - kv   8:                  general.base_model.0.name str              = Granite 4.0 H 350m
+llama_model_loader: - kv   9:          general.base_model.0.organization str              = Ibm Granite
+llama_model_loader: - kv  10:              general.base_model.0.repo_url str              = https://huggingface.co/ibm-granite/gr...
+llama_model_loader: - kv  11:                               general.tags arr[str,3]       = ["language", "unsloth", "granite-4.0"]
+llama_model_loader: - kv  12:                  granitehybrid.block_count u32              = 32
+llama_model_loader: - kv  13:               granitehybrid.context_length u32              = 1048576
+llama_model_loader: - kv  14:             granitehybrid.embedding_length u32              = 768
+llama_model_loader: - kv  15:          granitehybrid.feed_forward_length u32              = 2048
+llama_model_loader: - kv  16:         granitehybrid.attention.head_count u32              = 12
+llama_model_loader: - kv  17:      granitehybrid.attention.head_count_kv arr[i32,32]      = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, ...
+llama_model_loader: - kv  18:               granitehybrid.rope.freq_base f32              = 10000.000000
+llama_model_loader: - kv  19: granitehybrid.attention.layer_norm_rms_epsilon f32              = 0.000010
+llama_model_loader: - kv  20:                 granitehybrid.expert_count u32              = 0
+llama_model_loader: - kv  21:            granitehybrid.expert_used_count u32              = 0
+llama_model_loader: - kv  22:                   granitehybrid.vocab_size u32              = 100352
+llama_model_loader: - kv  23:         granitehybrid.rope.dimension_count u32              = 64
+llama_model_loader: - kv  24:              granitehybrid.attention.scale f32              = 0.015625
+llama_model_loader: - kv  25:              granitehybrid.embedding_scale f32              = 12.000000
+llama_model_loader: - kv  26:               granitehybrid.residual_scale f32              = 0.246000
+llama_model_loader: - kv  27:                  granitehybrid.logit_scale f32              = 3.000000
+llama_model_loader: - kv  28: granitehybrid.expert_shared_feed_forward_length u32              = 2048
+llama_model_loader: - kv  29:              granitehybrid.ssm.conv_kernel u32              = 4
+llama_model_loader: - kv  30:               granitehybrid.ssm.state_size u32              = 128
+llama_model_loader: - kv  31:              granitehybrid.ssm.group_count u32              = 1
+llama_model_loader: - kv  32:               granitehybrid.ssm.inner_size u32              = 1536
+llama_model_loader: - kv  33:           granitehybrid.ssm.time_step_rank u32              = 48
+llama_model_loader: - kv  34:       granitehybrid.rope.scaling.finetuned bool             = false
+llama_model_loader: - kv  35:                       tokenizer.ggml.model str              = gpt2
+llama_model_loader: - kv  36:                         tokenizer.ggml.pre str              = dbrx
+llama_model_loader: - kv  37:                      tokenizer.ggml.tokens arr[str,100352]  = ["!", "\"", "#", "$", "%", "&", "'", ...
+llama_model_loader: - kv  38:                  tokenizer.ggml.token_type arr[i32,100352]  = [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...
+llama_model_loader: - kv  39:                      tokenizer.ggml.merges arr[str,100000]  = ["Ġ Ġ", "ĠĠ ĠĠ", "i n", "Ġ t",...
+llama_model_loader: - kv  40:                tokenizer.ggml.bos_token_id u32              = 100257
+llama_model_loader: - kv  41:                tokenizer.ggml.eos_token_id u32              = 100257
+llama_model_loader: - kv  42:            tokenizer.ggml.unknown_token_id u32              = 100269
+llama_model_loader: - kv  43:            tokenizer.ggml.padding_token_id u32              = 100256
+llama_model_loader: - kv  44:               tokenizer.ggml.add_bos_token bool             = false
+llama_model_loader: - kv  45:                    tokenizer.chat_template str              = {%- set tools_system_message_prefix =...
+llama_model_loader: - kv  46:               general.quantization_version u32              = 2
+llama_model_loader: - kv  47:                          general.file_type u32              = 25
+llama_model_loader: - type  f32:  233 tensors
+llama_model_loader: - type q5_K:  113 tensors
+llama_model_loader: - type iq4_nl:   56 tensors
+print_info: file format = GGUF V3 (latest)
+print_info: file type   = IQ4_NL - 4.5 bpw
+print_info: file size   = 211.65 MiB (5.22 BPW) 
+load: printing all EOG tokens:
+load:   - 100257 ('<|end_of_text|>')
+load:   - 100261 ('<|fim_pad|>')
+load: special tokens cache size = 96
+load: token to piece cache size = 0.6152 MB
+print_info: arch             = granitehybrid
+print_info: vocab_only       = 0
+print_info: n_ctx_train      = 1048576
+print_info: n_embd           = 768
+print_info: n_embd_inp       = 768
+print_info: n_layer          = 32
+print_info: n_head           = 12
+print_info: n_head_kv        = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0]
+print_info: n_rot            = 64
+print_info: n_swa            = 0
+print_info: is_swa_any       = 0
+print_info: n_embd_head_k    = 64
+print_info: n_embd_head_v    = 64
+print_info: n_gqa            = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 3, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0]
+print_info: n_embd_k_gqa     = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 256, 0, 0, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 0, 0]
+print_info: n_embd_v_gqa     = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 256, 0, 0, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 0, 0]
+print_info: f_norm_eps       = 0.0e+00
+print_info: f_norm_rms_eps   = 1.0e-05
+print_info: f_clamp_kqv      = 0.0e+00
+print_info: f_max_alibi_bias = 0.0e+00
+print_info: f_logit_scale    = 3.0e+00
+print_info: f_attn_scale     = 1.6e-02
+print_info: n_ff             = 2048
+print_info: n_expert         = 0
+print_info: n_expert_used    = 0
+print_info: n_expert_groups  = 0
+print_info: n_group_used     = 0
+print_info: causal attn      = 1
+print_info: pooling type     = 0
+print_info: rope type        = 0
+print_info: rope scaling     = linear
+print_info: freq_base_train  = 10000.0
+print_info: freq_scale_train = 1
+print_info: n_ctx_orig_yarn  = 1048576
+print_info: rope_finetuned   = unknown
+print_info: ssm_d_conv       = 4
+print_info: ssm_d_inner      = 1536
+print_info: ssm_d_state      = 128
+print_info: ssm_dt_rank      = 48
+print_info: ssm_n_group      = 1
+print_info: ssm_dt_b_c_rms   = 0
+print_info: model type       = 350M
+print_info: model params     = 340.33 M
+print_info: general.name     = Granite 4.0 H 350m Unsloth
+print_info: f_embedding_scale = 12.000000
+print_info: f_residual_scale  = 0.246000
+print_info: f_attention_scale = 0.015625
+print_info: n_ff_shexp        = 2048
+print_info: vocab type       = BPE
+print_info: n_vocab          = 100352
+print_info: n_merges         = 100000
+print_info: BOS token        = 100257 '<|end_of_text|>'
+print_info: EOS token        = 100257 '<|end_of_text|>'
+print_info: EOT token        = 100257 '<|end_of_text|>'
+print_info: UNK token        = 100269 '<|unk|>'
+print_info: PAD token        = 100256 '<|pad|>'
+print_info: LF token         = 198 'Ċ'
+print_info: FIM PRE token    = 100258 '<|fim_prefix|>'
+print_info: FIM SUF token    = 100260 '<|fim_suffix|>'
+print_info: FIM MID token    = 100259 '<|fim_middle|>'
+print_info: FIM PAD token    = 100261 '<|fim_pad|>'
+print_info: EOG token        = 100257 '<|end_of_text|>'
+print_info: EOG token        = 100261 '<|fim_pad|>'
+print_info: max token length = 256
+load_tensors: loading model tensors, this can take a while... (mmap = true)
+load_tensors: offloading 20 repeating layers to GPU
+load_tensors: offloaded 20/33 layers to GPU
+load_tensors:   CPU_Mapped model buffer size =   111.47 MiB
+load_tensors:        CUDA0 model buffer size =    49.58 MiB
+load_tensors:        CUDA1 model buffer size =    50.62 MiB
+..............................................................................
+llama_context: constructing llama_context
+llama_context: n_seq_max     = 1
+llama_context: n_ctx         = 2048
+llama_context: n_ctx_seq     = 2048
+llama_context: n_batch       = 2048
+llama_context: n_ubatch      = 512
+llama_context: causal_attn   = 1
+llama_context: flash_attn    = auto
+llama_context: kv_unified    = false
+llama_context: freq_base     = 10000.0
+llama_context: freq_scale    = 1
+llama_context: n_ctx_seq (2048) < n_ctx_train (1048576) -- the full capacity of the model will not be utilized
+llama_context:        CPU  output buffer size =     0.38 MiB
+llama_kv_cache:        CPU KV buffer size =     2.00 MiB
+llama_kv_cache:      CUDA0 KV buffer size =     4.00 MiB
+llama_kv_cache:      CUDA1 KV buffer size =     2.00 MiB
+llama_kv_cache: size =    8.00 MiB (  2048 cells,   4 layers,  1/1 seqs), K (f16):    4.00 MiB, V (f16):    4.00 MiB
+llama_memory_recurrent:        CPU RS buffer size =     8.48 MiB
+llama_memory_recurrent:      CUDA0 RS buffer size =     6.16 MiB
+llama_memory_recurrent:      CUDA1 RS buffer size =     6.93 MiB
+llama_memory_recurrent: size =   21.57 MiB (     1 cells,  32 layers,  1 seqs), R (f32):    0.57 MiB, S (f32):   21.00 MiB
+llama_context: Flash Attention was auto, set to enabled
+llama_context:      CUDA0 compute buffer size =   248.03 MiB
+llama_context:      CUDA1 compute buffer size =    22.39 MiB
+llama_context:  CUDA_Host compute buffer size =    18.34 MiB
+llama_context: graph nodes  = 1815
+llama_context: graph splits = 182 (with bs=512), 41 (with bs=1)
+common_init_from_params: added <|end_of_text|> logit bias = -inf
+common_init_from_params: added <|fim_pad|> logit bias = -inf
+common_init_from_params: setting dry_penalty_last_n to ctx_size = 2048
+common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable)
+
+system_info: n_threads = 16 (n_threads_batch = 16) / 32 | CUDA : ARCHS = 860 | USE_GRAPHS = 1 | PEER_MAX_BATCH_SIZE = 128 | CPU : SSE3 = 1 | SSSE3 = 1 | AVX = 1 | AVX2 = 1 | F16C = 1 | FMA = 1 | BMI2 = 1 | AVX512 = 1 | AVX512_VBMI = 1 | AVX512_VNNI = 1 | AVX512_BF16 = 1 | LLAMAFILE = 1 | OPENMP = 1 | REPACK = 1 | 
+perplexity: tokenizing the input ..
+perplexity: tokenization took 33.775 ms
+perplexity: calculating perplexity over 15 chunks, n_ctx=2048, batch_size=2048, n_seq=1
+perplexity: 0.48 seconds per pass - ETA 0.12 minutes
+[1]9.8507,[2]11.1604,[3]10.6487,[4]11.1219,[5]11.3188,[6]11.3450,[7]11.5402,[8]11.2207,[9]11.2589,[10]11.2869,[11]11.5587,[12]11.6424,[13]11.8141,[14]11.7505,[15]11.6501,
+Final estimate: PPL = 11.6501 +/- 0.27272
+
+llama_perf_context_print:        load time =     190.44 ms
+llama_perf_context_print: prompt eval time =    4881.13 ms / 30720 tokens (    0.16 ms per token,  6293.63 tokens per second)
+llama_perf_context_print:        eval time =       0.00 ms /     1 runs   (    0.00 ms per token,      inf tokens per second)
+llama_perf_context_print:       total time =    5146.83 ms / 30721 tokens
+llama_perf_context_print:    graphs reused =          0
+llama_memory_breakdown_print: | memory breakdown [MiB] | total    free    self   model   context   compute    unaccounted |
+llama_memory_breakdown_print: |   - CUDA0 (RTX 3090)   | 24115 = 20561 + ( 315 =    49 +      10 +     256) +        3237 |
+llama_memory_breakdown_print: |   - CUDA1 (RTX 3090)   | 24124 = 23431 + (  81 =    50 +       8 +      22) +         610 |
+llama_memory_breakdown_print: |   - Host               |                   140 =   111 +      10 +      18                |

Benchmarks/DataCollection/granite-4.0-h-350m-unsloth-iq4_nl-attn_kv_Q6_K-attn_output_Q6_K-attn_q_Q6_K-embeddings_Q6_K-ffn_down_Q6_K-ffn_up_gate_Q6_K/bench_metrics.json

@@ -0,0 +1,44 @@
+{
+  "raw_metrics": {
+    "llamabench": {
+      "backend": "CUDA",
+      "log_path": "granite-4.0-h-350m-unsloth-iq4_nl-attn_kv_Q6_K-attn_output_Q6_K-attn_q_Q6_K-embeddings_Q6_K-ffn_down_Q6_K-ffn_up_gate_Q6_K/llamabench.md",
+      "ngl": "35",
+      "raw_row": {
+        "backend": "CUDA",
+        "model": "granitehybrid 350M IQ4_NL - 4.5 bpw",
+        "ngl": "35",
+        "params": "340.33 M",
+        "size": "241.33 MiB",
+        "t/s": "1569.41 \u00b1 64.68",
+        "test": "pp8",
+        "tps_value": 1569.41
+      },
+      "test": "pp8",
+      "tps": 1569.41
+    },
+    "perplexity": {
+      "code": {
+        "log_path": "granite-4.0-h-350m-unsloth-iq4_nl-attn_kv_Q6_K-attn_output_Q6_K-attn_q_Q6_K-embeddings_Q6_K-ffn_down_Q6_K-ffn_up_gate_Q6_K/perplexity_code.log",
+        "ppl": 2.0107,
+        "ppl_error": 0.01851
+      },
+      "general": {
+        "log_path": "granite-4.0-h-350m-unsloth-iq4_nl-attn_kv_Q6_K-attn_output_Q6_K-attn_q_Q6_K-embeddings_Q6_K-ffn_down_Q6_K-ffn_up_gate_Q6_K/perplexity_general.log",
+        "ppl": 20.5806,
+        "ppl_error": 0.54355
+      },
+      "math": {
+        "log_path": "granite-4.0-h-350m-unsloth-iq4_nl-attn_kv_Q6_K-attn_output_Q6_K-attn_q_Q6_K-embeddings_Q6_K-ffn_down_Q6_K-ffn_up_gate_Q6_K/perplexity_math.log",
+        "ppl": 10.5642,
+        "ppl_error": 0.2398
+      }
+    }
+  },
+  "summary": {
+    "avg_prec_loss_pct": 6.3495,
+    "bench_tps": 1569.41,
+    "file_size_bytes": 256623552,
+    "file_size_gb": 0.24
+  }
+}

Benchmarks/DataCollection/granite-4.0-h-350m-unsloth-iq4_nl-attn_kv_Q6_K-attn_output_Q6_K-attn_q_Q6_K-embeddings_Q6_K-ffn_down_Q6_K-ffn_up_gate_Q6_K/llamabench.md

@@ -0,0 +1,11 @@
+ggml_cuda_init: GGML_CUDA_FORCE_MMQ:    no
+ggml_cuda_init: GGML_CUDA_FORCE_CUBLAS: no
+ggml_cuda_init: found 2 CUDA devices:
+  Device 0: NVIDIA GeForce RTX 3090, compute capability 8.6, VMM: yes
+  Device 1: NVIDIA GeForce RTX 3090, compute capability 8.6, VMM: yes
+| model                          |       size |     params | backend    | ngl |            test |                  t/s |
+| ------------------------------ | ---------: | ---------: | ---------- | --: | --------------: | -------------------: |
+| granitehybrid 350M IQ4_NL - 4.5 bpw | 241.33 MiB |   340.33 M | CUDA       |  35 |             pp8 |      1569.41 ± 64.68 |
+| granitehybrid 350M IQ4_NL - 4.5 bpw | 241.33 MiB |   340.33 M | CUDA       |  35 |           tg128 |        323.25 ± 8.78 |
+
+build: 92bb442ad (7040)

Benchmarks/DataCollection/granite-4.0-h-350m-unsloth-iq4_nl-attn_kv_Q6_K-attn_output_Q6_K-attn_q_Q6_K-embeddings_Q6_K-ffn_down_Q6_K-ffn_up_gate_Q6_K/perplexity_code.log

@@ -0,0 +1,189 @@
+ggml_cuda_init: GGML_CUDA_FORCE_MMQ:    no
+ggml_cuda_init: GGML_CUDA_FORCE_CUBLAS: no
+ggml_cuda_init: found 2 CUDA devices:
+  Device 0: NVIDIA GeForce RTX 3090, compute capability 8.6, VMM: yes
+  Device 1: NVIDIA GeForce RTX 3090, compute capability 8.6, VMM: yes
+build: 7040 (92bb442ad) with cc (Ubuntu 13.3.0-6ubuntu2~24.04) 13.3.0 for x86_64-linux-gnu
+llama_model_load_from_file_impl: using device CUDA0 (NVIDIA GeForce RTX 3090) (0000:01:00.0) - 20941 MiB free
+llama_model_load_from_file_impl: using device CUDA1 (NVIDIA GeForce RTX 3090) (0000:03:00.0) - 23581 MiB free
+llama_model_loader: loaded meta data with 48 key-value pairs and 402 tensors from /mnt/world8/AI/ToBench/granite-4.0-h-350m-unsloth/Magic_Quant/GGUF/dc_round0_granite-4.0-h-350m-unsloth-iq4_nl-attn_kv_Q6_K-attn_output_Q6_K-attn_q_Q6_K-embeddings_Q6_K-ffn_down_Q6_K-ffn_up_gate_Q6_K.gguf (version GGUF V3 (latest))
+llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output.
+llama_model_loader: - kv   0:                       general.architecture str              = granitehybrid
+llama_model_loader: - kv   1:                               general.type str              = model
+llama_model_loader: - kv   2:                               general.name str              = Granite 4.0 H 350m Unsloth
+llama_model_loader: - kv   3:                           general.finetune str              = unsloth
+llama_model_loader: - kv   4:                           general.basename str              = granite-4.0-h
+llama_model_loader: - kv   5:                         general.size_label str              = 350M
+llama_model_loader: - kv   6:                            general.license str              = apache-2.0
+llama_model_loader: - kv   7:                   general.base_model.count u32              = 1
+llama_model_loader: - kv   8:                  general.base_model.0.name str              = Granite 4.0 H 350m
+llama_model_loader: - kv   9:          general.base_model.0.organization str              = Ibm Granite
+llama_model_loader: - kv  10:              general.base_model.0.repo_url str              = https://huggingface.co/ibm-granite/gr...
+llama_model_loader: - kv  11:                               general.tags arr[str,3]       = ["language", "unsloth", "granite-4.0"]
+llama_model_loader: - kv  12:                  granitehybrid.block_count u32              = 32
+llama_model_loader: - kv  13:               granitehybrid.context_length u32              = 1048576
+llama_model_loader: - kv  14:             granitehybrid.embedding_length u32              = 768
+llama_model_loader: - kv  15:          granitehybrid.feed_forward_length u32              = 2048
+llama_model_loader: - kv  16:         granitehybrid.attention.head_count u32              = 12
+llama_model_loader: - kv  17:      granitehybrid.attention.head_count_kv arr[i32,32]      = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, ...
+llama_model_loader: - kv  18:               granitehybrid.rope.freq_base f32              = 10000.000000
+llama_model_loader: - kv  19: granitehybrid.attention.layer_norm_rms_epsilon f32              = 0.000010
+llama_model_loader: - kv  20:                 granitehybrid.expert_count u32              = 0
+llama_model_loader: - kv  21:            granitehybrid.expert_used_count u32              = 0
+llama_model_loader: - kv  22:                   granitehybrid.vocab_size u32              = 100352
+llama_model_loader: - kv  23:         granitehybrid.rope.dimension_count u32              = 64
+llama_model_loader: - kv  24:              granitehybrid.attention.scale f32              = 0.015625
+llama_model_loader: - kv  25:              granitehybrid.embedding_scale f32              = 12.000000
+llama_model_loader: - kv  26:               granitehybrid.residual_scale f32              = 0.246000
+llama_model_loader: - kv  27:                  granitehybrid.logit_scale f32              = 3.000000
+llama_model_loader: - kv  28: granitehybrid.expert_shared_feed_forward_length u32              = 2048
+llama_model_loader: - kv  29:              granitehybrid.ssm.conv_kernel u32              = 4
+llama_model_loader: - kv  30:               granitehybrid.ssm.state_size u32              = 128
+llama_model_loader: - kv  31:              granitehybrid.ssm.group_count u32              = 1
+llama_model_loader: - kv  32:               granitehybrid.ssm.inner_size u32              = 1536
+llama_model_loader: - kv  33:           granitehybrid.ssm.time_step_rank u32              = 48
+llama_model_loader: - kv  34:       granitehybrid.rope.scaling.finetuned bool             = false
+llama_model_loader: - kv  35:                       tokenizer.ggml.model str              = gpt2
+llama_model_loader: - kv  36:                         tokenizer.ggml.pre str              = dbrx
+llama_model_loader: - kv  37:                      tokenizer.ggml.tokens arr[str,100352]  = ["!", "\"", "#", "$", "%", "&", "'", ...
+llama_model_loader: - kv  38:                  tokenizer.ggml.token_type arr[i32,100352]  = [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...
+llama_model_loader: - kv  39:                      tokenizer.ggml.merges arr[str,100000]  = ["Ġ Ġ", "ĠĠ ĠĠ", "i n", "Ġ t",...
+llama_model_loader: - kv  40:                tokenizer.ggml.bos_token_id u32              = 100257
+llama_model_loader: - kv  41:                tokenizer.ggml.eos_token_id u32              = 100257
+llama_model_loader: - kv  42:            tokenizer.ggml.unknown_token_id u32              = 100269
+llama_model_loader: - kv  43:            tokenizer.ggml.padding_token_id u32              = 100256
+llama_model_loader: - kv  44:               tokenizer.ggml.add_bos_token bool             = false
+llama_model_loader: - kv  45:                    tokenizer.chat_template str              = {%- set tools_system_message_prefix =...
+llama_model_loader: - kv  46:               general.quantization_version u32              = 2
+llama_model_loader: - kv  47:                          general.file_type u32              = 25
+llama_model_loader: - type  f32:  233 tensors
+llama_model_loader: - type q6_K:  113 tensors
+llama_model_loader: - type iq4_nl:   56 tensors
+print_info: file format = GGUF V3 (latest)
+print_info: file type   = IQ4_NL - 4.5 bpw
+print_info: file size   = 241.33 MiB (5.95 BPW) 
+load: printing all EOG tokens:
+load:   - 100257 ('<|end_of_text|>')
+load:   - 100261 ('<|fim_pad|>')
+load: special tokens cache size = 96
+load: token to piece cache size = 0.6152 MB
+print_info: arch             = granitehybrid
+print_info: vocab_only       = 0
+print_info: n_ctx_train      = 1048576
+print_info: n_embd           = 768
+print_info: n_embd_inp       = 768
+print_info: n_layer          = 32
+print_info: n_head           = 12
+print_info: n_head_kv        = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0]
+print_info: n_rot            = 64
+print_info: n_swa            = 0
+print_info: is_swa_any       = 0
+print_info: n_embd_head_k    = 64
+print_info: n_embd_head_v    = 64
+print_info: n_gqa            = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 3, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0]
+print_info: n_embd_k_gqa     = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 256, 0, 0, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 0, 0]
+print_info: n_embd_v_gqa     = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 256, 0, 0, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 0, 0]
+print_info: f_norm_eps       = 0.0e+00
+print_info: f_norm_rms_eps   = 1.0e-05
+print_info: f_clamp_kqv      = 0.0e+00
+print_info: f_max_alibi_bias = 0.0e+00
+print_info: f_logit_scale    = 3.0e+00
+print_info: f_attn_scale     = 1.6e-02
+print_info: n_ff             = 2048
+print_info: n_expert         = 0
+print_info: n_expert_used    = 0
+print_info: n_expert_groups  = 0
+print_info: n_group_used     = 0
+print_info: causal attn      = 1
+print_info: pooling type     = 0
+print_info: rope type        = 0
+print_info: rope scaling     = linear
+print_info: freq_base_train  = 10000.0
+print_info: freq_scale_train = 1
+print_info: n_ctx_orig_yarn  = 1048576
+print_info: rope_finetuned   = unknown
+print_info: ssm_d_conv       = 4
+print_info: ssm_d_inner      = 1536
+print_info: ssm_d_state      = 128
+print_info: ssm_dt_rank      = 48
+print_info: ssm_n_group      = 1
+print_info: ssm_dt_b_c_rms   = 0
+print_info: model type       = 350M
+print_info: model params     = 340.33 M
+print_info: general.name     = Granite 4.0 H 350m Unsloth
+print_info: f_embedding_scale = 12.000000
+print_info: f_residual_scale  = 0.246000
+print_info: f_attention_scale = 0.015625
+print_info: n_ff_shexp        = 2048
+print_info: vocab type       = BPE
+print_info: n_vocab          = 100352
+print_info: n_merges         = 100000
+print_info: BOS token        = 100257 '<|end_of_text|>'
+print_info: EOS token        = 100257 '<|end_of_text|>'
+print_info: EOT token        = 100257 '<|end_of_text|>'
+print_info: UNK token        = 100269 '<|unk|>'
+print_info: PAD token        = 100256 '<|pad|>'
+print_info: LF token         = 198 'Ċ'
+print_info: FIM PRE token    = 100258 '<|fim_prefix|>'
+print_info: FIM SUF token    = 100260 '<|fim_suffix|>'
+print_info: FIM MID token    = 100259 '<|fim_middle|>'
+print_info: FIM PAD token    = 100261 '<|fim_pad|>'
+print_info: EOG token        = 100257 '<|end_of_text|>'
+print_info: EOG token        = 100261 '<|fim_pad|>'
+print_info: max token length = 256
+load_tensors: loading model tensors, this can take a while... (mmap = true)
+load_tensors: offloading 20 repeating layers to GPU
+load_tensors: offloaded 20/33 layers to GPU
+load_tensors:   CPU_Mapped model buffer size =   128.60 MiB
+load_tensors:        CUDA0 model buffer size =    55.96 MiB
+load_tensors:        CUDA1 model buffer size =    56.79 MiB
+.............................................................................
+llama_context: constructing llama_context
+llama_context: n_seq_max     = 1
+llama_context: n_ctx         = 2048
+llama_context: n_ctx_seq     = 2048
+llama_context: n_batch       = 2048
+llama_context: n_ubatch      = 512
+llama_context: causal_attn   = 1
+llama_context: flash_attn    = auto
+llama_context: kv_unified    = false
+llama_context: freq_base     = 10000.0
+llama_context: freq_scale    = 1
+llama_context: n_ctx_seq (2048) < n_ctx_train (1048576) -- the full capacity of the model will not be utilized
+llama_context:        CPU  output buffer size =     0.38 MiB
+llama_kv_cache:        CPU KV buffer size =     2.00 MiB
+llama_kv_cache:      CUDA0 KV buffer size =     4.00 MiB
+llama_kv_cache:      CUDA1 KV buffer size =     2.00 MiB
+llama_kv_cache: size =    8.00 MiB (  2048 cells,   4 layers,  1/1 seqs), K (f16):    4.00 MiB, V (f16):    4.00 MiB
+llama_memory_recurrent:        CPU RS buffer size =     8.48 MiB
+llama_memory_recurrent:      CUDA0 RS buffer size =     6.16 MiB
+llama_memory_recurrent:      CUDA1 RS buffer size =     6.93 MiB
+llama_memory_recurrent: size =   21.57 MiB (     1 cells,  32 layers,  1 seqs), R (f32):    0.57 MiB, S (f32):   21.00 MiB
+llama_context: Flash Attention was auto, set to enabled
+llama_context:      CUDA0 compute buffer size =   266.04 MiB
+llama_context:      CUDA1 compute buffer size =    22.39 MiB
+llama_context:  CUDA_Host compute buffer size =    18.34 MiB
+llama_context: graph nodes  = 1815
+llama_context: graph splits = 182 (with bs=512), 41 (with bs=1)
+common_init_from_params: added <|end_of_text|> logit bias = -inf
+common_init_from_params: added <|fim_pad|> logit bias = -inf
+common_init_from_params: setting dry_penalty_last_n to ctx_size = 2048
+common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable)
+
+system_info: n_threads = 16 (n_threads_batch = 16) / 32 | CUDA : ARCHS = 860 | USE_GRAPHS = 1 | PEER_MAX_BATCH_SIZE = 128 | CPU : SSE3 = 1 | SSSE3 = 1 | AVX = 1 | AVX2 = 1 | F16C = 1 | FMA = 1 | BMI2 = 1 | AVX512 = 1 | AVX512_VBMI = 1 | AVX512_VNNI = 1 | AVX512_BF16 = 1 | LLAMAFILE = 1 | OPENMP = 1 | REPACK = 1 | 
+perplexity: tokenizing the input ..
+perplexity: tokenization took 95.037 ms
+perplexity: calculating perplexity over 44 chunks, n_ctx=2048, batch_size=2048, n_seq=1
+perplexity: 0.50 seconds per pass - ETA 0.37 minutes
+[1]4.6321,[2]4.2656,[3]2.7047,[4]2.4944,[5]2.7698,[6]3.0211,[7]2.8498,[8]2.6333,[9]2.4073,[10]2.2221,[11]2.1999,[12]2.2305,[13]2.1339,[14]2.1093,[15]2.1499,[16]2.0793,[17]2.0515,[18]2.0707,[19]2.0293,[20]1.9917,[21]1.9569,[22]1.9412,[23]1.9704,[24]1.9419,[25]1.9603,[26]1.9258,[27]1.9127,[28]1.9043,[29]1.9527,[30]1.9697,[31]1.9694,[32]1.9432,[33]1.9673,[34]1.9590,[35]1.9391,[36]1.9714,[37]1.9785,[38]1.9761,[39]1.9989,[40]1.9963,[41]1.9878,[42]2.0132,[43]2.0220,[44]2.0107,
+Final estimate: PPL = 2.0107 +/- 0.01851
+
+llama_perf_context_print:        load time =     215.13 ms
+llama_perf_context_print: prompt eval time =   15051.26 ms / 90112 tokens (    0.17 ms per token,  5987.01 tokens per second)
+llama_perf_context_print:        eval time =       0.00 ms /     1 runs   (    0.00 ms per token,      inf tokens per second)
+llama_perf_context_print:       total time =   15846.66 ms / 90113 tokens
+llama_perf_context_print:    graphs reused =          0
+llama_memory_breakdown_print: | memory breakdown [MiB] | total    free    self   model   context   compute    unaccounted |
+llama_memory_breakdown_print: |   - CUDA0 (RTX 3090)   | 24115 = 20539 + ( 333 =    55 +      10 +     267) +        3242 |
+llama_memory_breakdown_print: |   - CUDA1 (RTX 3090)   | 24124 = 23425 + (  88 =    56 +       8 +      22) +         610 |
+llama_memory_breakdown_print: |   - Host               |                   157 =   128 +      10 +      18                |

Benchmarks/DataCollection/granite-4.0-h-350m-unsloth-iq4_nl-attn_kv_Q6_K-attn_output_Q6_K-attn_q_Q6_K-embeddings_Q6_K-ffn_down_Q6_K-ffn_up_gate_Q6_K/perplexity_general.log

@@ -0,0 +1,189 @@
+ggml_cuda_init: GGML_CUDA_FORCE_MMQ:    no
+ggml_cuda_init: GGML_CUDA_FORCE_CUBLAS: no
+ggml_cuda_init: found 2 CUDA devices:
+  Device 0: NVIDIA GeForce RTX 3090, compute capability 8.6, VMM: yes
+  Device 1: NVIDIA GeForce RTX 3090, compute capability 8.6, VMM: yes
+build: 7040 (92bb442ad) with cc (Ubuntu 13.3.0-6ubuntu2~24.04) 13.3.0 for x86_64-linux-gnu
+llama_model_load_from_file_impl: using device CUDA0 (NVIDIA GeForce RTX 3090) (0000:01:00.0) - 20940 MiB free
+llama_model_load_from_file_impl: using device CUDA1 (NVIDIA GeForce RTX 3090) (0000:03:00.0) - 23581 MiB free
+llama_model_loader: loaded meta data with 48 key-value pairs and 402 tensors from /mnt/world8/AI/ToBench/granite-4.0-h-350m-unsloth/Magic_Quant/GGUF/dc_round0_granite-4.0-h-350m-unsloth-iq4_nl-attn_kv_Q6_K-attn_output_Q6_K-attn_q_Q6_K-embeddings_Q6_K-ffn_down_Q6_K-ffn_up_gate_Q6_K.gguf (version GGUF V3 (latest))
+llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output.
+llama_model_loader: - kv   0:                       general.architecture str              = granitehybrid
+llama_model_loader: - kv   1:                               general.type str              = model
+llama_model_loader: - kv   2:                               general.name str              = Granite 4.0 H 350m Unsloth
+llama_model_loader: - kv   3:                           general.finetune str              = unsloth
+llama_model_loader: - kv   4:                           general.basename str              = granite-4.0-h
+llama_model_loader: - kv   5:                         general.size_label str              = 350M
+llama_model_loader: - kv   6:                            general.license str              = apache-2.0
+llama_model_loader: - kv   7:                   general.base_model.count u32              = 1
+llama_model_loader: - kv   8:                  general.base_model.0.name str              = Granite 4.0 H 350m
+llama_model_loader: - kv   9:          general.base_model.0.organization str              = Ibm Granite
+llama_model_loader: - kv  10:              general.base_model.0.repo_url str              = https://huggingface.co/ibm-granite/gr...
+llama_model_loader: - kv  11:                               general.tags arr[str,3]       = ["language", "unsloth", "granite-4.0"]
+llama_model_loader: - kv  12:                  granitehybrid.block_count u32              = 32
+llama_model_loader: - kv  13:               granitehybrid.context_length u32              = 1048576
+llama_model_loader: - kv  14:             granitehybrid.embedding_length u32              = 768
+llama_model_loader: - kv  15:          granitehybrid.feed_forward_length u32              = 2048
+llama_model_loader: - kv  16:         granitehybrid.attention.head_count u32              = 12
+llama_model_loader: - kv  17:      granitehybrid.attention.head_count_kv arr[i32,32]      = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, ...
+llama_model_loader: - kv  18:               granitehybrid.rope.freq_base f32              = 10000.000000
+llama_model_loader: - kv  19: granitehybrid.attention.layer_norm_rms_epsilon f32              = 0.000010
+llama_model_loader: - kv  20:                 granitehybrid.expert_count u32              = 0
+llama_model_loader: - kv  21:            granitehybrid.expert_used_count u32              = 0
+llama_model_loader: - kv  22:                   granitehybrid.vocab_size u32              = 100352
+llama_model_loader: - kv  23:         granitehybrid.rope.dimension_count u32              = 64
+llama_model_loader: - kv  24:              granitehybrid.attention.scale f32              = 0.015625
+llama_model_loader: - kv  25:              granitehybrid.embedding_scale f32              = 12.000000
+llama_model_loader: - kv  26:               granitehybrid.residual_scale f32              = 0.246000
+llama_model_loader: - kv  27:                  granitehybrid.logit_scale f32              = 3.000000
+llama_model_loader: - kv  28: granitehybrid.expert_shared_feed_forward_length u32              = 2048
+llama_model_loader: - kv  29:              granitehybrid.ssm.conv_kernel u32              = 4
+llama_model_loader: - kv  30:               granitehybrid.ssm.state_size u32              = 128
+llama_model_loader: - kv  31:              granitehybrid.ssm.group_count u32              = 1
+llama_model_loader: - kv  32:               granitehybrid.ssm.inner_size u32              = 1536
+llama_model_loader: - kv  33:           granitehybrid.ssm.time_step_rank u32              = 48
+llama_model_loader: - kv  34:       granitehybrid.rope.scaling.finetuned bool             = false
+llama_model_loader: - kv  35:                       tokenizer.ggml.model str              = gpt2
+llama_model_loader: - kv  36:                         tokenizer.ggml.pre str              = dbrx
+llama_model_loader: - kv  37:                      tokenizer.ggml.tokens arr[str,100352]  = ["!", "\"", "#", "$", "%", "&", "'", ...
+llama_model_loader: - kv  38:                  tokenizer.ggml.token_type arr[i32,100352]  = [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...
+llama_model_loader: - kv  39:                      tokenizer.ggml.merges arr[str,100000]  = ["Ġ Ġ", "ĠĠ ĠĠ", "i n", "Ġ t",...
+llama_model_loader: - kv  40:                tokenizer.ggml.bos_token_id u32              = 100257
+llama_model_loader: - kv  41:                tokenizer.ggml.eos_token_id u32              = 100257
+llama_model_loader: - kv  42:            tokenizer.ggml.unknown_token_id u32              = 100269
+llama_model_loader: - kv  43:            tokenizer.ggml.padding_token_id u32              = 100256
+llama_model_loader: - kv  44:               tokenizer.ggml.add_bos_token bool             = false
+llama_model_loader: - kv  45:                    tokenizer.chat_template str              = {%- set tools_system_message_prefix =...
+llama_model_loader: - kv  46:               general.quantization_version u32              = 2
+llama_model_loader: - kv  47:                          general.file_type u32              = 25
+llama_model_loader: - type  f32:  233 tensors
+llama_model_loader: - type q6_K:  113 tensors
+llama_model_loader: - type iq4_nl:   56 tensors
+print_info: file format = GGUF V3 (latest)
+print_info: file type   = IQ4_NL - 4.5 bpw
+print_info: file size   = 241.33 MiB (5.95 BPW) 
+load: printing all EOG tokens:
+load:   - 100257 ('<|end_of_text|>')
+load:   - 100261 ('<|fim_pad|>')
+load: special tokens cache size = 96
+load: token to piece cache size = 0.6152 MB
+print_info: arch             = granitehybrid
+print_info: vocab_only       = 0
+print_info: n_ctx_train      = 1048576
+print_info: n_embd           = 768
+print_info: n_embd_inp       = 768
+print_info: n_layer          = 32
+print_info: n_head           = 12
+print_info: n_head_kv        = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0]
+print_info: n_rot            = 64
+print_info: n_swa            = 0
+print_info: is_swa_any       = 0
+print_info: n_embd_head_k    = 64
+print_info: n_embd_head_v    = 64
+print_info: n_gqa            = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 3, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0]
+print_info: n_embd_k_gqa     = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 256, 0, 0, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 0, 0]
+print_info: n_embd_v_gqa     = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 256, 0, 0, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 0, 0]
+print_info: f_norm_eps       = 0.0e+00
+print_info: f_norm_rms_eps   = 1.0e-05
+print_info: f_clamp_kqv      = 0.0e+00
+print_info: f_max_alibi_bias = 0.0e+00
+print_info: f_logit_scale    = 3.0e+00
+print_info: f_attn_scale     = 1.6e-02
+print_info: n_ff             = 2048
+print_info: n_expert         = 0
+print_info: n_expert_used    = 0
+print_info: n_expert_groups  = 0
+print_info: n_group_used     = 0
+print_info: causal attn      = 1
+print_info: pooling type     = 0
+print_info: rope type        = 0
+print_info: rope scaling     = linear
+print_info: freq_base_train  = 10000.0
+print_info: freq_scale_train = 1
+print_info: n_ctx_orig_yarn  = 1048576
+print_info: rope_finetuned   = unknown
+print_info: ssm_d_conv       = 4
+print_info: ssm_d_inner      = 1536
+print_info: ssm_d_state      = 128
+print_info: ssm_dt_rank      = 48
+print_info: ssm_n_group      = 1
+print_info: ssm_dt_b_c_rms   = 0
+print_info: model type       = 350M
+print_info: model params     = 340.33 M
+print_info: general.name     = Granite 4.0 H 350m Unsloth
+print_info: f_embedding_scale = 12.000000
+print_info: f_residual_scale  = 0.246000
+print_info: f_attention_scale = 0.015625
+print_info: n_ff_shexp        = 2048
+print_info: vocab type       = BPE
+print_info: n_vocab          = 100352
+print_info: n_merges         = 100000
+print_info: BOS token        = 100257 '<|end_of_text|>'
+print_info: EOS token        = 100257 '<|end_of_text|>'
+print_info: EOT token        = 100257 '<|end_of_text|>'
+print_info: UNK token        = 100269 '<|unk|>'
+print_info: PAD token        = 100256 '<|pad|>'
+print_info: LF token         = 198 'Ċ'
+print_info: FIM PRE token    = 100258 '<|fim_prefix|>'
+print_info: FIM SUF token    = 100260 '<|fim_suffix|>'
+print_info: FIM MID token    = 100259 '<|fim_middle|>'
+print_info: FIM PAD token    = 100261 '<|fim_pad|>'
+print_info: EOG token        = 100257 '<|end_of_text|>'
+print_info: EOG token        = 100261 '<|fim_pad|>'
+print_info: max token length = 256
+load_tensors: loading model tensors, this can take a while... (mmap = true)
+load_tensors: offloading 20 repeating layers to GPU
+load_tensors: offloaded 20/33 layers to GPU
+load_tensors:   CPU_Mapped model buffer size =   128.60 MiB
+load_tensors:        CUDA0 model buffer size =    55.96 MiB
+load_tensors:        CUDA1 model buffer size =    56.79 MiB
+.............................................................................
+llama_context: constructing llama_context
+llama_context: n_seq_max     = 1
+llama_context: n_ctx         = 2048
+llama_context: n_ctx_seq     = 2048
+llama_context: n_batch       = 2048
+llama_context: n_ubatch      = 512
+llama_context: causal_attn   = 1
+llama_context: flash_attn    = auto
+llama_context: kv_unified    = false
+llama_context: freq_base     = 10000.0
+llama_context: freq_scale    = 1
+llama_context: n_ctx_seq (2048) < n_ctx_train (1048576) -- the full capacity of the model will not be utilized
+llama_context:        CPU  output buffer size =     0.38 MiB
+llama_kv_cache:        CPU KV buffer size =     2.00 MiB
+llama_kv_cache:      CUDA0 KV buffer size =     4.00 MiB
+llama_kv_cache:      CUDA1 KV buffer size =     2.00 MiB
+llama_kv_cache: size =    8.00 MiB (  2048 cells,   4 layers,  1/1 seqs), K (f16):    4.00 MiB, V (f16):    4.00 MiB
+llama_memory_recurrent:        CPU RS buffer size =     8.48 MiB
+llama_memory_recurrent:      CUDA0 RS buffer size =     6.16 MiB
+llama_memory_recurrent:      CUDA1 RS buffer size =     6.93 MiB
+llama_memory_recurrent: size =   21.57 MiB (     1 cells,  32 layers,  1 seqs), R (f32):    0.57 MiB, S (f32):   21.00 MiB
+llama_context: Flash Attention was auto, set to enabled
+llama_context:      CUDA0 compute buffer size =   266.04 MiB
+llama_context:      CUDA1 compute buffer size =    22.39 MiB
+llama_context:  CUDA_Host compute buffer size =    18.34 MiB
+llama_context: graph nodes  = 1815
+llama_context: graph splits = 182 (with bs=512), 41 (with bs=1)
+common_init_from_params: added <|end_of_text|> logit bias = -inf
+common_init_from_params: added <|fim_pad|> logit bias = -inf
+common_init_from_params: setting dry_penalty_last_n to ctx_size = 2048
+common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable)
+
+system_info: n_threads = 16 (n_threads_batch = 16) / 32 | CUDA : ARCHS = 860 | USE_GRAPHS = 1 | PEER_MAX_BATCH_SIZE = 128 | CPU : SSE3 = 1 | SSSE3 = 1 | AVX = 1 | AVX2 = 1 | F16C = 1 | FMA = 1 | BMI2 = 1 | AVX512 = 1 | AVX512_VBMI = 1 | AVX512_VNNI = 1 | AVX512_BF16 = 1 | LLAMAFILE = 1 | OPENMP = 1 | REPACK = 1 | 
+perplexity: tokenizing the input ..
+perplexity: tokenization took 41.832 ms
+perplexity: calculating perplexity over 14 chunks, n_ctx=2048, batch_size=2048, n_seq=1
+perplexity: 0.51 seconds per pass - ETA 0.12 minutes
+[1]20.4763,[2]24.3817,[3]24.8760,[4]22.7249,[5]22.8850,[6]20.3788,[7]19.9490,[8]19.7860,[9]20.4590,[10]20.3654,[11]20.2543,[12]20.4593,[13]20.5731,[14]20.5806,
+Final estimate: PPL = 20.5806 +/- 0.54355
+
+llama_perf_context_print:        load time =     203.50 ms
+llama_perf_context_print: prompt eval time =    4906.32 ms / 28672 tokens (    0.17 ms per token,  5843.89 tokens per second)
+llama_perf_context_print:        eval time =       0.00 ms /     1 runs   (    0.00 ms per token,      inf tokens per second)
+llama_perf_context_print:       total time =    5170.88 ms / 28673 tokens
+llama_perf_context_print:    graphs reused =          0
+llama_memory_breakdown_print: | memory breakdown [MiB] | total    free    self   model   context   compute    unaccounted |
+llama_memory_breakdown_print: |   - CUDA0 (RTX 3090)   | 24115 = 20541 + ( 333 =    55 +      10 +     267) +        3240 |
+llama_memory_breakdown_print: |   - CUDA1 (RTX 3090)   | 24124 = 23425 + (  88 =    56 +       8 +      22) +         610 |
+llama_memory_breakdown_print: |   - Host               |                   157 =   128 +      10 +      18                |

Benchmarks/DataCollection/granite-4.0-h-350m-unsloth-iq4_nl-attn_kv_Q6_K-attn_output_Q6_K-attn_q_Q6_K-embeddings_Q6_K-ffn_down_Q6_K-ffn_up_gate_Q6_K/perplexity_math.log

@@ -0,0 +1,189 @@
+ggml_cuda_init: GGML_CUDA_FORCE_MMQ:    no
+ggml_cuda_init: GGML_CUDA_FORCE_CUBLAS: no
+ggml_cuda_init: found 2 CUDA devices:
+  Device 0: NVIDIA GeForce RTX 3090, compute capability 8.6, VMM: yes
+  Device 1: NVIDIA GeForce RTX 3090, compute capability 8.6, VMM: yes
+build: 7040 (92bb442ad) with cc (Ubuntu 13.3.0-6ubuntu2~24.04) 13.3.0 for x86_64-linux-gnu
+llama_model_load_from_file_impl: using device CUDA0 (NVIDIA GeForce RTX 3090) (0000:01:00.0) - 20939 MiB free
+llama_model_load_from_file_impl: using device CUDA1 (NVIDIA GeForce RTX 3090) (0000:03:00.0) - 23581 MiB free
+llama_model_loader: loaded meta data with 48 key-value pairs and 402 tensors from /mnt/world8/AI/ToBench/granite-4.0-h-350m-unsloth/Magic_Quant/GGUF/dc_round0_granite-4.0-h-350m-unsloth-iq4_nl-attn_kv_Q6_K-attn_output_Q6_K-attn_q_Q6_K-embeddings_Q6_K-ffn_down_Q6_K-ffn_up_gate_Q6_K.gguf (version GGUF V3 (latest))
+llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output.
+llama_model_loader: - kv   0:                       general.architecture str              = granitehybrid
+llama_model_loader: - kv   1:                               general.type str              = model
+llama_model_loader: - kv   2:                               general.name str              = Granite 4.0 H 350m Unsloth
+llama_model_loader: - kv   3:                           general.finetune str              = unsloth
+llama_model_loader: - kv   4:                           general.basename str              = granite-4.0-h
+llama_model_loader: - kv   5:                         general.size_label str              = 350M
+llama_model_loader: - kv   6:                            general.license str              = apache-2.0
+llama_model_loader: - kv   7:                   general.base_model.count u32              = 1
+llama_model_loader: - kv   8:                  general.base_model.0.name str              = Granite 4.0 H 350m
+llama_model_loader: - kv   9:          general.base_model.0.organization str              = Ibm Granite
+llama_model_loader: - kv  10:              general.base_model.0.repo_url str              = https://huggingface.co/ibm-granite/gr...
+llama_model_loader: - kv  11:                               general.tags arr[str,3]       = ["language", "unsloth", "granite-4.0"]
+llama_model_loader: - kv  12:                  granitehybrid.block_count u32              = 32
+llama_model_loader: - kv  13:               granitehybrid.context_length u32              = 1048576
+llama_model_loader: - kv  14:             granitehybrid.embedding_length u32              = 768
+llama_model_loader: - kv  15:          granitehybrid.feed_forward_length u32              = 2048
+llama_model_loader: - kv  16:         granitehybrid.attention.head_count u32              = 12
+llama_model_loader: - kv  17:      granitehybrid.attention.head_count_kv arr[i32,32]      = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, ...
+llama_model_loader: - kv  18:               granitehybrid.rope.freq_base f32              = 10000.000000
+llama_model_loader: - kv  19: granitehybrid.attention.layer_norm_rms_epsilon f32              = 0.000010
+llama_model_loader: - kv  20:                 granitehybrid.expert_count u32              = 0
+llama_model_loader: - kv  21:            granitehybrid.expert_used_count u32              = 0
+llama_model_loader: - kv  22:                   granitehybrid.vocab_size u32              = 100352
+llama_model_loader: - kv  23:         granitehybrid.rope.dimension_count u32              = 64
+llama_model_loader: - kv  24:              granitehybrid.attention.scale f32              = 0.015625
+llama_model_loader: - kv  25:              granitehybrid.embedding_scale f32              = 12.000000
+llama_model_loader: - kv  26:               granitehybrid.residual_scale f32              = 0.246000
+llama_model_loader: - kv  27:                  granitehybrid.logit_scale f32              = 3.000000
+llama_model_loader: - kv  28: granitehybrid.expert_shared_feed_forward_length u32              = 2048
+llama_model_loader: - kv  29:              granitehybrid.ssm.conv_kernel u32              = 4
+llama_model_loader: - kv  30:               granitehybrid.ssm.state_size u32              = 128
+llama_model_loader: - kv  31:              granitehybrid.ssm.group_count u32              = 1
+llama_model_loader: - kv  32:               granitehybrid.ssm.inner_size u32              = 1536
+llama_model_loader: - kv  33:           granitehybrid.ssm.time_step_rank u32              = 48
+llama_model_loader: - kv  34:       granitehybrid.rope.scaling.finetuned bool             = false
+llama_model_loader: - kv  35:                       tokenizer.ggml.model str              = gpt2
+llama_model_loader: - kv  36:                         tokenizer.ggml.pre str              = dbrx
+llama_model_loader: - kv  37:                      tokenizer.ggml.tokens arr[str,100352]  = ["!", "\"", "#", "$", "%", "&", "'", ...
+llama_model_loader: - kv  38:                  tokenizer.ggml.token_type arr[i32,100352]  = [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...
+llama_model_loader: - kv  39:                      tokenizer.ggml.merges arr[str,100000]  = ["Ġ Ġ", "ĠĠ ĠĠ", "i n", "Ġ t",...
+llama_model_loader: - kv  40:                tokenizer.ggml.bos_token_id u32              = 100257
+llama_model_loader: - kv  41:                tokenizer.ggml.eos_token_id u32              = 100257
+llama_model_loader: - kv  42:            tokenizer.ggml.unknown_token_id u32              = 100269
+llama_model_loader: - kv  43:            tokenizer.ggml.padding_token_id u32              = 100256
+llama_model_loader: - kv  44:               tokenizer.ggml.add_bos_token bool             = false
+llama_model_loader: - kv  45:                    tokenizer.chat_template str              = {%- set tools_system_message_prefix =...
+llama_model_loader: - kv  46:               general.quantization_version u32              = 2
+llama_model_loader: - kv  47:                          general.file_type u32              = 25
+llama_model_loader: - type  f32:  233 tensors
+llama_model_loader: - type q6_K:  113 tensors
+llama_model_loader: - type iq4_nl:   56 tensors
+print_info: file format = GGUF V3 (latest)
+print_info: file type   = IQ4_NL - 4.5 bpw
+print_info: file size   = 241.33 MiB (5.95 BPW) 
+load: printing all EOG tokens:
+load:   - 100257 ('<|end_of_text|>')
+load:   - 100261 ('<|fim_pad|>')
+load: special tokens cache size = 96
+load: token to piece cache size = 0.6152 MB
+print_info: arch             = granitehybrid
+print_info: vocab_only       = 0
+print_info: n_ctx_train      = 1048576
+print_info: n_embd           = 768
+print_info: n_embd_inp       = 768
+print_info: n_layer          = 32
+print_info: n_head           = 12
+print_info: n_head_kv        = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0]
+print_info: n_rot            = 64
+print_info: n_swa            = 0
+print_info: is_swa_any       = 0
+print_info: n_embd_head_k    = 64
+print_info: n_embd_head_v    = 64
+print_info: n_gqa            = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 3, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0]
+print_info: n_embd_k_gqa     = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 256, 0, 0, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 0, 0]
+print_info: n_embd_v_gqa     = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 256, 0, 0, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 0, 0]
+print_info: f_norm_eps       = 0.0e+00
+print_info: f_norm_rms_eps   = 1.0e-05
+print_info: f_clamp_kqv      = 0.0e+00
+print_info: f_max_alibi_bias = 0.0e+00
+print_info: f_logit_scale    = 3.0e+00
+print_info: f_attn_scale     = 1.6e-02
+print_info: n_ff             = 2048
+print_info: n_expert         = 0
+print_info: n_expert_used    = 0
+print_info: n_expert_groups  = 0
+print_info: n_group_used     = 0
+print_info: causal attn      = 1
+print_info: pooling type     = 0
+print_info: rope type        = 0
+print_info: rope scaling     = linear
+print_info: freq_base_train  = 10000.0
+print_info: freq_scale_train = 1
+print_info: n_ctx_orig_yarn  = 1048576
+print_info: rope_finetuned   = unknown
+print_info: ssm_d_conv       = 4
+print_info: ssm_d_inner      = 1536
+print_info: ssm_d_state      = 128
+print_info: ssm_dt_rank      = 48
+print_info: ssm_n_group      = 1
+print_info: ssm_dt_b_c_rms   = 0
+print_info: model type       = 350M
+print_info: model params     = 340.33 M
+print_info: general.name     = Granite 4.0 H 350m Unsloth
+print_info: f_embedding_scale = 12.000000
+print_info: f_residual_scale  = 0.246000
+print_info: f_attention_scale = 0.015625
+print_info: n_ff_shexp        = 2048
+print_info: vocab type       = BPE
+print_info: n_vocab          = 100352
+print_info: n_merges         = 100000
+print_info: BOS token        = 100257 '<|end_of_text|>'
+print_info: EOS token        = 100257 '<|end_of_text|>'
+print_info: EOT token        = 100257 '<|end_of_text|>'
+print_info: UNK token        = 100269 '<|unk|>'
+print_info: PAD token        = 100256 '<|pad|>'
+print_info: LF token         = 198 'Ċ'
+print_info: FIM PRE token    = 100258 '<|fim_prefix|>'
+print_info: FIM SUF token    = 100260 '<|fim_suffix|>'
+print_info: FIM MID token    = 100259 '<|fim_middle|>'
+print_info: FIM PAD token    = 100261 '<|fim_pad|>'
+print_info: EOG token        = 100257 '<|end_of_text|>'
+print_info: EOG token        = 100261 '<|fim_pad|>'
+print_info: max token length = 256
+load_tensors: loading model tensors, this can take a while... (mmap = true)
+load_tensors: offloading 20 repeating layers to GPU
+load_tensors: offloaded 20/33 layers to GPU
+load_tensors:   CPU_Mapped model buffer size =   128.60 MiB
+load_tensors:        CUDA0 model buffer size =    55.96 MiB
+load_tensors:        CUDA1 model buffer size =    56.79 MiB
+.............................................................................
+llama_context: constructing llama_context
+llama_context: n_seq_max     = 1
+llama_context: n_ctx         = 2048
+llama_context: n_ctx_seq     = 2048
+llama_context: n_batch       = 2048
+llama_context: n_ubatch      = 512
+llama_context: causal_attn   = 1
+llama_context: flash_attn    = auto
+llama_context: kv_unified    = false
+llama_context: freq_base     = 10000.0
+llama_context: freq_scale    = 1
+llama_context: n_ctx_seq (2048) < n_ctx_train (1048576) -- the full capacity of the model will not be utilized
+llama_context:        CPU  output buffer size =     0.38 MiB
+llama_kv_cache:        CPU KV buffer size =     2.00 MiB
+llama_kv_cache:      CUDA0 KV buffer size =     4.00 MiB
+llama_kv_cache:      CUDA1 KV buffer size =     2.00 MiB
+llama_kv_cache: size =    8.00 MiB (  2048 cells,   4 layers,  1/1 seqs), K (f16):    4.00 MiB, V (f16):    4.00 MiB
+llama_memory_recurrent:        CPU RS buffer size =     8.48 MiB
+llama_memory_recurrent:      CUDA0 RS buffer size =     6.16 MiB
+llama_memory_recurrent:      CUDA1 RS buffer size =     6.93 MiB
+llama_memory_recurrent: size =   21.57 MiB (     1 cells,  32 layers,  1 seqs), R (f32):    0.57 MiB, S (f32):   21.00 MiB
+llama_context: Flash Attention was auto, set to enabled
+llama_context:      CUDA0 compute buffer size =   266.04 MiB
+llama_context:      CUDA1 compute buffer size =    22.39 MiB
+llama_context:  CUDA_Host compute buffer size =    18.34 MiB
+llama_context: graph nodes  = 1815
+llama_context: graph splits = 182 (with bs=512), 41 (with bs=1)
+common_init_from_params: added <|end_of_text|> logit bias = -inf
+common_init_from_params: added <|fim_pad|> logit bias = -inf
+common_init_from_params: setting dry_penalty_last_n to ctx_size = 2048
+common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable)
+
+system_info: n_threads = 16 (n_threads_batch = 16) / 32 | CUDA : ARCHS = 860 | USE_GRAPHS = 1 | PEER_MAX_BATCH_SIZE = 128 | CPU : SSE3 = 1 | SSSE3 = 1 | AVX = 1 | AVX2 = 1 | F16C = 1 | FMA = 1 | BMI2 = 1 | AVX512 = 1 | AVX512_VBMI = 1 | AVX512_VNNI = 1 | AVX512_BF16 = 1 | LLAMAFILE = 1 | OPENMP = 1 | REPACK = 1 | 
+perplexity: tokenizing the input ..
+perplexity: tokenization took 35.059 ms
+perplexity: calculating perplexity over 15 chunks, n_ctx=2048, batch_size=2048, n_seq=1
+perplexity: 0.50 seconds per pass - ETA 0.12 minutes
+[1]8.9715,[2]10.1342,[3]9.6674,[4]10.0008,[5]10.2128,[6]10.2812,[7]10.4484,[8]10.1583,[9]10.1825,[10]10.2151,[11]10.4616,[12]10.5437,[13]10.6726,[14]10.6452,[15]10.5642,
+Final estimate: PPL = 10.5642 +/- 0.23980
+
+llama_perf_context_print:        load time =     209.47 ms
+llama_perf_context_print: prompt eval time =    5225.84 ms / 30720 tokens (    0.17 ms per token,  5878.48 tokens per second)
+llama_perf_context_print:        eval time =       0.00 ms /     1 runs   (    0.00 ms per token,      inf tokens per second)
+llama_perf_context_print:       total time =    5498.38 ms / 30721 tokens
+llama_perf_context_print:    graphs reused =          0
+llama_memory_breakdown_print: | memory breakdown [MiB] | total    free    self   model   context   compute    unaccounted |
+llama_memory_breakdown_print: |   - CUDA0 (RTX 3090)   | 24115 = 20539 + ( 333 =    55 +      10 +     267) +        3242 |
+llama_memory_breakdown_print: |   - CUDA1 (RTX 3090)   | 24124 = 23425 + (  88 =    56 +       8 +      22) +         610 |
+llama_memory_breakdown_print: |   - Host               |                   157 =   128 +      10 +      18                |

Benchmarks/DataCollection/granite-4.0-h-350m-unsloth-mxfp4_moe-attn_kv_BF16-attn_output_BF16-attn_q_BF16-embeddings_BF16-ffn_down_BF16-ffn_up_gate_BF16/bench_metrics.json

@@ -0,0 +1,44 @@
+{
+  "raw_metrics": {
+    "llamabench": {
+      "backend": "CUDA",
+      "log_path": "granite-4.0-h-350m-unsloth-mxfp4_moe-attn_kv_BF16-attn_output_BF16-attn_q_BF16-embeddings_BF16-ffn_down_BF16-ffn_up_gate_BF16/llamabench.md",
+      "ngl": "35",
+      "raw_row": {
+        "backend": "CUDA",
+        "model": "granitehybrid 350M MXFP4 MoE",
+        "ngl": "35",
+        "params": "340.33 M",
+        "size": "555.36 MiB",
+        "t/s": "1695.58 \u00b1 39.40",
+        "test": "pp8",
+        "tps_value": 1695.58
+      },
+      "test": "pp8",
+      "tps": 1695.58
+    },
+    "perplexity": {
+      "code": {
+        "log_path": "granite-4.0-h-350m-unsloth-mxfp4_moe-attn_kv_BF16-attn_output_BF16-attn_q_BF16-embeddings_BF16-ffn_down_BF16-ffn_up_gate_BF16/perplexity_code.log",
+        "ppl": 1.9549,
+        "ppl_error": 0.01753
+      },
+      "general": {
+        "log_path": "granite-4.0-h-350m-unsloth-mxfp4_moe-attn_kv_BF16-attn_output_BF16-attn_q_BF16-embeddings_BF16-ffn_down_BF16-ffn_up_gate_BF16/perplexity_general.log",
+        "ppl": 18.1504,
+        "ppl_error": 0.46645
+      },
+      "math": {
+        "log_path": "granite-4.0-h-350m-unsloth-mxfp4_moe-attn_kv_BF16-attn_output_BF16-attn_q_BF16-embeddings_BF16-ffn_down_BF16-ffn_up_gate_BF16/perplexity_math.log",
+        "ppl": 10.297,
+        "ppl_error": 0.23199
+      }
+    }
+  },
+  "summary": {
+    "avg_prec_loss_pct": 0.0645,
+    "bench_tps": 1695.58,
+    "file_size_bytes": 585905088,
+    "file_size_gb": 0.55
+  }
+}

Benchmarks/DataCollection/granite-4.0-h-350m-unsloth-mxfp4_moe-attn_kv_BF16-attn_output_BF16-attn_q_BF16-embeddings_BF16-ffn_down_BF16-ffn_up_gate_BF16/llamabench.md

@@ -0,0 +1,11 @@
+ggml_cuda_init: GGML_CUDA_FORCE_MMQ:    no
+ggml_cuda_init: GGML_CUDA_FORCE_CUBLAS: no
+ggml_cuda_init: found 2 CUDA devices:
+  Device 0: NVIDIA GeForce RTX 3090, compute capability 8.6, VMM: yes
+  Device 1: NVIDIA GeForce RTX 3090, compute capability 8.6, VMM: yes
+| model                          |       size |     params | backend    | ngl |            test |                  t/s |
+| ------------------------------ | ---------: | ---------: | ---------- | --: | --------------: | -------------------: |
+| granitehybrid 350M MXFP4 MoE   | 555.36 MiB |   340.33 M | CUDA       |  35 |             pp8 |      1695.58 ± 39.40 |
+| granitehybrid 350M MXFP4 MoE   | 555.36 MiB |   340.33 M | CUDA       |  35 |           tg128 |        309.34 ± 9.13 |
+
+build: 92bb442ad (7040)

Benchmarks/DataCollection/granite-4.0-h-350m-unsloth-mxfp4_moe-attn_kv_BF16-attn_output_BF16-attn_q_BF16-embeddings_BF16-ffn_down_BF16-ffn_up_gate_BF16/perplexity_code.log

@@ -0,0 +1,189 @@
+ggml_cuda_init: GGML_CUDA_FORCE_MMQ:    no
+ggml_cuda_init: GGML_CUDA_FORCE_CUBLAS: no
+ggml_cuda_init: found 2 CUDA devices:
+  Device 0: NVIDIA GeForce RTX 3090, compute capability 8.6, VMM: yes
+  Device 1: NVIDIA GeForce RTX 3090, compute capability 8.6, VMM: yes
+build: 7040 (92bb442ad) with cc (Ubuntu 13.3.0-6ubuntu2~24.04) 13.3.0 for x86_64-linux-gnu
+llama_model_load_from_file_impl: using device CUDA0 (NVIDIA GeForce RTX 3090) (0000:01:00.0) - 20828 MiB free
+llama_model_load_from_file_impl: using device CUDA1 (NVIDIA GeForce RTX 3090) (0000:03:00.0) - 23581 MiB free
+llama_model_loader: loaded meta data with 48 key-value pairs and 402 tensors from /mnt/world8/AI/ToBench/granite-4.0-h-350m-unsloth/Magic_Quant/GGUF/dc_round-1_granite-4.0-h-350m-unsloth-mxfp4_moe-attn_kv_BF16-attn_output_BF16-attn_q_BF16-embeddings_BF16-ffn_down_BF16-ffn_up_gate_BF16.gguf (version GGUF V3 (latest))
+llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output.
+llama_model_loader: - kv   0:                       general.architecture str              = granitehybrid
+llama_model_loader: - kv   1:                               general.type str              = model
+llama_model_loader: - kv   2:                               general.name str              = Granite 4.0 H 350m Unsloth
+llama_model_loader: - kv   3:                           general.finetune str              = unsloth
+llama_model_loader: - kv   4:                           general.basename str              = granite-4.0-h
+llama_model_loader: - kv   5:                         general.size_label str              = 350M
+llama_model_loader: - kv   6:                            general.license str              = apache-2.0
+llama_model_loader: - kv   7:                   general.base_model.count u32              = 1
+llama_model_loader: - kv   8:                  general.base_model.0.name str              = Granite 4.0 H 350m
+llama_model_loader: - kv   9:          general.base_model.0.organization str              = Ibm Granite
+llama_model_loader: - kv  10:              general.base_model.0.repo_url str              = https://huggingface.co/ibm-granite/gr...
+llama_model_loader: - kv  11:                               general.tags arr[str,3]       = ["language", "unsloth", "granite-4.0"]
+llama_model_loader: - kv  12:                  granitehybrid.block_count u32              = 32
+llama_model_loader: - kv  13:               granitehybrid.context_length u32              = 1048576
+llama_model_loader: - kv  14:             granitehybrid.embedding_length u32              = 768
+llama_model_loader: - kv  15:          granitehybrid.feed_forward_length u32              = 2048
+llama_model_loader: - kv  16:         granitehybrid.attention.head_count u32              = 12
+llama_model_loader: - kv  17:      granitehybrid.attention.head_count_kv arr[i32,32]      = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, ...
+llama_model_loader: - kv  18:               granitehybrid.rope.freq_base f32              = 10000.000000
+llama_model_loader: - kv  19: granitehybrid.attention.layer_norm_rms_epsilon f32              = 0.000010
+llama_model_loader: - kv  20:                 granitehybrid.expert_count u32              = 0
+llama_model_loader: - kv  21:            granitehybrid.expert_used_count u32              = 0
+llama_model_loader: - kv  22:                   granitehybrid.vocab_size u32              = 100352
+llama_model_loader: - kv  23:         granitehybrid.rope.dimension_count u32              = 64
+llama_model_loader: - kv  24:              granitehybrid.attention.scale f32              = 0.015625
+llama_model_loader: - kv  25:              granitehybrid.embedding_scale f32              = 12.000000
+llama_model_loader: - kv  26:               granitehybrid.residual_scale f32              = 0.246000
+llama_model_loader: - kv  27:                  granitehybrid.logit_scale f32              = 3.000000
+llama_model_loader: - kv  28: granitehybrid.expert_shared_feed_forward_length u32              = 2048
+llama_model_loader: - kv  29:              granitehybrid.ssm.conv_kernel u32              = 4
+llama_model_loader: - kv  30:               granitehybrid.ssm.state_size u32              = 128
+llama_model_loader: - kv  31:              granitehybrid.ssm.group_count u32              = 1
+llama_model_loader: - kv  32:               granitehybrid.ssm.inner_size u32              = 1536
+llama_model_loader: - kv  33:           granitehybrid.ssm.time_step_rank u32              = 48
+llama_model_loader: - kv  34:       granitehybrid.rope.scaling.finetuned bool             = false
+llama_model_loader: - kv  35:                       tokenizer.ggml.model str              = gpt2
+llama_model_loader: - kv  36:                         tokenizer.ggml.pre str              = dbrx
+llama_model_loader: - kv  37:                      tokenizer.ggml.tokens arr[str,100352]  = ["!", "\"", "#", "$", "%", "&", "'", ...
+llama_model_loader: - kv  38:                  tokenizer.ggml.token_type arr[i32,100352]  = [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...
+llama_model_loader: - kv  39:                      tokenizer.ggml.merges arr[str,100000]  = ["Ġ Ġ", "ĠĠ ĠĠ", "i n", "Ġ t",...
+llama_model_loader: - kv  40:                tokenizer.ggml.bos_token_id u32              = 100257
+llama_model_loader: - kv  41:                tokenizer.ggml.eos_token_id u32              = 100257
+llama_model_loader: - kv  42:            tokenizer.ggml.unknown_token_id u32              = 100269
+llama_model_loader: - kv  43:            tokenizer.ggml.padding_token_id u32              = 100256
+llama_model_loader: - kv  44:               tokenizer.ggml.add_bos_token bool             = false
+llama_model_loader: - kv  45:                    tokenizer.chat_template str              = {%- set tools_system_message_prefix =...
+llama_model_loader: - kv  46:               general.quantization_version u32              = 2
+llama_model_loader: - kv  47:                          general.file_type u32              = 38
+llama_model_loader: - type  f32:  233 tensors
+llama_model_loader: - type q8_0:   56 tensors
+llama_model_loader: - type bf16:  113 tensors
+print_info: file format = GGUF V3 (latest)
+print_info: file type   = MXFP4 MoE
+print_info: file size   = 555.36 MiB (13.69 BPW) 
+load: printing all EOG tokens:
+load:   - 100257 ('<|end_of_text|>')
+load:   - 100261 ('<|fim_pad|>')
+load: special tokens cache size = 96
+load: token to piece cache size = 0.6152 MB
+print_info: arch             = granitehybrid
+print_info: vocab_only       = 0
+print_info: n_ctx_train      = 1048576
+print_info: n_embd           = 768
+print_info: n_embd_inp       = 768
+print_info: n_layer          = 32
+print_info: n_head           = 12
+print_info: n_head_kv        = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0]
+print_info: n_rot            = 64
+print_info: n_swa            = 0
+print_info: is_swa_any       = 0
+print_info: n_embd_head_k    = 64
+print_info: n_embd_head_v    = 64
+print_info: n_gqa            = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 3, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0]
+print_info: n_embd_k_gqa     = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 256, 0, 0, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 0, 0]
+print_info: n_embd_v_gqa     = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 256, 0, 0, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 0, 0]
+print_info: f_norm_eps       = 0.0e+00
+print_info: f_norm_rms_eps   = 1.0e-05
+print_info: f_clamp_kqv      = 0.0e+00
+print_info: f_max_alibi_bias = 0.0e+00
+print_info: f_logit_scale    = 3.0e+00
+print_info: f_attn_scale     = 1.6e-02
+print_info: n_ff             = 2048
+print_info: n_expert         = 0
+print_info: n_expert_used    = 0
+print_info: n_expert_groups  = 0
+print_info: n_group_used     = 0
+print_info: causal attn      = 1
+print_info: pooling type     = 0
+print_info: rope type        = 0
+print_info: rope scaling     = linear
+print_info: freq_base_train  = 10000.0
+print_info: freq_scale_train = 1
+print_info: n_ctx_orig_yarn  = 1048576
+print_info: rope_finetuned   = unknown
+print_info: ssm_d_conv       = 4
+print_info: ssm_d_inner      = 1536
+print_info: ssm_d_state      = 128
+print_info: ssm_dt_rank      = 48
+print_info: ssm_n_group      = 1
+print_info: ssm_dt_b_c_rms   = 0
+print_info: model type       = 350M
+print_info: model params     = 340.33 M
+print_info: general.name     = Granite 4.0 H 350m Unsloth
+print_info: f_embedding_scale = 12.000000
+print_info: f_residual_scale  = 0.246000
+print_info: f_attention_scale = 0.015625
+print_info: n_ff_shexp        = 2048
+print_info: vocab type       = BPE
+print_info: n_vocab          = 100352
+print_info: n_merges         = 100000
+print_info: BOS token        = 100257 '<|end_of_text|>'
+print_info: EOS token        = 100257 '<|end_of_text|>'
+print_info: EOT token        = 100257 '<|end_of_text|>'
+print_info: UNK token        = 100269 '<|unk|>'
+print_info: PAD token        = 100256 '<|pad|>'
+print_info: LF token         = 198 'Ċ'
+print_info: FIM PRE token    = 100258 '<|fim_prefix|>'
+print_info: FIM SUF token    = 100260 '<|fim_suffix|>'
+print_info: FIM MID token    = 100259 '<|fim_middle|>'
+print_info: FIM PAD token    = 100261 '<|fim_pad|>'
+print_info: EOG token        = 100257 '<|end_of_text|>'
+print_info: EOG token        = 100261 '<|fim_pad|>'
+print_info: max token length = 256
+load_tensors: loading model tensors, this can take a while... (mmap = true)
+load_tensors: offloading 20 repeating layers to GPU
+load_tensors: offloaded 20/33 layers to GPU
+load_tensors:   CPU_Mapped model buffer size =   300.57 MiB
+load_tensors:        CUDA0 model buffer size =   126.97 MiB
+load_tensors:        CUDA1 model buffer size =   127.83 MiB
+...........................................................................
+llama_context: constructing llama_context
+llama_context: n_seq_max     = 1
+llama_context: n_ctx         = 2048
+llama_context: n_ctx_seq     = 2048
+llama_context: n_batch       = 2048
+llama_context: n_ubatch      = 512
+llama_context: causal_attn   = 1
+llama_context: flash_attn    = auto
+llama_context: kv_unified    = false
+llama_context: freq_base     = 10000.0
+llama_context: freq_scale    = 1
+llama_context: n_ctx_seq (2048) < n_ctx_train (1048576) -- the full capacity of the model will not be utilized
+llama_context:        CPU  output buffer size =     0.38 MiB
+llama_kv_cache:        CPU KV buffer size =     2.00 MiB
+llama_kv_cache:      CUDA0 KV buffer size =     4.00 MiB
+llama_kv_cache:      CUDA1 KV buffer size =     2.00 MiB
+llama_kv_cache: size =    8.00 MiB (  2048 cells,   4 layers,  1/1 seqs), K (f16):    4.00 MiB, V (f16):    4.00 MiB
+llama_memory_recurrent:        CPU RS buffer size =     8.48 MiB
+llama_memory_recurrent:      CUDA0 RS buffer size =     6.16 MiB
+llama_memory_recurrent:      CUDA1 RS buffer size =     6.93 MiB
+llama_memory_recurrent: size =   21.57 MiB (     1 cells,  32 layers,  1 seqs), R (f32):    0.57 MiB, S (f32):   21.00 MiB
+llama_context: Flash Attention was auto, set to enabled
+llama_context:      CUDA0 compute buffer size =   352.59 MiB
+llama_context:      CUDA1 compute buffer size =    22.39 MiB
+llama_context:  CUDA_Host compute buffer size =    18.34 MiB
+llama_context: graph nodes  = 1815
+llama_context: graph splits = 182 (with bs=512), 41 (with bs=1)
+common_init_from_params: added <|end_of_text|> logit bias = -inf
+common_init_from_params: added <|fim_pad|> logit bias = -inf
+common_init_from_params: setting dry_penalty_last_n to ctx_size = 2048
+common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable)
+
+system_info: n_threads = 16 (n_threads_batch = 16) / 32 | CUDA : ARCHS = 860 | USE_GRAPHS = 1 | PEER_MAX_BATCH_SIZE = 128 | CPU : SSE3 = 1 | SSSE3 = 1 | AVX = 1 | AVX2 = 1 | F16C = 1 | FMA = 1 | BMI2 = 1 | AVX512 = 1 | AVX512_VBMI = 1 | AVX512_VNNI = 1 | AVX512_BF16 = 1 | LLAMAFILE = 1 | OPENMP = 1 | REPACK = 1 | 
+perplexity: tokenizing the input ..
+perplexity: tokenization took 95.441 ms
+perplexity: calculating perplexity over 44 chunks, n_ctx=2048, batch_size=2048, n_seq=1
+perplexity: 0.62 seconds per pass - ETA 0.45 minutes
+[1]4.3648,[2]3.9780,[3]2.5671,[4]2.3654,[5]2.6041,[6]2.8480,[7]2.6998,[8]2.5084,[9]2.3056,[10]2.1370,[11]2.1196,[12]2.1459,[13]2.0574,[14]2.0374,[15]2.0779,[16]2.0122,[17]1.9867,[18]2.0052,[19]1.9658,[20]1.9304,[21]1.8975,[22]1.8828,[23]1.9118,[24]1.8853,[25]1.9043,[26]1.8724,[27]1.8594,[28]1.8512,[29]1.8967,[30]1.9132,[31]1.9123,[32]1.8880,[33]1.9116,[34]1.9038,[35]1.8852,[36]1.9165,[37]1.9230,[38]1.9209,[39]1.9426,[40]1.9401,[41]1.9330,[42]1.9569,[43]1.9656,[44]1.9549,
+Final estimate: PPL = 1.9549 +/- 0.01753
+
+llama_perf_context_print:        load time =     253.41 ms
+llama_perf_context_print: prompt eval time =   16214.33 ms / 90112 tokens (    0.18 ms per token,  5557.55 tokens per second)
+llama_perf_context_print:        eval time =       0.00 ms /     1 runs   (    0.00 ms per token,      inf tokens per second)
+llama_perf_context_print:       total time =   17011.52 ms / 90113 tokens
+llama_perf_context_print:    graphs reused =          0
+llama_memory_breakdown_print: | memory breakdown [MiB] | total    free    self   model   context   compute    unaccounted |
+llama_memory_breakdown_print: |   - CUDA0 (RTX 3090)   | 24115 = 20191 + ( 490 =   126 +      10 +     353) +        3433 |
+llama_memory_breakdown_print: |   - CUDA1 (RTX 3090)   | 24124 = 23339 + ( 159 =   127 +       8 +      22) +         625 |
+llama_memory_breakdown_print: |   - Host               |                   329 =   300 +      10 +      18                |

Benchmarks/DataCollection/granite-4.0-h-350m-unsloth-mxfp4_moe-attn_kv_BF16-attn_output_BF16-attn_q_BF16-embeddings_BF16-ffn_down_BF16-ffn_up_gate_BF16/perplexity_general.log

@@ -0,0 +1,189 @@
+ggml_cuda_init: GGML_CUDA_FORCE_MMQ:    no
+ggml_cuda_init: GGML_CUDA_FORCE_CUBLAS: no
+ggml_cuda_init: found 2 CUDA devices:
+  Device 0: NVIDIA GeForce RTX 3090, compute capability 8.6, VMM: yes
+  Device 1: NVIDIA GeForce RTX 3090, compute capability 8.6, VMM: yes
+build: 7040 (92bb442ad) with cc (Ubuntu 13.3.0-6ubuntu2~24.04) 13.3.0 for x86_64-linux-gnu
+llama_model_load_from_file_impl: using device CUDA0 (NVIDIA GeForce RTX 3090) (0000:01:00.0) - 20824 MiB free
+llama_model_load_from_file_impl: using device CUDA1 (NVIDIA GeForce RTX 3090) (0000:03:00.0) - 23581 MiB free
+llama_model_loader: loaded meta data with 48 key-value pairs and 402 tensors from /mnt/world8/AI/ToBench/granite-4.0-h-350m-unsloth/Magic_Quant/GGUF/dc_round-1_granite-4.0-h-350m-unsloth-mxfp4_moe-attn_kv_BF16-attn_output_BF16-attn_q_BF16-embeddings_BF16-ffn_down_BF16-ffn_up_gate_BF16.gguf (version GGUF V3 (latest))
+llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output.
+llama_model_loader: - kv   0:                       general.architecture str              = granitehybrid
+llama_model_loader: - kv   1:                               general.type str              = model
+llama_model_loader: - kv   2:                               general.name str              = Granite 4.0 H 350m Unsloth
+llama_model_loader: - kv   3:                           general.finetune str              = unsloth
+llama_model_loader: - kv   4:                           general.basename str              = granite-4.0-h
+llama_model_loader: - kv   5:                         general.size_label str              = 350M
+llama_model_loader: - kv   6:                            general.license str              = apache-2.0
+llama_model_loader: - kv   7:                   general.base_model.count u32              = 1
+llama_model_loader: - kv   8:                  general.base_model.0.name str              = Granite 4.0 H 350m
+llama_model_loader: - kv   9:          general.base_model.0.organization str              = Ibm Granite
+llama_model_loader: - kv  10:              general.base_model.0.repo_url str              = https://huggingface.co/ibm-granite/gr...
+llama_model_loader: - kv  11:                               general.tags arr[str,3]       = ["language", "unsloth", "granite-4.0"]
+llama_model_loader: - kv  12:                  granitehybrid.block_count u32              = 32
+llama_model_loader: - kv  13:               granitehybrid.context_length u32              = 1048576
+llama_model_loader: - kv  14:             granitehybrid.embedding_length u32              = 768
+llama_model_loader: - kv  15:          granitehybrid.feed_forward_length u32              = 2048
+llama_model_loader: - kv  16:         granitehybrid.attention.head_count u32              = 12
+llama_model_loader: - kv  17:      granitehybrid.attention.head_count_kv arr[i32,32]      = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, ...
+llama_model_loader: - kv  18:               granitehybrid.rope.freq_base f32              = 10000.000000
+llama_model_loader: - kv  19: granitehybrid.attention.layer_norm_rms_epsilon f32              = 0.000010
+llama_model_loader: - kv  20:                 granitehybrid.expert_count u32              = 0
+llama_model_loader: - kv  21:            granitehybrid.expert_used_count u32              = 0
+llama_model_loader: - kv  22:                   granitehybrid.vocab_size u32              = 100352
+llama_model_loader: - kv  23:         granitehybrid.rope.dimension_count u32              = 64
+llama_model_loader: - kv  24:              granitehybrid.attention.scale f32              = 0.015625
+llama_model_loader: - kv  25:              granitehybrid.embedding_scale f32              = 12.000000
+llama_model_loader: - kv  26:               granitehybrid.residual_scale f32              = 0.246000
+llama_model_loader: - kv  27:                  granitehybrid.logit_scale f32              = 3.000000
+llama_model_loader: - kv  28: granitehybrid.expert_shared_feed_forward_length u32              = 2048
+llama_model_loader: - kv  29:              granitehybrid.ssm.conv_kernel u32              = 4
+llama_model_loader: - kv  30:               granitehybrid.ssm.state_size u32              = 128
+llama_model_loader: - kv  31:              granitehybrid.ssm.group_count u32              = 1
+llama_model_loader: - kv  32:               granitehybrid.ssm.inner_size u32              = 1536
+llama_model_loader: - kv  33:           granitehybrid.ssm.time_step_rank u32              = 48
+llama_model_loader: - kv  34:       granitehybrid.rope.scaling.finetuned bool             = false
+llama_model_loader: - kv  35:                       tokenizer.ggml.model str              = gpt2
+llama_model_loader: - kv  36:                         tokenizer.ggml.pre str              = dbrx
+llama_model_loader: - kv  37:                      tokenizer.ggml.tokens arr[str,100352]  = ["!", "\"", "#", "$", "%", "&", "'", ...
+llama_model_loader: - kv  38:                  tokenizer.ggml.token_type arr[i32,100352]  = [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...
+llama_model_loader: - kv  39:                      tokenizer.ggml.merges arr[str,100000]  = ["Ġ Ġ", "ĠĠ ĠĠ", "i n", "Ġ t",...
+llama_model_loader: - kv  40:                tokenizer.ggml.bos_token_id u32              = 100257
+llama_model_loader: - kv  41:                tokenizer.ggml.eos_token_id u32              = 100257
+llama_model_loader: - kv  42:            tokenizer.ggml.unknown_token_id u32              = 100269
+llama_model_loader: - kv  43:            tokenizer.ggml.padding_token_id u32              = 100256
+llama_model_loader: - kv  44:               tokenizer.ggml.add_bos_token bool             = false
+llama_model_loader: - kv  45:                    tokenizer.chat_template str              = {%- set tools_system_message_prefix =...
+llama_model_loader: - kv  46:               general.quantization_version u32              = 2
+llama_model_loader: - kv  47:                          general.file_type u32              = 38
+llama_model_loader: - type  f32:  233 tensors
+llama_model_loader: - type q8_0:   56 tensors
+llama_model_loader: - type bf16:  113 tensors
+print_info: file format = GGUF V3 (latest)
+print_info: file type   = MXFP4 MoE
+print_info: file size   = 555.36 MiB (13.69 BPW) 
+load: printing all EOG tokens:
+load:   - 100257 ('<|end_of_text|>')
+load:   - 100261 ('<|fim_pad|>')
+load: special tokens cache size = 96
+load: token to piece cache size = 0.6152 MB
+print_info: arch             = granitehybrid
+print_info: vocab_only       = 0
+print_info: n_ctx_train      = 1048576
+print_info: n_embd           = 768
+print_info: n_embd_inp       = 768
+print_info: n_layer          = 32
+print_info: n_head           = 12
+print_info: n_head_kv        = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0]
+print_info: n_rot            = 64
+print_info: n_swa            = 0
+print_info: is_swa_any       = 0
+print_info: n_embd_head_k    = 64
+print_info: n_embd_head_v    = 64
+print_info: n_gqa            = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 3, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0]
+print_info: n_embd_k_gqa     = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 256, 0, 0, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 0, 0]
+print_info: n_embd_v_gqa     = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 256, 0, 0, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 0, 0]
+print_info: f_norm_eps       = 0.0e+00
+print_info: f_norm_rms_eps   = 1.0e-05
+print_info: f_clamp_kqv      = 0.0e+00
+print_info: f_max_alibi_bias = 0.0e+00
+print_info: f_logit_scale    = 3.0e+00
+print_info: f_attn_scale     = 1.6e-02
+print_info: n_ff             = 2048
+print_info: n_expert         = 0
+print_info: n_expert_used    = 0
+print_info: n_expert_groups  = 0
+print_info: n_group_used     = 0
+print_info: causal attn      = 1
+print_info: pooling type     = 0
+print_info: rope type        = 0
+print_info: rope scaling     = linear
+print_info: freq_base_train  = 10000.0
+print_info: freq_scale_train = 1
+print_info: n_ctx_orig_yarn  = 1048576
+print_info: rope_finetuned   = unknown
+print_info: ssm_d_conv       = 4
+print_info: ssm_d_inner      = 1536
+print_info: ssm_d_state      = 128
+print_info: ssm_dt_rank      = 48
+print_info: ssm_n_group      = 1
+print_info: ssm_dt_b_c_rms   = 0
+print_info: model type       = 350M
+print_info: model params     = 340.33 M
+print_info: general.name     = Granite 4.0 H 350m Unsloth
+print_info: f_embedding_scale = 12.000000
+print_info: f_residual_scale  = 0.246000
+print_info: f_attention_scale = 0.015625
+print_info: n_ff_shexp        = 2048
+print_info: vocab type       = BPE
+print_info: n_vocab          = 100352
+print_info: n_merges         = 100000
+print_info: BOS token        = 100257 '<|end_of_text|>'
+print_info: EOS token        = 100257 '<|end_of_text|>'
+print_info: EOT token        = 100257 '<|end_of_text|>'
+print_info: UNK token        = 100269 '<|unk|>'
+print_info: PAD token        = 100256 '<|pad|>'
+print_info: LF token         = 198 'Ċ'
+print_info: FIM PRE token    = 100258 '<|fim_prefix|>'
+print_info: FIM SUF token    = 100260 '<|fim_suffix|>'
+print_info: FIM MID token    = 100259 '<|fim_middle|>'
+print_info: FIM PAD token    = 100261 '<|fim_pad|>'
+print_info: EOG token        = 100257 '<|end_of_text|>'
+print_info: EOG token        = 100261 '<|fim_pad|>'
+print_info: max token length = 256
+load_tensors: loading model tensors, this can take a while... (mmap = true)
+load_tensors: offloading 20 repeating layers to GPU
+load_tensors: offloaded 20/33 layers to GPU
+load_tensors:   CPU_Mapped model buffer size =   300.57 MiB
+load_tensors:        CUDA0 model buffer size =   126.97 MiB
+load_tensors:        CUDA1 model buffer size =   127.83 MiB
+...........................................................................
+llama_context: constructing llama_context
+llama_context: n_seq_max     = 1
+llama_context: n_ctx         = 2048
+llama_context: n_ctx_seq     = 2048
+llama_context: n_batch       = 2048
+llama_context: n_ubatch      = 512
+llama_context: causal_attn   = 1
+llama_context: flash_attn    = auto
+llama_context: kv_unified    = false
+llama_context: freq_base     = 10000.0
+llama_context: freq_scale    = 1
+llama_context: n_ctx_seq (2048) < n_ctx_train (1048576) -- the full capacity of the model will not be utilized
+llama_context:        CPU  output buffer size =     0.38 MiB
+llama_kv_cache:        CPU KV buffer size =     2.00 MiB
+llama_kv_cache:      CUDA0 KV buffer size =     4.00 MiB
+llama_kv_cache:      CUDA1 KV buffer size =     2.00 MiB
+llama_kv_cache: size =    8.00 MiB (  2048 cells,   4 layers,  1/1 seqs), K (f16):    4.00 MiB, V (f16):    4.00 MiB
+llama_memory_recurrent:        CPU RS buffer size =     8.48 MiB
+llama_memory_recurrent:      CUDA0 RS buffer size =     6.16 MiB
+llama_memory_recurrent:      CUDA1 RS buffer size =     6.93 MiB
+llama_memory_recurrent: size =   21.57 MiB (     1 cells,  32 layers,  1 seqs), R (f32):    0.57 MiB, S (f32):   21.00 MiB
+llama_context: Flash Attention was auto, set to enabled
+llama_context:      CUDA0 compute buffer size =   352.59 MiB
+llama_context:      CUDA1 compute buffer size =    22.39 MiB
+llama_context:  CUDA_Host compute buffer size =    18.34 MiB
+llama_context: graph nodes  = 1815
+llama_context: graph splits = 182 (with bs=512), 41 (with bs=1)
+common_init_from_params: added <|end_of_text|> logit bias = -inf
+common_init_from_params: added <|fim_pad|> logit bias = -inf
+common_init_from_params: setting dry_penalty_last_n to ctx_size = 2048
+common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable)
+
+system_info: n_threads = 16 (n_threads_batch = 16) / 32 | CUDA : ARCHS = 860 | USE_GRAPHS = 1 | PEER_MAX_BATCH_SIZE = 128 | CPU : SSE3 = 1 | SSSE3 = 1 | AVX = 1 | AVX2 = 1 | F16C = 1 | FMA = 1 | BMI2 = 1 | AVX512 = 1 | AVX512_VBMI = 1 | AVX512_VNNI = 1 | AVX512_BF16 = 1 | LLAMAFILE = 1 | OPENMP = 1 | REPACK = 1 | 
+perplexity: tokenizing the input ..
+perplexity: tokenization took 36.784 ms
+perplexity: calculating perplexity over 14 chunks, n_ctx=2048, batch_size=2048, n_seq=1
+perplexity: 0.60 seconds per pass - ETA 0.13 minutes
+[1]18.5301,[2]21.5751,[3]22.2087,[4]20.1928,[5]20.1978,[6]18.0174,[7]17.6484,[8]17.6034,[9]18.1079,[10]18.0862,[11]17.9257,[12]18.0408,[13]18.1116,[14]18.1504,
+Final estimate: PPL = 18.1504 +/- 0.46645
+
+llama_perf_context_print:        load time =     248.09 ms
+llama_perf_context_print: prompt eval time =    5300.18 ms / 28672 tokens (    0.18 ms per token,  5409.62 tokens per second)
+llama_perf_context_print:        eval time =       0.00 ms /     1 runs   (    0.00 ms per token,      inf tokens per second)
+llama_perf_context_print:       total time =    5574.70 ms / 28673 tokens
+llama_perf_context_print:    graphs reused =          0
+llama_memory_breakdown_print: | memory breakdown [MiB] | total    free    self   model   context   compute    unaccounted |
+llama_memory_breakdown_print: |   - CUDA0 (RTX 3090)   | 24115 = 20188 + ( 490 =   126 +      10 +     353) +        3436 |
+llama_memory_breakdown_print: |   - CUDA1 (RTX 3090)   | 24124 = 23339 + ( 159 =   127 +       8 +      22) +         625 |
+llama_memory_breakdown_print: |   - Host               |                   329 =   300 +      10 +      18                |

Benchmarks/DataCollection/granite-4.0-h-350m-unsloth-mxfp4_moe-attn_kv_BF16-attn_output_BF16-attn_q_BF16-embeddings_BF16-ffn_down_BF16-ffn_up_gate_BF16/perplexity_math.log

@@ -0,0 +1,189 @@
+ggml_cuda_init: GGML_CUDA_FORCE_MMQ:    no
+ggml_cuda_init: GGML_CUDA_FORCE_CUBLAS: no
+ggml_cuda_init: found 2 CUDA devices:
+  Device 0: NVIDIA GeForce RTX 3090, compute capability 8.6, VMM: yes
+  Device 1: NVIDIA GeForce RTX 3090, compute capability 8.6, VMM: yes
+build: 7040 (92bb442ad) with cc (Ubuntu 13.3.0-6ubuntu2~24.04) 13.3.0 for x86_64-linux-gnu
+llama_model_load_from_file_impl: using device CUDA0 (NVIDIA GeForce RTX 3090) (0000:01:00.0) - 20831 MiB free
+llama_model_load_from_file_impl: using device CUDA1 (NVIDIA GeForce RTX 3090) (0000:03:00.0) - 23581 MiB free
+llama_model_loader: loaded meta data with 48 key-value pairs and 402 tensors from /mnt/world8/AI/ToBench/granite-4.0-h-350m-unsloth/Magic_Quant/GGUF/dc_round-1_granite-4.0-h-350m-unsloth-mxfp4_moe-attn_kv_BF16-attn_output_BF16-attn_q_BF16-embeddings_BF16-ffn_down_BF16-ffn_up_gate_BF16.gguf (version GGUF V3 (latest))
+llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output.
+llama_model_loader: - kv   0:                       general.architecture str              = granitehybrid
+llama_model_loader: - kv   1:                               general.type str              = model
+llama_model_loader: - kv   2:                               general.name str              = Granite 4.0 H 350m Unsloth
+llama_model_loader: - kv   3:                           general.finetune str              = unsloth
+llama_model_loader: - kv   4:                           general.basename str              = granite-4.0-h
+llama_model_loader: - kv   5:                         general.size_label str              = 350M
+llama_model_loader: - kv   6:                            general.license str              = apache-2.0
+llama_model_loader: - kv   7:                   general.base_model.count u32              = 1
+llama_model_loader: - kv   8:                  general.base_model.0.name str              = Granite 4.0 H 350m
+llama_model_loader: - kv   9:          general.base_model.0.organization str              = Ibm Granite
+llama_model_loader: - kv  10:              general.base_model.0.repo_url str              = https://huggingface.co/ibm-granite/gr...
+llama_model_loader: - kv  11:                               general.tags arr[str,3]       = ["language", "unsloth", "granite-4.0"]
+llama_model_loader: - kv  12:                  granitehybrid.block_count u32              = 32
+llama_model_loader: - kv  13:               granitehybrid.context_length u32              = 1048576
+llama_model_loader: - kv  14:             granitehybrid.embedding_length u32              = 768
+llama_model_loader: - kv  15:          granitehybrid.feed_forward_length u32              = 2048
+llama_model_loader: - kv  16:         granitehybrid.attention.head_count u32              = 12
+llama_model_loader: - kv  17:      granitehybrid.attention.head_count_kv arr[i32,32]      = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, ...
+llama_model_loader: - kv  18:               granitehybrid.rope.freq_base f32              = 10000.000000
+llama_model_loader: - kv  19: granitehybrid.attention.layer_norm_rms_epsilon f32              = 0.000010
+llama_model_loader: - kv  20:                 granitehybrid.expert_count u32              = 0
+llama_model_loader: - kv  21:            granitehybrid.expert_used_count u32              = 0
+llama_model_loader: - kv  22:                   granitehybrid.vocab_size u32              = 100352
+llama_model_loader: - kv  23:         granitehybrid.rope.dimension_count u32              = 64
+llama_model_loader: - kv  24:              granitehybrid.attention.scale f32              = 0.015625
+llama_model_loader: - kv  25:              granitehybrid.embedding_scale f32              = 12.000000
+llama_model_loader: - kv  26:               granitehybrid.residual_scale f32              = 0.246000
+llama_model_loader: - kv  27:                  granitehybrid.logit_scale f32              = 3.000000
+llama_model_loader: - kv  28: granitehybrid.expert_shared_feed_forward_length u32              = 2048
+llama_model_loader: - kv  29:              granitehybrid.ssm.conv_kernel u32              = 4
+llama_model_loader: - kv  30:               granitehybrid.ssm.state_size u32              = 128
+llama_model_loader: - kv  31:              granitehybrid.ssm.group_count u32              = 1
+llama_model_loader: - kv  32:               granitehybrid.ssm.inner_size u32              = 1536
+llama_model_loader: - kv  33:           granitehybrid.ssm.time_step_rank u32              = 48
+llama_model_loader: - kv  34:       granitehybrid.rope.scaling.finetuned bool             = false
+llama_model_loader: - kv  35:                       tokenizer.ggml.model str              = gpt2
+llama_model_loader: - kv  36:                         tokenizer.ggml.pre str              = dbrx
+llama_model_loader: - kv  37:                      tokenizer.ggml.tokens arr[str,100352]  = ["!", "\"", "#", "$", "%", "&", "'", ...
+llama_model_loader: - kv  38:                  tokenizer.ggml.token_type arr[i32,100352]  = [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...
+llama_model_loader: - kv  39:                      tokenizer.ggml.merges arr[str,100000]  = ["Ġ Ġ", "ĠĠ ĠĠ", "i n", "Ġ t",...
+llama_model_loader: - kv  40:                tokenizer.ggml.bos_token_id u32              = 100257
+llama_model_loader: - kv  41:                tokenizer.ggml.eos_token_id u32              = 100257
+llama_model_loader: - kv  42:            tokenizer.ggml.unknown_token_id u32              = 100269
+llama_model_loader: - kv  43:            tokenizer.ggml.padding_token_id u32              = 100256
+llama_model_loader: - kv  44:               tokenizer.ggml.add_bos_token bool             = false
+llama_model_loader: - kv  45:                    tokenizer.chat_template str              = {%- set tools_system_message_prefix =...
+llama_model_loader: - kv  46:               general.quantization_version u32              = 2
+llama_model_loader: - kv  47:                          general.file_type u32              = 38
+llama_model_loader: - type  f32:  233 tensors
+llama_model_loader: - type q8_0:   56 tensors
+llama_model_loader: - type bf16:  113 tensors
+print_info: file format = GGUF V3 (latest)
+print_info: file type   = MXFP4 MoE
+print_info: file size   = 555.36 MiB (13.69 BPW) 
+load: printing all EOG tokens:
+load:   - 100257 ('<|end_of_text|>')
+load:   - 100261 ('<|fim_pad|>')
+load: special tokens cache size = 96
+load: token to piece cache size = 0.6152 MB
+print_info: arch             = granitehybrid
+print_info: vocab_only       = 0
+print_info: n_ctx_train      = 1048576
+print_info: n_embd           = 768
+print_info: n_embd_inp       = 768
+print_info: n_layer          = 32
+print_info: n_head           = 12
+print_info: n_head_kv        = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0]
+print_info: n_rot            = 64
+print_info: n_swa            = 0
+print_info: is_swa_any       = 0
+print_info: n_embd_head_k    = 64
+print_info: n_embd_head_v    = 64
+print_info: n_gqa            = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 3, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0]
+print_info: n_embd_k_gqa     = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 256, 0, 0, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 0, 0]
+print_info: n_embd_v_gqa     = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 256, 0, 0, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 0, 0]
+print_info: f_norm_eps       = 0.0e+00
+print_info: f_norm_rms_eps   = 1.0e-05
+print_info: f_clamp_kqv      = 0.0e+00
+print_info: f_max_alibi_bias = 0.0e+00
+print_info: f_logit_scale    = 3.0e+00
+print_info: f_attn_scale     = 1.6e-02
+print_info: n_ff             = 2048
+print_info: n_expert         = 0
+print_info: n_expert_used    = 0
+print_info: n_expert_groups  = 0
+print_info: n_group_used     = 0
+print_info: causal attn      = 1
+print_info: pooling type     = 0
+print_info: rope type        = 0
+print_info: rope scaling     = linear
+print_info: freq_base_train  = 10000.0
+print_info: freq_scale_train = 1
+print_info: n_ctx_orig_yarn  = 1048576
+print_info: rope_finetuned   = unknown
+print_info: ssm_d_conv       = 4
+print_info: ssm_d_inner      = 1536
+print_info: ssm_d_state      = 128
+print_info: ssm_dt_rank      = 48
+print_info: ssm_n_group      = 1
+print_info: ssm_dt_b_c_rms   = 0
+print_info: model type       = 350M
+print_info: model params     = 340.33 M
+print_info: general.name     = Granite 4.0 H 350m Unsloth
+print_info: f_embedding_scale = 12.000000
+print_info: f_residual_scale  = 0.246000
+print_info: f_attention_scale = 0.015625
+print_info: n_ff_shexp        = 2048
+print_info: vocab type       = BPE
+print_info: n_vocab          = 100352
+print_info: n_merges         = 100000
+print_info: BOS token        = 100257 '<|end_of_text|>'
+print_info: EOS token        = 100257 '<|end_of_text|>'
+print_info: EOT token        = 100257 '<|end_of_text|>'
+print_info: UNK token        = 100269 '<|unk|>'
+print_info: PAD token        = 100256 '<|pad|>'
+print_info: LF token         = 198 'Ċ'
+print_info: FIM PRE token    = 100258 '<|fim_prefix|>'
+print_info: FIM SUF token    = 100260 '<|fim_suffix|>'
+print_info: FIM MID token    = 100259 '<|fim_middle|>'
+print_info: FIM PAD token    = 100261 '<|fim_pad|>'
+print_info: EOG token        = 100257 '<|end_of_text|>'
+print_info: EOG token        = 100261 '<|fim_pad|>'
+print_info: max token length = 256
+load_tensors: loading model tensors, this can take a while... (mmap = true)
+load_tensors: offloading 20 repeating layers to GPU
+load_tensors: offloaded 20/33 layers to GPU
+load_tensors:   CPU_Mapped model buffer size =   300.57 MiB
+load_tensors:        CUDA0 model buffer size =   126.97 MiB
+load_tensors:        CUDA1 model buffer size =   127.83 MiB
+...........................................................................
+llama_context: constructing llama_context
+llama_context: n_seq_max     = 1
+llama_context: n_ctx         = 2048
+llama_context: n_ctx_seq     = 2048
+llama_context: n_batch       = 2048
+llama_context: n_ubatch      = 512
+llama_context: causal_attn   = 1
+llama_context: flash_attn    = auto
+llama_context: kv_unified    = false
+llama_context: freq_base     = 10000.0
+llama_context: freq_scale    = 1
+llama_context: n_ctx_seq (2048) < n_ctx_train (1048576) -- the full capacity of the model will not be utilized
+llama_context:        CPU  output buffer size =     0.38 MiB
+llama_kv_cache:        CPU KV buffer size =     2.00 MiB
+llama_kv_cache:      CUDA0 KV buffer size =     4.00 MiB
+llama_kv_cache:      CUDA1 KV buffer size =     2.00 MiB
+llama_kv_cache: size =    8.00 MiB (  2048 cells,   4 layers,  1/1 seqs), K (f16):    4.00 MiB, V (f16):    4.00 MiB
+llama_memory_recurrent:        CPU RS buffer size =     8.48 MiB
+llama_memory_recurrent:      CUDA0 RS buffer size =     6.16 MiB
+llama_memory_recurrent:      CUDA1 RS buffer size =     6.93 MiB
+llama_memory_recurrent: size =   21.57 MiB (     1 cells,  32 layers,  1 seqs), R (f32):    0.57 MiB, S (f32):   21.00 MiB
+llama_context: Flash Attention was auto, set to enabled
+llama_context:      CUDA0 compute buffer size =   352.59 MiB
+llama_context:      CUDA1 compute buffer size =    22.39 MiB
+llama_context:  CUDA_Host compute buffer size =    18.34 MiB
+llama_context: graph nodes  = 1815
+llama_context: graph splits = 182 (with bs=512), 41 (with bs=1)
+common_init_from_params: added <|end_of_text|> logit bias = -inf
+common_init_from_params: added <|fim_pad|> logit bias = -inf
+common_init_from_params: setting dry_penalty_last_n to ctx_size = 2048
+common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable)
+
+system_info: n_threads = 16 (n_threads_batch = 16) / 32 | CUDA : ARCHS = 860 | USE_GRAPHS = 1 | PEER_MAX_BATCH_SIZE = 128 | CPU : SSE3 = 1 | SSSE3 = 1 | AVX = 1 | AVX2 = 1 | F16C = 1 | FMA = 1 | BMI2 = 1 | AVX512 = 1 | AVX512_VBMI = 1 | AVX512_VNNI = 1 | AVX512_BF16 = 1 | LLAMAFILE = 1 | OPENMP = 1 | REPACK = 1 | 
+perplexity: tokenizing the input ..
+perplexity: tokenization took 35.1 ms
+perplexity: calculating perplexity over 15 chunks, n_ctx=2048, batch_size=2048, n_seq=1
+perplexity: 0.61 seconds per pass - ETA 0.15 minutes
+[1]8.7168,[2]9.9322,[3]9.4907,[4]9.8241,[5]9.9717,[6]10.0546,[7]10.2087,[8]9.9017,[9]9.9610,[10]9.9720,[11]10.2124,[12]10.2978,[13]10.4200,[14]10.3965,[15]10.2970,
+Final estimate: PPL = 10.2970 +/- 0.23199
+
+llama_perf_context_print:        load time =     245.36 ms
+llama_perf_context_print: prompt eval time =    5661.06 ms / 30720 tokens (    0.18 ms per token,  5426.55 tokens per second)
+llama_perf_context_print:        eval time =       0.00 ms /     1 runs   (    0.00 ms per token,      inf tokens per second)
+llama_perf_context_print:       total time =    5935.12 ms / 30721 tokens
+llama_perf_context_print:    graphs reused =          0
+llama_memory_breakdown_print: | memory breakdown [MiB] | total    free    self   model   context   compute    unaccounted |
+llama_memory_breakdown_print: |   - CUDA0 (RTX 3090)   | 24115 = 20184 + ( 490 =   126 +      10 +     353) +        3440 |
+llama_memory_breakdown_print: |   - CUDA1 (RTX 3090)   | 24124 = 23339 + ( 159 =   127 +       8 +      22) +         625 |
+llama_memory_breakdown_print: |   - Host               |                   329 =   300 +      10 +      18                |

Benchmarks/DataCollection/granite-4.0-h-350m-unsloth-mxfp4_moe-attn_kv_BF16-attn_output_BF16-attn_q_BF16-embeddings_BF16-ffn_down_BF16-ffn_up_gate_MXFP4/bench_metrics.json

@@ -0,0 +1,44 @@
+{
+  "raw_metrics": {
+    "llamabench": {
+      "backend": "CUDA",
+      "log_path": "granite-4.0-h-350m-unsloth-mxfp4_moe-attn_kv_BF16-attn_output_BF16-attn_q_BF16-embeddings_BF16-ffn_down_BF16-ffn_up_gate_MXFP4/llamabench.md",
+      "ngl": "35",
+      "raw_row": {
+        "backend": "CUDA",
+        "model": "granitehybrid 350M MXFP4 MoE",
+        "ngl": "35",
+        "params": "340.33 M",
+        "size": "414.36 MiB",
+        "t/s": "1734.54 \u00b1 51.60",
+        "test": "pp8",
+        "tps_value": 1734.54
+      },
+      "test": "pp8",
+      "tps": 1734.54
+    },
+    "perplexity": {
+      "code": {
+        "log_path": "granite-4.0-h-350m-unsloth-mxfp4_moe-attn_kv_BF16-attn_output_BF16-attn_q_BF16-embeddings_BF16-ffn_down_BF16-ffn_up_gate_MXFP4/perplexity_code.log",
+        "ppl": 18.2982,
+        "ppl_error": 0.36832
+      },
+      "general": {
+        "log_path": "granite-4.0-h-350m-unsloth-mxfp4_moe-attn_kv_BF16-attn_output_BF16-attn_q_BF16-embeddings_BF16-ffn_down_BF16-ffn_up_gate_MXFP4/perplexity_general.log",
+        "ppl": 125.3344,
+        "ppl_error": 4.38064
+      },
+      "math": {
+        "log_path": "granite-4.0-h-350m-unsloth-mxfp4_moe-attn_kv_BF16-attn_output_BF16-attn_q_BF16-embeddings_BF16-ffn_down_BF16-ffn_up_gate_MXFP4/perplexity_math.log",
+        "ppl": 44.1016,
+        "ppl_error": 1.32144
+      }
+    }
+  },
+  "summary": {
+    "avg_prec_loss_pct": 585.3171,
+    "bench_tps": 1734.54,
+    "file_size_bytes": 438055872,
+    "file_size_gb": 0.41
+  }
+}

Benchmarks/DataCollection/granite-4.0-h-350m-unsloth-mxfp4_moe-attn_kv_BF16-attn_output_BF16-attn_q_BF16-embeddings_BF16-ffn_down_BF16-ffn_up_gate_MXFP4/llamabench.md

@@ -0,0 +1,11 @@
+ggml_cuda_init: GGML_CUDA_FORCE_MMQ:    no
+ggml_cuda_init: GGML_CUDA_FORCE_CUBLAS: no
+ggml_cuda_init: found 2 CUDA devices:
+  Device 0: NVIDIA GeForce RTX 3090, compute capability 8.6, VMM: yes
+  Device 1: NVIDIA GeForce RTX 3090, compute capability 8.6, VMM: yes
+| model                          |       size |     params | backend    | ngl |            test |                  t/s |
+| ------------------------------ | ---------: | ---------: | ---------- | --: | --------------: | -------------------: |
+| granitehybrid 350M MXFP4 MoE   | 414.36 MiB |   340.33 M | CUDA       |  35 |             pp8 |      1734.54 ± 51.60 |
+| granitehybrid 350M MXFP4 MoE   | 414.36 MiB |   340.33 M | CUDA       |  35 |           tg128 |        317.05 ± 8.36 |
+
+build: 92bb442ad (7040)

Benchmarks/DataCollection/granite-4.0-h-350m-unsloth-mxfp4_moe-attn_kv_BF16-attn_output_BF16-attn_q_BF16-embeddings_BF16-ffn_down_BF16-ffn_up_gate_MXFP4/perplexity_code.log

@@ -0,0 +1,190 @@
+ggml_cuda_init: GGML_CUDA_FORCE_MMQ:    no
+ggml_cuda_init: GGML_CUDA_FORCE_CUBLAS: no
+ggml_cuda_init: found 2 CUDA devices:
+  Device 0: NVIDIA GeForce RTX 3090, compute capability 8.6, VMM: yes
+  Device 1: NVIDIA GeForce RTX 3090, compute capability 8.6, VMM: yes
+build: 7040 (92bb442ad) with cc (Ubuntu 13.3.0-6ubuntu2~24.04) 13.3.0 for x86_64-linux-gnu
+llama_model_load_from_file_impl: using device CUDA0 (NVIDIA GeForce RTX 3090) (0000:01:00.0) - 20964 MiB free
+llama_model_load_from_file_impl: using device CUDA1 (NVIDIA GeForce RTX 3090) (0000:03:00.0) - 23581 MiB free
+llama_model_loader: loaded meta data with 48 key-value pairs and 402 tensors from /mnt/world8/AI/ToBench/granite-4.0-h-350m-unsloth/Magic_Quant/GGUF/dc_round-1_granite-4.0-h-350m-unsloth-mxfp4_moe-attn_kv_BF16-attn_output_BF16-attn_q_BF16-embeddings_BF16-ffn_down_BF16-ffn_up_gate_MXFP4.gguf (version GGUF V3 (latest))
+llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output.
+llama_model_loader: - kv   0:                       general.architecture str              = granitehybrid
+llama_model_loader: - kv   1:                               general.type str              = model
+llama_model_loader: - kv   2:                               general.name str              = Granite 4.0 H 350m Unsloth
+llama_model_loader: - kv   3:                           general.finetune str              = unsloth
+llama_model_loader: - kv   4:                           general.basename str              = granite-4.0-h
+llama_model_loader: - kv   5:                         general.size_label str              = 350M
+llama_model_loader: - kv   6:                            general.license str              = apache-2.0
+llama_model_loader: - kv   7:                   general.base_model.count u32              = 1
+llama_model_loader: - kv   8:                  general.base_model.0.name str              = Granite 4.0 H 350m
+llama_model_loader: - kv   9:          general.base_model.0.organization str              = Ibm Granite
+llama_model_loader: - kv  10:              general.base_model.0.repo_url str              = https://huggingface.co/ibm-granite/gr...
+llama_model_loader: - kv  11:                               general.tags arr[str,3]       = ["language", "unsloth", "granite-4.0"]
+llama_model_loader: - kv  12:                  granitehybrid.block_count u32              = 32
+llama_model_loader: - kv  13:               granitehybrid.context_length u32              = 1048576
+llama_model_loader: - kv  14:             granitehybrid.embedding_length u32              = 768
+llama_model_loader: - kv  15:          granitehybrid.feed_forward_length u32              = 2048
+llama_model_loader: - kv  16:         granitehybrid.attention.head_count u32              = 12
+llama_model_loader: - kv  17:      granitehybrid.attention.head_count_kv arr[i32,32]      = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, ...
+llama_model_loader: - kv  18:               granitehybrid.rope.freq_base f32              = 10000.000000
+llama_model_loader: - kv  19: granitehybrid.attention.layer_norm_rms_epsilon f32              = 0.000010
+llama_model_loader: - kv  20:                 granitehybrid.expert_count u32              = 0
+llama_model_loader: - kv  21:            granitehybrid.expert_used_count u32              = 0
+llama_model_loader: - kv  22:                   granitehybrid.vocab_size u32              = 100352
+llama_model_loader: - kv  23:         granitehybrid.rope.dimension_count u32              = 64
+llama_model_loader: - kv  24:              granitehybrid.attention.scale f32              = 0.015625
+llama_model_loader: - kv  25:              granitehybrid.embedding_scale f32              = 12.000000
+llama_model_loader: - kv  26:               granitehybrid.residual_scale f32              = 0.246000
+llama_model_loader: - kv  27:                  granitehybrid.logit_scale f32              = 3.000000
+llama_model_loader: - kv  28: granitehybrid.expert_shared_feed_forward_length u32              = 2048
+llama_model_loader: - kv  29:              granitehybrid.ssm.conv_kernel u32              = 4
+llama_model_loader: - kv  30:               granitehybrid.ssm.state_size u32              = 128
+llama_model_loader: - kv  31:              granitehybrid.ssm.group_count u32              = 1
+llama_model_loader: - kv  32:               granitehybrid.ssm.inner_size u32              = 1536
+llama_model_loader: - kv  33:           granitehybrid.ssm.time_step_rank u32              = 48
+llama_model_loader: - kv  34:       granitehybrid.rope.scaling.finetuned bool             = false
+llama_model_loader: - kv  35:                       tokenizer.ggml.model str              = gpt2
+llama_model_loader: - kv  36:                         tokenizer.ggml.pre str              = dbrx
+llama_model_loader: - kv  37:                      tokenizer.ggml.tokens arr[str,100352]  = ["!", "\"", "#", "$", "%", "&", "'", ...
+llama_model_loader: - kv  38:                  tokenizer.ggml.token_type arr[i32,100352]  = [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...
+llama_model_loader: - kv  39:                      tokenizer.ggml.merges arr[str,100000]  = ["Ġ Ġ", "ĠĠ ĠĠ", "i n", "Ġ t",...
+llama_model_loader: - kv  40:                tokenizer.ggml.bos_token_id u32              = 100257
+llama_model_loader: - kv  41:                tokenizer.ggml.eos_token_id u32              = 100257
+llama_model_loader: - kv  42:            tokenizer.ggml.unknown_token_id u32              = 100269
+llama_model_loader: - kv  43:            tokenizer.ggml.padding_token_id u32              = 100256
+llama_model_loader: - kv  44:               tokenizer.ggml.add_bos_token bool             = false
+llama_model_loader: - kv  45:                    tokenizer.chat_template str              = {%- set tools_system_message_prefix =...
+llama_model_loader: - kv  46:               general.quantization_version u32              = 2
+llama_model_loader: - kv  47:                          general.file_type u32              = 38
+llama_model_loader: - type  f32:  233 tensors
+llama_model_loader: - type q8_0:   56 tensors
+llama_model_loader: - type bf16:   49 tensors
+llama_model_loader: - type mxfp4:   64 tensors
+print_info: file format = GGUF V3 (latest)
+print_info: file type   = MXFP4 MoE
+print_info: file size   = 414.36 MiB (10.21 BPW) 
+load: printing all EOG tokens:
+load:   - 100257 ('<|end_of_text|>')
+load:   - 100261 ('<|fim_pad|>')
+load: special tokens cache size = 96
+load: token to piece cache size = 0.6152 MB
+print_info: arch             = granitehybrid
+print_info: vocab_only       = 0
+print_info: n_ctx_train      = 1048576
+print_info: n_embd           = 768
+print_info: n_embd_inp       = 768
+print_info: n_layer          = 32
+print_info: n_head           = 12
+print_info: n_head_kv        = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0]
+print_info: n_rot            = 64
+print_info: n_swa            = 0
+print_info: is_swa_any       = 0
+print_info: n_embd_head_k    = 64
+print_info: n_embd_head_v    = 64
+print_info: n_gqa            = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 3, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0]
+print_info: n_embd_k_gqa     = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 256, 0, 0, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 0, 0]
+print_info: n_embd_v_gqa     = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 256, 0, 0, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 0, 0]
+print_info: f_norm_eps       = 0.0e+00
+print_info: f_norm_rms_eps   = 1.0e-05
+print_info: f_clamp_kqv      = 0.0e+00
+print_info: f_max_alibi_bias = 0.0e+00
+print_info: f_logit_scale    = 3.0e+00
+print_info: f_attn_scale     = 1.6e-02
+print_info: n_ff             = 2048
+print_info: n_expert         = 0
+print_info: n_expert_used    = 0
+print_info: n_expert_groups  = 0
+print_info: n_group_used     = 0
+print_info: causal attn      = 1
+print_info: pooling type     = 0
+print_info: rope type        = 0
+print_info: rope scaling     = linear
+print_info: freq_base_train  = 10000.0
+print_info: freq_scale_train = 1
+print_info: n_ctx_orig_yarn  = 1048576
+print_info: rope_finetuned   = unknown
+print_info: ssm_d_conv       = 4
+print_info: ssm_d_inner      = 1536
+print_info: ssm_d_state      = 128
+print_info: ssm_dt_rank      = 48
+print_info: ssm_n_group      = 1
+print_info: ssm_dt_b_c_rms   = 0
+print_info: model type       = 350M
+print_info: model params     = 340.33 M
+print_info: general.name     = Granite 4.0 H 350m Unsloth
+print_info: f_embedding_scale = 12.000000
+print_info: f_residual_scale  = 0.246000
+print_info: f_attention_scale = 0.015625
+print_info: n_ff_shexp        = 2048
+print_info: vocab type       = BPE
+print_info: n_vocab          = 100352
+print_info: n_merges         = 100000
+print_info: BOS token        = 100257 '<|end_of_text|>'
+print_info: EOS token        = 100257 '<|end_of_text|>'
+print_info: EOT token        = 100257 '<|end_of_text|>'
+print_info: UNK token        = 100269 '<|unk|>'
+print_info: PAD token        = 100256 '<|pad|>'
+print_info: LF token         = 198 'Ċ'
+print_info: FIM PRE token    = 100258 '<|fim_prefix|>'
+print_info: FIM SUF token    = 100260 '<|fim_suffix|>'
+print_info: FIM MID token    = 100259 '<|fim_middle|>'
+print_info: FIM PAD token    = 100261 '<|fim_pad|>'
+print_info: EOG token        = 100257 '<|end_of_text|>'
+print_info: EOG token        = 100261 '<|fim_pad|>'
+print_info: max token length = 256
+load_tensors: loading model tensors, this can take a while... (mmap = true)
+load_tensors: offloading 20 repeating layers to GPU
+load_tensors: offloaded 20/33 layers to GPU
+load_tensors:   CPU_Mapped model buffer size =   247.69 MiB
+load_tensors:        CUDA0 model buffer size =    82.91 MiB
+load_tensors:        CUDA1 model buffer size =    83.77 MiB
+..................................................................
+llama_context: constructing llama_context
+llama_context: n_seq_max     = 1
+llama_context: n_ctx         = 2048
+llama_context: n_ctx_seq     = 2048
+llama_context: n_batch       = 2048
+llama_context: n_ubatch      = 512
+llama_context: causal_attn   = 1
+llama_context: flash_attn    = auto
+llama_context: kv_unified    = false
+llama_context: freq_base     = 10000.0
+llama_context: freq_scale    = 1
+llama_context: n_ctx_seq (2048) < n_ctx_train (1048576) -- the full capacity of the model will not be utilized
+llama_context:        CPU  output buffer size =     0.38 MiB
+llama_kv_cache:        CPU KV buffer size =     2.00 MiB
+llama_kv_cache:      CUDA0 KV buffer size =     4.00 MiB
+llama_kv_cache:      CUDA1 KV buffer size =     2.00 MiB
+llama_kv_cache: size =    8.00 MiB (  2048 cells,   4 layers,  1/1 seqs), K (f16):    4.00 MiB, V (f16):    4.00 MiB
+llama_memory_recurrent:        CPU RS buffer size =     8.48 MiB
+llama_memory_recurrent:      CUDA0 RS buffer size =     6.16 MiB
+llama_memory_recurrent:      CUDA1 RS buffer size =     6.93 MiB
+llama_memory_recurrent: size =   21.57 MiB (     1 cells,  32 layers,  1 seqs), R (f32):    0.57 MiB, S (f32):   21.00 MiB
+llama_context: Flash Attention was auto, set to enabled
+llama_context:      CUDA0 compute buffer size =   354.62 MiB
+llama_context:      CUDA1 compute buffer size =    22.39 MiB
+llama_context:  CUDA_Host compute buffer size =    18.34 MiB
+llama_context: graph nodes  = 1815
+llama_context: graph splits = 182 (with bs=512), 41 (with bs=1)
+common_init_from_params: added <|end_of_text|> logit bias = -inf
+common_init_from_params: added <|fim_pad|> logit bias = -inf
+common_init_from_params: setting dry_penalty_last_n to ctx_size = 2048
+common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable)
+
+system_info: n_threads = 16 (n_threads_batch = 16) / 32 | CUDA : ARCHS = 860 | USE_GRAPHS = 1 | PEER_MAX_BATCH_SIZE = 128 | CPU : SSE3 = 1 | SSSE3 = 1 | AVX = 1 | AVX2 = 1 | F16C = 1 | FMA = 1 | BMI2 = 1 | AVX512 = 1 | AVX512_VBMI = 1 | AVX512_VNNI = 1 | AVX512_BF16 = 1 | LLAMAFILE = 1 | OPENMP = 1 | REPACK = 1 | 
+perplexity: tokenizing the input ..
+perplexity: tokenization took 91.882 ms
+perplexity: calculating perplexity over 44 chunks, n_ctx=2048, batch_size=2048, n_seq=1
+perplexity: 0.58 seconds per pass - ETA 0.42 minutes
+[1]52.9552,[2]43.7918,[3]22.9055,[4]16.9887,[5]21.3592,[6]28.0513,[7]25.8100,[8]22.7415,[9]17.8488,[10]15.0846,[11]14.9668,[12]16.4021,[13]17.1044,[14]17.6651,[15]17.8092,[16]16.6204,[17]14.9950,[18]14.7711,[19]15.0464,[20]14.2824,[21]13.8918,[22]13.6903,[23]14.9231,[24]15.3913,[25]16.3277,[26]16.4284,[27]16.8133,[28]17.3285,[29]18.3640,[30]18.6652,[31]18.3083,[32]18.9257,[33]19.1146,[34]19.5353,[35]19.1515,[36]19.1113,[37]18.8390,[38]18.6095,[39]19.1577,[40]19.2457,[41]18.9582,[42]19.2107,[43]18.5828,[44]18.2982,
+Final estimate: PPL = 18.2982 +/- 0.36832
+
+llama_perf_context_print:        load time =     219.05 ms
+llama_perf_context_print: prompt eval time =   15276.69 ms / 90112 tokens (    0.17 ms per token,  5898.66 tokens per second)
+llama_perf_context_print:        eval time =       0.00 ms /     1 runs   (    0.00 ms per token,      inf tokens per second)
+llama_perf_context_print:       total time =   16068.96 ms / 90113 tokens
+llama_perf_context_print:    graphs reused =          0
+llama_memory_breakdown_print: | memory breakdown [MiB] | total    free    self   model   context   compute    unaccounted |
+llama_memory_breakdown_print: |   - CUDA0 (RTX 3090)   | 24115 = 20366 + ( 447 =    82 +      10 +     354) +        3301 |
+llama_memory_breakdown_print: |   - CUDA1 (RTX 3090)   | 24124 = 23383 + ( 115 =    83 +       8 +      22) +         625 |
+llama_memory_breakdown_print: |   - Host               |                   276 =   247 +      10 +      18                |

Benchmarks/DataCollection/granite-4.0-h-350m-unsloth-mxfp4_moe-attn_kv_BF16-attn_output_BF16-attn_q_BF16-embeddings_BF16-ffn_down_BF16-ffn_up_gate_MXFP4/perplexity_general.log

@@ -0,0 +1,190 @@
+ggml_cuda_init: GGML_CUDA_FORCE_MMQ:    no
+ggml_cuda_init: GGML_CUDA_FORCE_CUBLAS: no
+ggml_cuda_init: found 2 CUDA devices:
+  Device 0: NVIDIA GeForce RTX 3090, compute capability 8.6, VMM: yes
+  Device 1: NVIDIA GeForce RTX 3090, compute capability 8.6, VMM: yes
+build: 7040 (92bb442ad) with cc (Ubuntu 13.3.0-6ubuntu2~24.04) 13.3.0 for x86_64-linux-gnu
+llama_model_load_from_file_impl: using device CUDA0 (NVIDIA GeForce RTX 3090) (0000:01:00.0) - 20957 MiB free
+llama_model_load_from_file_impl: using device CUDA1 (NVIDIA GeForce RTX 3090) (0000:03:00.0) - 23581 MiB free
+llama_model_loader: loaded meta data with 48 key-value pairs and 402 tensors from /mnt/world8/AI/ToBench/granite-4.0-h-350m-unsloth/Magic_Quant/GGUF/dc_round-1_granite-4.0-h-350m-unsloth-mxfp4_moe-attn_kv_BF16-attn_output_BF16-attn_q_BF16-embeddings_BF16-ffn_down_BF16-ffn_up_gate_MXFP4.gguf (version GGUF V3 (latest))
+llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output.
+llama_model_loader: - kv   0:                       general.architecture str              = granitehybrid
+llama_model_loader: - kv   1:                               general.type str              = model
+llama_model_loader: - kv   2:                               general.name str              = Granite 4.0 H 350m Unsloth
+llama_model_loader: - kv   3:                           general.finetune str              = unsloth
+llama_model_loader: - kv   4:                           general.basename str              = granite-4.0-h
+llama_model_loader: - kv   5:                         general.size_label str              = 350M
+llama_model_loader: - kv   6:                            general.license str              = apache-2.0
+llama_model_loader: - kv   7:                   general.base_model.count u32              = 1
+llama_model_loader: - kv   8:                  general.base_model.0.name str              = Granite 4.0 H 350m
+llama_model_loader: - kv   9:          general.base_model.0.organization str              = Ibm Granite
+llama_model_loader: - kv  10:              general.base_model.0.repo_url str              = https://huggingface.co/ibm-granite/gr...
+llama_model_loader: - kv  11:                               general.tags arr[str,3]       = ["language", "unsloth", "granite-4.0"]
+llama_model_loader: - kv  12:                  granitehybrid.block_count u32              = 32
+llama_model_loader: - kv  13:               granitehybrid.context_length u32              = 1048576
+llama_model_loader: - kv  14:             granitehybrid.embedding_length u32              = 768
+llama_model_loader: - kv  15:          granitehybrid.feed_forward_length u32              = 2048
+llama_model_loader: - kv  16:         granitehybrid.attention.head_count u32              = 12
+llama_model_loader: - kv  17:      granitehybrid.attention.head_count_kv arr[i32,32]      = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, ...
+llama_model_loader: - kv  18:               granitehybrid.rope.freq_base f32              = 10000.000000
+llama_model_loader: - kv  19: granitehybrid.attention.layer_norm_rms_epsilon f32              = 0.000010
+llama_model_loader: - kv  20:                 granitehybrid.expert_count u32              = 0
+llama_model_loader: - kv  21:            granitehybrid.expert_used_count u32              = 0
+llama_model_loader: - kv  22:                   granitehybrid.vocab_size u32              = 100352
+llama_model_loader: - kv  23:         granitehybrid.rope.dimension_count u32              = 64
+llama_model_loader: - kv  24:              granitehybrid.attention.scale f32              = 0.015625
+llama_model_loader: - kv  25:              granitehybrid.embedding_scale f32              = 12.000000
+llama_model_loader: - kv  26:               granitehybrid.residual_scale f32              = 0.246000
+llama_model_loader: - kv  27:                  granitehybrid.logit_scale f32              = 3.000000
+llama_model_loader: - kv  28: granitehybrid.expert_shared_feed_forward_length u32              = 2048
+llama_model_loader: - kv  29:              granitehybrid.ssm.conv_kernel u32              = 4
+llama_model_loader: - kv  30:               granitehybrid.ssm.state_size u32              = 128
+llama_model_loader: - kv  31:              granitehybrid.ssm.group_count u32              = 1
+llama_model_loader: - kv  32:               granitehybrid.ssm.inner_size u32              = 1536
+llama_model_loader: - kv  33:           granitehybrid.ssm.time_step_rank u32              = 48
+llama_model_loader: - kv  34:       granitehybrid.rope.scaling.finetuned bool             = false
+llama_model_loader: - kv  35:                       tokenizer.ggml.model str              = gpt2
+llama_model_loader: - kv  36:                         tokenizer.ggml.pre str              = dbrx
+llama_model_loader: - kv  37:                      tokenizer.ggml.tokens arr[str,100352]  = ["!", "\"", "#", "$", "%", "&", "'", ...
+llama_model_loader: - kv  38:                  tokenizer.ggml.token_type arr[i32,100352]  = [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...
+llama_model_loader: - kv  39:                      tokenizer.ggml.merges arr[str,100000]  = ["Ġ Ġ", "ĠĠ ĠĠ", "i n", "Ġ t",...
+llama_model_loader: - kv  40:                tokenizer.ggml.bos_token_id u32              = 100257
+llama_model_loader: - kv  41:                tokenizer.ggml.eos_token_id u32              = 100257
+llama_model_loader: - kv  42:            tokenizer.ggml.unknown_token_id u32              = 100269
+llama_model_loader: - kv  43:            tokenizer.ggml.padding_token_id u32              = 100256
+llama_model_loader: - kv  44:               tokenizer.ggml.add_bos_token bool             = false
+llama_model_loader: - kv  45:                    tokenizer.chat_template str              = {%- set tools_system_message_prefix =...
+llama_model_loader: - kv  46:               general.quantization_version u32              = 2
+llama_model_loader: - kv  47:                          general.file_type u32              = 38
+llama_model_loader: - type  f32:  233 tensors
+llama_model_loader: - type q8_0:   56 tensors
+llama_model_loader: - type bf16:   49 tensors
+llama_model_loader: - type mxfp4:   64 tensors
+print_info: file format = GGUF V3 (latest)
+print_info: file type   = MXFP4 MoE
+print_info: file size   = 414.36 MiB (10.21 BPW) 
+load: printing all EOG tokens:
+load:   - 100257 ('<|end_of_text|>')
+load:   - 100261 ('<|fim_pad|>')
+load: special tokens cache size = 96
+load: token to piece cache size = 0.6152 MB
+print_info: arch             = granitehybrid
+print_info: vocab_only       = 0
+print_info: n_ctx_train      = 1048576
+print_info: n_embd           = 768
+print_info: n_embd_inp       = 768
+print_info: n_layer          = 32
+print_info: n_head           = 12
+print_info: n_head_kv        = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0]
+print_info: n_rot            = 64
+print_info: n_swa            = 0
+print_info: is_swa_any       = 0
+print_info: n_embd_head_k    = 64
+print_info: n_embd_head_v    = 64
+print_info: n_gqa            = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 3, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0]
+print_info: n_embd_k_gqa     = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 256, 0, 0, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 0, 0]
+print_info: n_embd_v_gqa     = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 256, 0, 0, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 0, 0]
+print_info: f_norm_eps       = 0.0e+00
+print_info: f_norm_rms_eps   = 1.0e-05
+print_info: f_clamp_kqv      = 0.0e+00
+print_info: f_max_alibi_bias = 0.0e+00
+print_info: f_logit_scale    = 3.0e+00
+print_info: f_attn_scale     = 1.6e-02
+print_info: n_ff             = 2048
+print_info: n_expert         = 0
+print_info: n_expert_used    = 0
+print_info: n_expert_groups  = 0
+print_info: n_group_used     = 0
+print_info: causal attn      = 1
+print_info: pooling type     = 0
+print_info: rope type        = 0
+print_info: rope scaling     = linear
+print_info: freq_base_train  = 10000.0
+print_info: freq_scale_train = 1
+print_info: n_ctx_orig_yarn  = 1048576
+print_info: rope_finetuned   = unknown
+print_info: ssm_d_conv       = 4
+print_info: ssm_d_inner      = 1536
+print_info: ssm_d_state      = 128
+print_info: ssm_dt_rank      = 48
+print_info: ssm_n_group      = 1
+print_info: ssm_dt_b_c_rms   = 0
+print_info: model type       = 350M
+print_info: model params     = 340.33 M
+print_info: general.name     = Granite 4.0 H 350m Unsloth
+print_info: f_embedding_scale = 12.000000
+print_info: f_residual_scale  = 0.246000
+print_info: f_attention_scale = 0.015625
+print_info: n_ff_shexp        = 2048
+print_info: vocab type       = BPE
+print_info: n_vocab          = 100352
+print_info: n_merges         = 100000
+print_info: BOS token        = 100257 '<|end_of_text|>'
+print_info: EOS token        = 100257 '<|end_of_text|>'
+print_info: EOT token        = 100257 '<|end_of_text|>'
+print_info: UNK token        = 100269 '<|unk|>'
+print_info: PAD token        = 100256 '<|pad|>'
+print_info: LF token         = 198 'Ċ'
+print_info: FIM PRE token    = 100258 '<|fim_prefix|>'
+print_info: FIM SUF token    = 100260 '<|fim_suffix|>'
+print_info: FIM MID token    = 100259 '<|fim_middle|>'
+print_info: FIM PAD token    = 100261 '<|fim_pad|>'
+print_info: EOG token        = 100257 '<|end_of_text|>'
+print_info: EOG token        = 100261 '<|fim_pad|>'
+print_info: max token length = 256
+load_tensors: loading model tensors, this can take a while... (mmap = true)
+load_tensors: offloading 20 repeating layers to GPU
+load_tensors: offloaded 20/33 layers to GPU
+load_tensors:   CPU_Mapped model buffer size =   247.69 MiB
+load_tensors:        CUDA0 model buffer size =    82.91 MiB
+load_tensors:        CUDA1 model buffer size =    83.77 MiB
+..................................................................
+llama_context: constructing llama_context
+llama_context: n_seq_max     = 1
+llama_context: n_ctx         = 2048
+llama_context: n_ctx_seq     = 2048
+llama_context: n_batch       = 2048
+llama_context: n_ubatch      = 512
+llama_context: causal_attn   = 1
+llama_context: flash_attn    = auto
+llama_context: kv_unified    = false
+llama_context: freq_base     = 10000.0
+llama_context: freq_scale    = 1
+llama_context: n_ctx_seq (2048) < n_ctx_train (1048576) -- the full capacity of the model will not be utilized
+llama_context:        CPU  output buffer size =     0.38 MiB
+llama_kv_cache:        CPU KV buffer size =     2.00 MiB
+llama_kv_cache:      CUDA0 KV buffer size =     4.00 MiB
+llama_kv_cache:      CUDA1 KV buffer size =     2.00 MiB
+llama_kv_cache: size =    8.00 MiB (  2048 cells,   4 layers,  1/1 seqs), K (f16):    4.00 MiB, V (f16):    4.00 MiB
+llama_memory_recurrent:        CPU RS buffer size =     8.48 MiB
+llama_memory_recurrent:      CUDA0 RS buffer size =     6.16 MiB
+llama_memory_recurrent:      CUDA1 RS buffer size =     6.93 MiB
+llama_memory_recurrent: size =   21.57 MiB (     1 cells,  32 layers,  1 seqs), R (f32):    0.57 MiB, S (f32):   21.00 MiB
+llama_context: Flash Attention was auto, set to enabled
+llama_context:      CUDA0 compute buffer size =   354.62 MiB
+llama_context:      CUDA1 compute buffer size =    22.39 MiB
+llama_context:  CUDA_Host compute buffer size =    18.34 MiB
+llama_context: graph nodes  = 1815
+llama_context: graph splits = 182 (with bs=512), 41 (with bs=1)
+common_init_from_params: added <|end_of_text|> logit bias = -inf
+common_init_from_params: added <|fim_pad|> logit bias = -inf
+common_init_from_params: setting dry_penalty_last_n to ctx_size = 2048
+common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable)
+
+system_info: n_threads = 16 (n_threads_batch = 16) / 32 | CUDA : ARCHS = 860 | USE_GRAPHS = 1 | PEER_MAX_BATCH_SIZE = 128 | CPU : SSE3 = 1 | SSSE3 = 1 | AVX = 1 | AVX2 = 1 | F16C = 1 | FMA = 1 | BMI2 = 1 | AVX512 = 1 | AVX512_VBMI = 1 | AVX512_VNNI = 1 | AVX512_BF16 = 1 | LLAMAFILE = 1 | OPENMP = 1 | REPACK = 1 | 
+perplexity: tokenizing the input ..
+perplexity: tokenization took 39.609 ms
+perplexity: calculating perplexity over 14 chunks, n_ctx=2048, batch_size=2048, n_seq=1
+perplexity: 0.60 seconds per pass - ETA 0.13 minutes
+[1]74.0659,[2]150.2700,[3]145.2615,[4]120.6200,[5]129.2156,[6]120.8379,[7]141.1093,[8]148.1939,[9]158.4463,[10]153.5553,[11]151.6594,[12]144.2039,[13]141.5364,[14]125.3344,
+Final estimate: PPL = 125.3344 +/- 4.38064
+
+llama_perf_context_print:        load time =     228.94 ms
+llama_perf_context_print: prompt eval time =    5019.83 ms / 28672 tokens (    0.18 ms per token,  5711.75 tokens per second)
+llama_perf_context_print:        eval time =       0.00 ms /     1 runs   (    0.00 ms per token,      inf tokens per second)
+llama_perf_context_print:       total time =    5283.03 ms / 28673 tokens
+llama_perf_context_print:    graphs reused =          0
+llama_memory_breakdown_print: | memory breakdown [MiB] | total    free    self   model   context   compute    unaccounted |
+llama_memory_breakdown_print: |   - CUDA0 (RTX 3090)   | 24115 = 20366 + ( 447 =    82 +      10 +     354) +        3301 |
+llama_memory_breakdown_print: |   - CUDA1 (RTX 3090)   | 24124 = 23383 + ( 115 =    83 +       8 +      22) +         625 |
+llama_memory_breakdown_print: |   - Host               |                   276 =   247 +      10 +      18                |

Benchmarks/DataCollection/granite-4.0-h-350m-unsloth-mxfp4_moe-attn_kv_BF16-attn_output_BF16-attn_q_BF16-embeddings_BF16-ffn_down_BF16-ffn_up_gate_MXFP4/perplexity_math.log

@@ -0,0 +1,190 @@
+ggml_cuda_init: GGML_CUDA_FORCE_MMQ:    no
+ggml_cuda_init: GGML_CUDA_FORCE_CUBLAS: no
+ggml_cuda_init: found 2 CUDA devices:
+  Device 0: NVIDIA GeForce RTX 3090, compute capability 8.6, VMM: yes
+  Device 1: NVIDIA GeForce RTX 3090, compute capability 8.6, VMM: yes
+build: 7040 (92bb442ad) with cc (Ubuntu 13.3.0-6ubuntu2~24.04) 13.3.0 for x86_64-linux-gnu
+llama_model_load_from_file_impl: using device CUDA0 (NVIDIA GeForce RTX 3090) (0000:01:00.0) - 20964 MiB free
+llama_model_load_from_file_impl: using device CUDA1 (NVIDIA GeForce RTX 3090) (0000:03:00.0) - 23581 MiB free
+llama_model_loader: loaded meta data with 48 key-value pairs and 402 tensors from /mnt/world8/AI/ToBench/granite-4.0-h-350m-unsloth/Magic_Quant/GGUF/dc_round-1_granite-4.0-h-350m-unsloth-mxfp4_moe-attn_kv_BF16-attn_output_BF16-attn_q_BF16-embeddings_BF16-ffn_down_BF16-ffn_up_gate_MXFP4.gguf (version GGUF V3 (latest))
+llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output.
+llama_model_loader: - kv   0:                       general.architecture str              = granitehybrid
+llama_model_loader: - kv   1:                               general.type str              = model
+llama_model_loader: - kv   2:                               general.name str              = Granite 4.0 H 350m Unsloth
+llama_model_loader: - kv   3:                           general.finetune str              = unsloth
+llama_model_loader: - kv   4:                           general.basename str              = granite-4.0-h
+llama_model_loader: - kv   5:                         general.size_label str              = 350M
+llama_model_loader: - kv   6:                            general.license str              = apache-2.0
+llama_model_loader: - kv   7:                   general.base_model.count u32              = 1
+llama_model_loader: - kv   8:                  general.base_model.0.name str              = Granite 4.0 H 350m
+llama_model_loader: - kv   9:          general.base_model.0.organization str              = Ibm Granite
+llama_model_loader: - kv  10:              general.base_model.0.repo_url str              = https://huggingface.co/ibm-granite/gr...
+llama_model_loader: - kv  11:                               general.tags arr[str,3]       = ["language", "unsloth", "granite-4.0"]
+llama_model_loader: - kv  12:                  granitehybrid.block_count u32              = 32
+llama_model_loader: - kv  13:               granitehybrid.context_length u32              = 1048576
+llama_model_loader: - kv  14:             granitehybrid.embedding_length u32              = 768
+llama_model_loader: - kv  15:          granitehybrid.feed_forward_length u32              = 2048
+llama_model_loader: - kv  16:         granitehybrid.attention.head_count u32              = 12
+llama_model_loader: - kv  17:      granitehybrid.attention.head_count_kv arr[i32,32]      = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, ...
+llama_model_loader: - kv  18:               granitehybrid.rope.freq_base f32              = 10000.000000
+llama_model_loader: - kv  19: granitehybrid.attention.layer_norm_rms_epsilon f32              = 0.000010
+llama_model_loader: - kv  20:                 granitehybrid.expert_count u32              = 0
+llama_model_loader: - kv  21:            granitehybrid.expert_used_count u32              = 0
+llama_model_loader: - kv  22:                   granitehybrid.vocab_size u32              = 100352
+llama_model_loader: - kv  23:         granitehybrid.rope.dimension_count u32              = 64
+llama_model_loader: - kv  24:              granitehybrid.attention.scale f32              = 0.015625
+llama_model_loader: - kv  25:              granitehybrid.embedding_scale f32              = 12.000000
+llama_model_loader: - kv  26:               granitehybrid.residual_scale f32              = 0.246000
+llama_model_loader: - kv  27:                  granitehybrid.logit_scale f32              = 3.000000
+llama_model_loader: - kv  28: granitehybrid.expert_shared_feed_forward_length u32              = 2048
+llama_model_loader: - kv  29:              granitehybrid.ssm.conv_kernel u32              = 4
+llama_model_loader: - kv  30:               granitehybrid.ssm.state_size u32              = 128
+llama_model_loader: - kv  31:              granitehybrid.ssm.group_count u32              = 1
+llama_model_loader: - kv  32:               granitehybrid.ssm.inner_size u32              = 1536
+llama_model_loader: - kv  33:           granitehybrid.ssm.time_step_rank u32              = 48
+llama_model_loader: - kv  34:       granitehybrid.rope.scaling.finetuned bool             = false
+llama_model_loader: - kv  35:                       tokenizer.ggml.model str              = gpt2
+llama_model_loader: - kv  36:                         tokenizer.ggml.pre str              = dbrx
+llama_model_loader: - kv  37:                      tokenizer.ggml.tokens arr[str,100352]  = ["!", "\"", "#", "$", "%", "&", "'", ...
+llama_model_loader: - kv  38:                  tokenizer.ggml.token_type arr[i32,100352]  = [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...
+llama_model_loader: - kv  39:                      tokenizer.ggml.merges arr[str,100000]  = ["Ġ Ġ", "ĠĠ ĠĠ", "i n", "Ġ t",...
+llama_model_loader: - kv  40:                tokenizer.ggml.bos_token_id u32              = 100257
+llama_model_loader: - kv  41:                tokenizer.ggml.eos_token_id u32              = 100257
+llama_model_loader: - kv  42:            tokenizer.ggml.unknown_token_id u32              = 100269
+llama_model_loader: - kv  43:            tokenizer.ggml.padding_token_id u32              = 100256
+llama_model_loader: - kv  44:               tokenizer.ggml.add_bos_token bool             = false
+llama_model_loader: - kv  45:                    tokenizer.chat_template str              = {%- set tools_system_message_prefix =...
+llama_model_loader: - kv  46:               general.quantization_version u32              = 2
+llama_model_loader: - kv  47:                          general.file_type u32              = 38
+llama_model_loader: - type  f32:  233 tensors
+llama_model_loader: - type q8_0:   56 tensors
+llama_model_loader: - type bf16:   49 tensors
+llama_model_loader: - type mxfp4:   64 tensors
+print_info: file format = GGUF V3 (latest)
+print_info: file type   = MXFP4 MoE
+print_info: file size   = 414.36 MiB (10.21 BPW) 
+load: printing all EOG tokens:
+load:   - 100257 ('<|end_of_text|>')
+load:   - 100261 ('<|fim_pad|>')
+load: special tokens cache size = 96
+load: token to piece cache size = 0.6152 MB
+print_info: arch             = granitehybrid
+print_info: vocab_only       = 0
+print_info: n_ctx_train      = 1048576
+print_info: n_embd           = 768
+print_info: n_embd_inp       = 768
+print_info: n_layer          = 32
+print_info: n_head           = 12
+print_info: n_head_kv        = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0]
+print_info: n_rot            = 64
+print_info: n_swa            = 0
+print_info: is_swa_any       = 0
+print_info: n_embd_head_k    = 64
+print_info: n_embd_head_v    = 64
+print_info: n_gqa            = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 3, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0]
+print_info: n_embd_k_gqa     = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 256, 0, 0, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 0, 0]
+print_info: n_embd_v_gqa     = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 256, 0, 0, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 0, 0]
+print_info: f_norm_eps       = 0.0e+00
+print_info: f_norm_rms_eps   = 1.0e-05
+print_info: f_clamp_kqv      = 0.0e+00
+print_info: f_max_alibi_bias = 0.0e+00
+print_info: f_logit_scale    = 3.0e+00
+print_info: f_attn_scale     = 1.6e-02
+print_info: n_ff             = 2048
+print_info: n_expert         = 0
+print_info: n_expert_used    = 0
+print_info: n_expert_groups  = 0
+print_info: n_group_used     = 0
+print_info: causal attn      = 1
+print_info: pooling type     = 0
+print_info: rope type        = 0
+print_info: rope scaling     = linear
+print_info: freq_base_train  = 10000.0
+print_info: freq_scale_train = 1
+print_info: n_ctx_orig_yarn  = 1048576
+print_info: rope_finetuned   = unknown
+print_info: ssm_d_conv       = 4
+print_info: ssm_d_inner      = 1536
+print_info: ssm_d_state      = 128
+print_info: ssm_dt_rank      = 48
+print_info: ssm_n_group      = 1
+print_info: ssm_dt_b_c_rms   = 0
+print_info: model type       = 350M
+print_info: model params     = 340.33 M
+print_info: general.name     = Granite 4.0 H 350m Unsloth
+print_info: f_embedding_scale = 12.000000
+print_info: f_residual_scale  = 0.246000
+print_info: f_attention_scale = 0.015625
+print_info: n_ff_shexp        = 2048
+print_info: vocab type       = BPE
+print_info: n_vocab          = 100352
+print_info: n_merges         = 100000
+print_info: BOS token        = 100257 '<|end_of_text|>'
+print_info: EOS token        = 100257 '<|end_of_text|>'
+print_info: EOT token        = 100257 '<|end_of_text|>'
+print_info: UNK token        = 100269 '<|unk|>'
+print_info: PAD token        = 100256 '<|pad|>'
+print_info: LF token         = 198 'Ċ'
+print_info: FIM PRE token    = 100258 '<|fim_prefix|>'
+print_info: FIM SUF token    = 100260 '<|fim_suffix|>'
+print_info: FIM MID token    = 100259 '<|fim_middle|>'
+print_info: FIM PAD token    = 100261 '<|fim_pad|>'
+print_info: EOG token        = 100257 '<|end_of_text|>'
+print_info: EOG token        = 100261 '<|fim_pad|>'
+print_info: max token length = 256
+load_tensors: loading model tensors, this can take a while... (mmap = true)
+load_tensors: offloading 20 repeating layers to GPU
+load_tensors: offloaded 20/33 layers to GPU
+load_tensors:   CPU_Mapped model buffer size =   247.69 MiB
+load_tensors:        CUDA0 model buffer size =    82.91 MiB
+load_tensors:        CUDA1 model buffer size =    83.77 MiB
+..................................................................
+llama_context: constructing llama_context
+llama_context: n_seq_max     = 1
+llama_context: n_ctx         = 2048
+llama_context: n_ctx_seq     = 2048
+llama_context: n_batch       = 2048
+llama_context: n_ubatch      = 512
+llama_context: causal_attn   = 1
+llama_context: flash_attn    = auto
+llama_context: kv_unified    = false
+llama_context: freq_base     = 10000.0
+llama_context: freq_scale    = 1
+llama_context: n_ctx_seq (2048) < n_ctx_train (1048576) -- the full capacity of the model will not be utilized
+llama_context:        CPU  output buffer size =     0.38 MiB
+llama_kv_cache:        CPU KV buffer size =     2.00 MiB
+llama_kv_cache:      CUDA0 KV buffer size =     4.00 MiB
+llama_kv_cache:      CUDA1 KV buffer size =     2.00 MiB
+llama_kv_cache: size =    8.00 MiB (  2048 cells,   4 layers,  1/1 seqs), K (f16):    4.00 MiB, V (f16):    4.00 MiB
+llama_memory_recurrent:        CPU RS buffer size =     8.48 MiB
+llama_memory_recurrent:      CUDA0 RS buffer size =     6.16 MiB
+llama_memory_recurrent:      CUDA1 RS buffer size =     6.93 MiB
+llama_memory_recurrent: size =   21.57 MiB (     1 cells,  32 layers,  1 seqs), R (f32):    0.57 MiB, S (f32):   21.00 MiB
+llama_context: Flash Attention was auto, set to enabled
+llama_context:      CUDA0 compute buffer size =   354.62 MiB
+llama_context:      CUDA1 compute buffer size =    22.39 MiB
+llama_context:  CUDA_Host compute buffer size =    18.34 MiB
+llama_context: graph nodes  = 1815
+llama_context: graph splits = 182 (with bs=512), 41 (with bs=1)
+common_init_from_params: added <|end_of_text|> logit bias = -inf
+common_init_from_params: added <|fim_pad|> logit bias = -inf
+common_init_from_params: setting dry_penalty_last_n to ctx_size = 2048
+common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable)
+
+system_info: n_threads = 16 (n_threads_batch = 16) / 32 | CUDA : ARCHS = 860 | USE_GRAPHS = 1 | PEER_MAX_BATCH_SIZE = 128 | CPU : SSE3 = 1 | SSSE3 = 1 | AVX = 1 | AVX2 = 1 | F16C = 1 | FMA = 1 | BMI2 = 1 | AVX512 = 1 | AVX512_VBMI = 1 | AVX512_VNNI = 1 | AVX512_BF16 = 1 | LLAMAFILE = 1 | OPENMP = 1 | REPACK = 1 | 
+perplexity: tokenizing the input ..
+perplexity: tokenization took 34.765 ms
+perplexity: calculating perplexity over 15 chunks, n_ctx=2048, batch_size=2048, n_seq=1
+perplexity: 0.58 seconds per pass - ETA 0.13 minutes
+[1]39.6244,[2]43.3639,[3]41.1845,[4]41.3907,[5]43.5928,[6]43.0914,[7]43.5843,[8]40.3787,[9]40.8358,[10]43.4638,[11]44.0296,[12]44.7357,[13]46.3341,[14]44.8815,[15]44.1016,
+Final estimate: PPL = 44.1016 +/- 1.32144
+
+llama_perf_context_print:        load time =     225.20 ms
+llama_perf_context_print: prompt eval time =    5331.90 ms / 30720 tokens (    0.17 ms per token,  5761.55 tokens per second)
+llama_perf_context_print:        eval time =       0.00 ms /     1 runs   (    0.00 ms per token,      inf tokens per second)
+llama_perf_context_print:       total time =    5606.71 ms / 30721 tokens
+llama_perf_context_print:    graphs reused =          0
+llama_memory_breakdown_print: | memory breakdown [MiB] | total    free    self   model   context   compute    unaccounted |
+llama_memory_breakdown_print: |   - CUDA0 (RTX 3090)   | 24115 = 20369 + ( 447 =    82 +      10 +     354) +        3297 |
+llama_memory_breakdown_print: |   - CUDA1 (RTX 3090)   | 24124 = 23383 + ( 115 =    83 +       8 +      22) +         625 |
+llama_memory_breakdown_print: |   - Host               |                   276 =   247 +      10 +      18                |

Benchmarks/DataCollection/granite-4.0-h-350m-unsloth-mxfp4_moe-attn_kv_BF16-attn_output_BF16-attn_q_BF16-embeddings_BF16-ffn_down_MXFP4-ffn_up_gate_BF16/bench_metrics.json

@@ -0,0 +1,44 @@
+{
+  "raw_metrics": {
+    "llamabench": {
+      "backend": "CUDA",
+      "log_path": "granite-4.0-h-350m-unsloth-mxfp4_moe-attn_kv_BF16-attn_output_BF16-attn_q_BF16-embeddings_BF16-ffn_down_MXFP4-ffn_up_gate_BF16/llamabench.md",
+      "ngl": "35",
+      "raw_row": {
+        "backend": "CUDA",
+        "model": "granitehybrid 350M MXFP4 MoE",
+        "ngl": "35",
+        "params": "340.33 M",
+        "size": "484.86 MiB",
+        "t/s": "1815.38 \u00b1 57.44",
+        "test": "pp8",
+        "tps_value": 1815.38
+      },
+      "test": "pp8",
+      "tps": 1815.38
+    },
+    "perplexity": {
+      "code": {
+        "log_path": "granite-4.0-h-350m-unsloth-mxfp4_moe-attn_kv_BF16-attn_output_BF16-attn_q_BF16-embeddings_BF16-ffn_down_MXFP4-ffn_up_gate_BF16/perplexity_code.log",
+        "ppl": 2.5714,
+        "ppl_error": 0.02823
+      },
+      "general": {
+        "log_path": "granite-4.0-h-350m-unsloth-mxfp4_moe-attn_kv_BF16-attn_output_BF16-attn_q_BF16-embeddings_BF16-ffn_down_MXFP4-ffn_up_gate_BF16/perplexity_general.log",
+        "ppl": 27.3444,
+        "ppl_error": 0.76026
+      },
+      "math": {
+        "log_path": "granite-4.0-h-350m-unsloth-mxfp4_moe-attn_kv_BF16-attn_output_BF16-attn_q_BF16-embeddings_BF16-ffn_down_MXFP4-ffn_up_gate_BF16/perplexity_math.log",
+        "ppl": 21.3446,
+        "ppl_error": 0.5654
+      }
+    }
+  },
+  "summary": {
+    "avg_prec_loss_pct": 63.2737,
+    "bench_tps": 1815.38,
+    "file_size_bytes": 511980480,
+    "file_size_gb": 0.48
+  }
+}

Benchmarks/DataCollection/granite-4.0-h-350m-unsloth-mxfp4_moe-attn_kv_BF16-attn_output_BF16-attn_q_BF16-embeddings_BF16-ffn_down_MXFP4-ffn_up_gate_BF16/llamabench.md

@@ -0,0 +1,11 @@
+ggml_cuda_init: GGML_CUDA_FORCE_MMQ:    no
+ggml_cuda_init: GGML_CUDA_FORCE_CUBLAS: no
+ggml_cuda_init: found 2 CUDA devices:
+  Device 0: NVIDIA GeForce RTX 3090, compute capability 8.6, VMM: yes
+  Device 1: NVIDIA GeForce RTX 3090, compute capability 8.6, VMM: yes
+| model                          |       size |     params | backend    | ngl |            test |                  t/s |
+| ------------------------------ | ---------: | ---------: | ---------- | --: | --------------: | -------------------: |
+| granitehybrid 350M MXFP4 MoE   | 484.86 MiB |   340.33 M | CUDA       |  35 |             pp8 |      1815.38 ± 57.44 |
+| granitehybrid 350M MXFP4 MoE   | 484.86 MiB |   340.33 M | CUDA       |  35 |           tg128 |        312.31 ± 7.89 |
+
+build: 92bb442ad (7040)

Benchmarks/DataCollection/granite-4.0-h-350m-unsloth-mxfp4_moe-attn_kv_BF16-attn_output_BF16-attn_q_BF16-embeddings_BF16-ffn_down_MXFP4-ffn_up_gate_BF16/perplexity_code.log

@@ -0,0 +1,190 @@
+ggml_cuda_init: GGML_CUDA_FORCE_MMQ:    no
+ggml_cuda_init: GGML_CUDA_FORCE_CUBLAS: no
+ggml_cuda_init: found 2 CUDA devices:
+  Device 0: NVIDIA GeForce RTX 3090, compute capability 8.6, VMM: yes
+  Device 1: NVIDIA GeForce RTX 3090, compute capability 8.6, VMM: yes
+build: 7040 (92bb442ad) with cc (Ubuntu 13.3.0-6ubuntu2~24.04) 13.3.0 for x86_64-linux-gnu
+llama_model_load_from_file_impl: using device CUDA0 (NVIDIA GeForce RTX 3090) (0000:01:00.0) - 20963 MiB free
+llama_model_load_from_file_impl: using device CUDA1 (NVIDIA GeForce RTX 3090) (0000:03:00.0) - 23581 MiB free
+llama_model_loader: loaded meta data with 48 key-value pairs and 402 tensors from /mnt/world8/AI/ToBench/granite-4.0-h-350m-unsloth/Magic_Quant/GGUF/dc_round-1_granite-4.0-h-350m-unsloth-mxfp4_moe-attn_kv_BF16-attn_output_BF16-attn_q_BF16-embeddings_BF16-ffn_down_MXFP4-ffn_up_gate_BF16.gguf (version GGUF V3 (latest))
+llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output.
+llama_model_loader: - kv   0:                       general.architecture str              = granitehybrid
+llama_model_loader: - kv   1:                               general.type str              = model
+llama_model_loader: - kv   2:                               general.name str              = Granite 4.0 H 350m Unsloth
+llama_model_loader: - kv   3:                           general.finetune str              = unsloth
+llama_model_loader: - kv   4:                           general.basename str              = granite-4.0-h
+llama_model_loader: - kv   5:                         general.size_label str              = 350M
+llama_model_loader: - kv   6:                            general.license str              = apache-2.0
+llama_model_loader: - kv   7:                   general.base_model.count u32              = 1
+llama_model_loader: - kv   8:                  general.base_model.0.name str              = Granite 4.0 H 350m
+llama_model_loader: - kv   9:          general.base_model.0.organization str              = Ibm Granite
+llama_model_loader: - kv  10:              general.base_model.0.repo_url str              = https://huggingface.co/ibm-granite/gr...
+llama_model_loader: - kv  11:                               general.tags arr[str,3]       = ["language", "unsloth", "granite-4.0"]
+llama_model_loader: - kv  12:                  granitehybrid.block_count u32              = 32
+llama_model_loader: - kv  13:               granitehybrid.context_length u32              = 1048576
+llama_model_loader: - kv  14:             granitehybrid.embedding_length u32              = 768
+llama_model_loader: - kv  15:          granitehybrid.feed_forward_length u32              = 2048
+llama_model_loader: - kv  16:         granitehybrid.attention.head_count u32              = 12
+llama_model_loader: - kv  17:      granitehybrid.attention.head_count_kv arr[i32,32]      = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, ...
+llama_model_loader: - kv  18:               granitehybrid.rope.freq_base f32              = 10000.000000
+llama_model_loader: - kv  19: granitehybrid.attention.layer_norm_rms_epsilon f32              = 0.000010
+llama_model_loader: - kv  20:                 granitehybrid.expert_count u32              = 0
+llama_model_loader: - kv  21:            granitehybrid.expert_used_count u32              = 0
+llama_model_loader: - kv  22:                   granitehybrid.vocab_size u32              = 100352
+llama_model_loader: - kv  23:         granitehybrid.rope.dimension_count u32              = 64
+llama_model_loader: - kv  24:              granitehybrid.attention.scale f32              = 0.015625
+llama_model_loader: - kv  25:              granitehybrid.embedding_scale f32              = 12.000000
+llama_model_loader: - kv  26:               granitehybrid.residual_scale f32              = 0.246000
+llama_model_loader: - kv  27:                  granitehybrid.logit_scale f32              = 3.000000
+llama_model_loader: - kv  28: granitehybrid.expert_shared_feed_forward_length u32              = 2048
+llama_model_loader: - kv  29:              granitehybrid.ssm.conv_kernel u32              = 4
+llama_model_loader: - kv  30:               granitehybrid.ssm.state_size u32              = 128
+llama_model_loader: - kv  31:              granitehybrid.ssm.group_count u32              = 1
+llama_model_loader: - kv  32:               granitehybrid.ssm.inner_size u32              = 1536
+llama_model_loader: - kv  33:           granitehybrid.ssm.time_step_rank u32              = 48
+llama_model_loader: - kv  34:       granitehybrid.rope.scaling.finetuned bool             = false
+llama_model_loader: - kv  35:                       tokenizer.ggml.model str              = gpt2
+llama_model_loader: - kv  36:                         tokenizer.ggml.pre str              = dbrx
+llama_model_loader: - kv  37:                      tokenizer.ggml.tokens arr[str,100352]  = ["!", "\"", "#", "$", "%", "&", "'", ...
+llama_model_loader: - kv  38:                  tokenizer.ggml.token_type arr[i32,100352]  = [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...
+llama_model_loader: - kv  39:                      tokenizer.ggml.merges arr[str,100000]  = ["Ġ Ġ", "ĠĠ ĠĠ", "i n", "Ġ t",...
+llama_model_loader: - kv  40:                tokenizer.ggml.bos_token_id u32              = 100257
+llama_model_loader: - kv  41:                tokenizer.ggml.eos_token_id u32              = 100257
+llama_model_loader: - kv  42:            tokenizer.ggml.unknown_token_id u32              = 100269
+llama_model_loader: - kv  43:            tokenizer.ggml.padding_token_id u32              = 100256
+llama_model_loader: - kv  44:               tokenizer.ggml.add_bos_token bool             = false
+llama_model_loader: - kv  45:                    tokenizer.chat_template str              = {%- set tools_system_message_prefix =...
+llama_model_loader: - kv  46:               general.quantization_version u32              = 2
+llama_model_loader: - kv  47:                          general.file_type u32              = 38
+llama_model_loader: - type  f32:  233 tensors
+llama_model_loader: - type q8_0:   56 tensors
+llama_model_loader: - type bf16:   81 tensors
+llama_model_loader: - type mxfp4:   32 tensors
+print_info: file format = GGUF V3 (latest)
+print_info: file type   = MXFP4 MoE
+print_info: file size   = 484.86 MiB (11.95 BPW) 
+load: printing all EOG tokens:
+load:   - 100257 ('<|end_of_text|>')
+load:   - 100261 ('<|fim_pad|>')
+load: special tokens cache size = 96
+load: token to piece cache size = 0.6152 MB
+print_info: arch             = granitehybrid
+print_info: vocab_only       = 0
+print_info: n_ctx_train      = 1048576
+print_info: n_embd           = 768
+print_info: n_embd_inp       = 768
+print_info: n_layer          = 32
+print_info: n_head           = 12
+print_info: n_head_kv        = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0]
+print_info: n_rot            = 64
+print_info: n_swa            = 0
+print_info: is_swa_any       = 0
+print_info: n_embd_head_k    = 64
+print_info: n_embd_head_v    = 64
+print_info: n_gqa            = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 3, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0]
+print_info: n_embd_k_gqa     = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 256, 0, 0, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 0, 0]
+print_info: n_embd_v_gqa     = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 256, 0, 0, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 0, 0]
+print_info: f_norm_eps       = 0.0e+00
+print_info: f_norm_rms_eps   = 1.0e-05
+print_info: f_clamp_kqv      = 0.0e+00
+print_info: f_max_alibi_bias = 0.0e+00
+print_info: f_logit_scale    = 3.0e+00
+print_info: f_attn_scale     = 1.6e-02
+print_info: n_ff             = 2048
+print_info: n_expert         = 0
+print_info: n_expert_used    = 0
+print_info: n_expert_groups  = 0
+print_info: n_group_used     = 0
+print_info: causal attn      = 1
+print_info: pooling type     = 0
+print_info: rope type        = 0
+print_info: rope scaling     = linear
+print_info: freq_base_train  = 10000.0
+print_info: freq_scale_train = 1
+print_info: n_ctx_orig_yarn  = 1048576
+print_info: rope_finetuned   = unknown
+print_info: ssm_d_conv       = 4
+print_info: ssm_d_inner      = 1536
+print_info: ssm_d_state      = 128
+print_info: ssm_dt_rank      = 48
+print_info: ssm_n_group      = 1
+print_info: ssm_dt_b_c_rms   = 0
+print_info: model type       = 350M
+print_info: model params     = 340.33 M
+print_info: general.name     = Granite 4.0 H 350m Unsloth
+print_info: f_embedding_scale = 12.000000
+print_info: f_residual_scale  = 0.246000
+print_info: f_attention_scale = 0.015625
+print_info: n_ff_shexp        = 2048
+print_info: vocab type       = BPE
+print_info: n_vocab          = 100352
+print_info: n_merges         = 100000
+print_info: BOS token        = 100257 '<|end_of_text|>'
+print_info: EOS token        = 100257 '<|end_of_text|>'
+print_info: EOT token        = 100257 '<|end_of_text|>'
+print_info: UNK token        = 100269 '<|unk|>'
+print_info: PAD token        = 100256 '<|pad|>'
+print_info: LF token         = 198 'Ċ'
+print_info: FIM PRE token    = 100258 '<|fim_prefix|>'
+print_info: FIM SUF token    = 100260 '<|fim_suffix|>'
+print_info: FIM MID token    = 100259 '<|fim_middle|>'
+print_info: FIM PAD token    = 100261 '<|fim_pad|>'
+print_info: EOG token        = 100257 '<|end_of_text|>'
+print_info: EOG token        = 100261 '<|fim_pad|>'
+print_info: max token length = 256
+load_tensors: loading model tensors, this can take a while... (mmap = true)
+load_tensors: offloading 20 repeating layers to GPU
+load_tensors: offloaded 20/33 layers to GPU
+load_tensors:   CPU_Mapped model buffer size =   274.13 MiB
+load_tensors:        CUDA0 model buffer size =   104.94 MiB
+load_tensors:        CUDA1 model buffer size =   105.80 MiB
+.......................................................................
+llama_context: constructing llama_context
+llama_context: n_seq_max     = 1
+llama_context: n_ctx         = 2048
+llama_context: n_ctx_seq     = 2048
+llama_context: n_batch       = 2048
+llama_context: n_ubatch      = 512
+llama_context: causal_attn   = 1
+llama_context: flash_attn    = auto
+llama_context: kv_unified    = false
+llama_context: freq_base     = 10000.0
+llama_context: freq_scale    = 1
+llama_context: n_ctx_seq (2048) < n_ctx_train (1048576) -- the full capacity of the model will not be utilized
+llama_context:        CPU  output buffer size =     0.38 MiB
+llama_kv_cache:        CPU KV buffer size =     2.00 MiB
+llama_kv_cache:      CUDA0 KV buffer size =     4.00 MiB
+llama_kv_cache:      CUDA1 KV buffer size =     2.00 MiB
+llama_kv_cache: size =    8.00 MiB (  2048 cells,   4 layers,  1/1 seqs), K (f16):    4.00 MiB, V (f16):    4.00 MiB
+llama_memory_recurrent:        CPU RS buffer size =     8.48 MiB
+llama_memory_recurrent:      CUDA0 RS buffer size =     6.16 MiB
+llama_memory_recurrent:      CUDA1 RS buffer size =     6.93 MiB
+llama_memory_recurrent: size =   21.57 MiB (     1 cells,  32 layers,  1 seqs), R (f32):    0.57 MiB, S (f32):   21.00 MiB
+llama_context: Flash Attention was auto, set to enabled
+llama_context:      CUDA0 compute buffer size =   360.52 MiB
+llama_context:      CUDA1 compute buffer size =    22.39 MiB
+llama_context:  CUDA_Host compute buffer size =    18.34 MiB
+llama_context: graph nodes  = 1815
+llama_context: graph splits = 182 (with bs=512), 41 (with bs=1)
+common_init_from_params: added <|end_of_text|> logit bias = -inf
+common_init_from_params: added <|fim_pad|> logit bias = -inf
+common_init_from_params: setting dry_penalty_last_n to ctx_size = 2048
+common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable)
+
+system_info: n_threads = 16 (n_threads_batch = 16) / 32 | CUDA : ARCHS = 860 | USE_GRAPHS = 1 | PEER_MAX_BATCH_SIZE = 128 | CPU : SSE3 = 1 | SSSE3 = 1 | AVX = 1 | AVX2 = 1 | F16C = 1 | FMA = 1 | BMI2 = 1 | AVX512 = 1 | AVX512_VBMI = 1 | AVX512_VNNI = 1 | AVX512_BF16 = 1 | LLAMAFILE = 1 | OPENMP = 1 | REPACK = 1 | 
+perplexity: tokenizing the input ..
+perplexity: tokenization took 92.667 ms
+perplexity: calculating perplexity over 44 chunks, n_ctx=2048, batch_size=2048, n_seq=1
+perplexity: 0.60 seconds per pass - ETA 0.43 minutes
+[1]7.0228,[2]6.5608,[3]3.7026,[4]3.3045,[5]3.9243,[6]4.4408,[7]4.0967,[8]3.6949,[9]3.2801,[10]2.9558,[11]2.9009,[12]2.9191,[13]2.7597,[14]2.7182,[15]2.7982,[16]2.6794,[17]2.6100,[18]2.6272,[19]2.5606,[20]2.5013,[21]2.4530,[22]2.4222,[23]2.4894,[24]2.4613,[25]2.4833,[26]2.4360,[27]2.4166,[28]2.4053,[29]2.4833,[30]2.5182,[31]2.5118,[32]2.4784,[33]2.5059,[34]2.4944,[35]2.4625,[36]2.5075,[37]2.5175,[38]2.5107,[39]2.5510,[40]2.5635,[41]2.5532,[42]2.5891,[43]2.5920,[44]2.5714,
+Final estimate: PPL = 2.5714 +/- 0.02823
+
+llama_perf_context_print:        load time =     245.66 ms
+llama_perf_context_print: prompt eval time =   15787.92 ms / 90112 tokens (    0.18 ms per token,  5707.66 tokens per second)
+llama_perf_context_print:        eval time =       0.00 ms /     1 runs   (    0.00 ms per token,      inf tokens per second)
+llama_perf_context_print:       total time =   16586.22 ms / 90113 tokens
+llama_perf_context_print:    graphs reused =          0
+llama_memory_breakdown_print: | memory breakdown [MiB] | total    free    self   model   context   compute    unaccounted |
+llama_memory_breakdown_print: |   - CUDA0 (RTX 3090)   | 24115 = 20340 + ( 475 =   104 +      10 +     360) +        3299 |
+llama_memory_breakdown_print: |   - CUDA1 (RTX 3090)   | 24124 = 23361 + ( 137 =   105 +       8 +      22) +         625 |
+llama_memory_breakdown_print: |   - Host               |                   302 =   274 +      10 +      18                |

Benchmarks/DataCollection/granite-4.0-h-350m-unsloth-mxfp4_moe-attn_kv_BF16-attn_output_BF16-attn_q_BF16-embeddings_BF16-ffn_down_MXFP4-ffn_up_gate_BF16/perplexity_general.log

@@ -0,0 +1,190 @@
+ggml_cuda_init: GGML_CUDA_FORCE_MMQ:    no
+ggml_cuda_init: GGML_CUDA_FORCE_CUBLAS: no
+ggml_cuda_init: found 2 CUDA devices:
+  Device 0: NVIDIA GeForce RTX 3090, compute capability 8.6, VMM: yes
+  Device 1: NVIDIA GeForce RTX 3090, compute capability 8.6, VMM: yes
+build: 7040 (92bb442ad) with cc (Ubuntu 13.3.0-6ubuntu2~24.04) 13.3.0 for x86_64-linux-gnu
+llama_model_load_from_file_impl: using device CUDA0 (NVIDIA GeForce RTX 3090) (0000:01:00.0) - 20965 MiB free
+llama_model_load_from_file_impl: using device CUDA1 (NVIDIA GeForce RTX 3090) (0000:03:00.0) - 23581 MiB free
+llama_model_loader: loaded meta data with 48 key-value pairs and 402 tensors from /mnt/world8/AI/ToBench/granite-4.0-h-350m-unsloth/Magic_Quant/GGUF/dc_round-1_granite-4.0-h-350m-unsloth-mxfp4_moe-attn_kv_BF16-attn_output_BF16-attn_q_BF16-embeddings_BF16-ffn_down_MXFP4-ffn_up_gate_BF16.gguf (version GGUF V3 (latest))
+llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output.
+llama_model_loader: - kv   0:                       general.architecture str              = granitehybrid
+llama_model_loader: - kv   1:                               general.type str              = model
+llama_model_loader: - kv   2:                               general.name str              = Granite 4.0 H 350m Unsloth
+llama_model_loader: - kv   3:                           general.finetune str              = unsloth
+llama_model_loader: - kv   4:                           general.basename str              = granite-4.0-h
+llama_model_loader: - kv   5:                         general.size_label str              = 350M
+llama_model_loader: - kv   6:                            general.license str              = apache-2.0
+llama_model_loader: - kv   7:                   general.base_model.count u32              = 1
+llama_model_loader: - kv   8:                  general.base_model.0.name str              = Granite 4.0 H 350m
+llama_model_loader: - kv   9:          general.base_model.0.organization str              = Ibm Granite
+llama_model_loader: - kv  10:              general.base_model.0.repo_url str              = https://huggingface.co/ibm-granite/gr...
+llama_model_loader: - kv  11:                               general.tags arr[str,3]       = ["language", "unsloth", "granite-4.0"]
+llama_model_loader: - kv  12:                  granitehybrid.block_count u32              = 32
+llama_model_loader: - kv  13:               granitehybrid.context_length u32              = 1048576
+llama_model_loader: - kv  14:             granitehybrid.embedding_length u32              = 768
+llama_model_loader: - kv  15:          granitehybrid.feed_forward_length u32              = 2048
+llama_model_loader: - kv  16:         granitehybrid.attention.head_count u32              = 12
+llama_model_loader: - kv  17:      granitehybrid.attention.head_count_kv arr[i32,32]      = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, ...
+llama_model_loader: - kv  18:               granitehybrid.rope.freq_base f32              = 10000.000000
+llama_model_loader: - kv  19: granitehybrid.attention.layer_norm_rms_epsilon f32              = 0.000010
+llama_model_loader: - kv  20:                 granitehybrid.expert_count u32              = 0
+llama_model_loader: - kv  21:            granitehybrid.expert_used_count u32              = 0
+llama_model_loader: - kv  22:                   granitehybrid.vocab_size u32              = 100352
+llama_model_loader: - kv  23:         granitehybrid.rope.dimension_count u32              = 64
+llama_model_loader: - kv  24:              granitehybrid.attention.scale f32              = 0.015625
+llama_model_loader: - kv  25:              granitehybrid.embedding_scale f32              = 12.000000
+llama_model_loader: - kv  26:               granitehybrid.residual_scale f32              = 0.246000
+llama_model_loader: - kv  27:                  granitehybrid.logit_scale f32              = 3.000000
+llama_model_loader: - kv  28: granitehybrid.expert_shared_feed_forward_length u32              = 2048
+llama_model_loader: - kv  29:              granitehybrid.ssm.conv_kernel u32              = 4
+llama_model_loader: - kv  30:               granitehybrid.ssm.state_size u32              = 128
+llama_model_loader: - kv  31:              granitehybrid.ssm.group_count u32              = 1
+llama_model_loader: - kv  32:               granitehybrid.ssm.inner_size u32              = 1536
+llama_model_loader: - kv  33:           granitehybrid.ssm.time_step_rank u32              = 48
+llama_model_loader: - kv  34:       granitehybrid.rope.scaling.finetuned bool             = false
+llama_model_loader: - kv  35:                       tokenizer.ggml.model str              = gpt2
+llama_model_loader: - kv  36:                         tokenizer.ggml.pre str              = dbrx
+llama_model_loader: - kv  37:                      tokenizer.ggml.tokens arr[str,100352]  = ["!", "\"", "#", "$", "%", "&", "'", ...
+llama_model_loader: - kv  38:                  tokenizer.ggml.token_type arr[i32,100352]  = [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...
+llama_model_loader: - kv  39:                      tokenizer.ggml.merges arr[str,100000]  = ["Ġ Ġ", "ĠĠ ĠĠ", "i n", "Ġ t",...
+llama_model_loader: - kv  40:                tokenizer.ggml.bos_token_id u32              = 100257
+llama_model_loader: - kv  41:                tokenizer.ggml.eos_token_id u32              = 100257
+llama_model_loader: - kv  42:            tokenizer.ggml.unknown_token_id u32              = 100269
+llama_model_loader: - kv  43:            tokenizer.ggml.padding_token_id u32              = 100256
+llama_model_loader: - kv  44:               tokenizer.ggml.add_bos_token bool             = false
+llama_model_loader: - kv  45:                    tokenizer.chat_template str              = {%- set tools_system_message_prefix =...
+llama_model_loader: - kv  46:               general.quantization_version u32              = 2
+llama_model_loader: - kv  47:                          general.file_type u32              = 38
+llama_model_loader: - type  f32:  233 tensors
+llama_model_loader: - type q8_0:   56 tensors
+llama_model_loader: - type bf16:   81 tensors
+llama_model_loader: - type mxfp4:   32 tensors
+print_info: file format = GGUF V3 (latest)
+print_info: file type   = MXFP4 MoE
+print_info: file size   = 484.86 MiB (11.95 BPW) 
+load: printing all EOG tokens:
+load:   - 100257 ('<|end_of_text|>')
+load:   - 100261 ('<|fim_pad|>')
+load: special tokens cache size = 96
+load: token to piece cache size = 0.6152 MB
+print_info: arch             = granitehybrid
+print_info: vocab_only       = 0
+print_info: n_ctx_train      = 1048576
+print_info: n_embd           = 768
+print_info: n_embd_inp       = 768
+print_info: n_layer          = 32
+print_info: n_head           = 12
+print_info: n_head_kv        = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0]
+print_info: n_rot            = 64
+print_info: n_swa            = 0
+print_info: is_swa_any       = 0
+print_info: n_embd_head_k    = 64
+print_info: n_embd_head_v    = 64
+print_info: n_gqa            = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 3, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0]
+print_info: n_embd_k_gqa     = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 256, 0, 0, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 0, 0]
+print_info: n_embd_v_gqa     = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 256, 0, 0, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 0, 0]
+print_info: f_norm_eps       = 0.0e+00
+print_info: f_norm_rms_eps   = 1.0e-05
+print_info: f_clamp_kqv      = 0.0e+00
+print_info: f_max_alibi_bias = 0.0e+00
+print_info: f_logit_scale    = 3.0e+00
+print_info: f_attn_scale     = 1.6e-02
+print_info: n_ff             = 2048
+print_info: n_expert         = 0
+print_info: n_expert_used    = 0
+print_info: n_expert_groups  = 0
+print_info: n_group_used     = 0
+print_info: causal attn      = 1
+print_info: pooling type     = 0
+print_info: rope type        = 0
+print_info: rope scaling     = linear
+print_info: freq_base_train  = 10000.0
+print_info: freq_scale_train = 1
+print_info: n_ctx_orig_yarn  = 1048576
+print_info: rope_finetuned   = unknown
+print_info: ssm_d_conv       = 4
+print_info: ssm_d_inner      = 1536
+print_info: ssm_d_state      = 128
+print_info: ssm_dt_rank      = 48
+print_info: ssm_n_group      = 1
+print_info: ssm_dt_b_c_rms   = 0
+print_info: model type       = 350M
+print_info: model params     = 340.33 M
+print_info: general.name     = Granite 4.0 H 350m Unsloth
+print_info: f_embedding_scale = 12.000000
+print_info: f_residual_scale  = 0.246000
+print_info: f_attention_scale = 0.015625
+print_info: n_ff_shexp        = 2048
+print_info: vocab type       = BPE
+print_info: n_vocab          = 100352
+print_info: n_merges         = 100000
+print_info: BOS token        = 100257 '<|end_of_text|>'
+print_info: EOS token        = 100257 '<|end_of_text|>'
+print_info: EOT token        = 100257 '<|end_of_text|>'
+print_info: UNK token        = 100269 '<|unk|>'
+print_info: PAD token        = 100256 '<|pad|>'
+print_info: LF token         = 198 'Ċ'
+print_info: FIM PRE token    = 100258 '<|fim_prefix|>'
+print_info: FIM SUF token    = 100260 '<|fim_suffix|>'
+print_info: FIM MID token    = 100259 '<|fim_middle|>'
+print_info: FIM PAD token    = 100261 '<|fim_pad|>'
+print_info: EOG token        = 100257 '<|end_of_text|>'
+print_info: EOG token        = 100261 '<|fim_pad|>'
+print_info: max token length = 256
+load_tensors: loading model tensors, this can take a while... (mmap = true)
+load_tensors: offloading 20 repeating layers to GPU
+load_tensors: offloaded 20/33 layers to GPU
+load_tensors:   CPU_Mapped model buffer size =   274.13 MiB
+load_tensors:        CUDA0 model buffer size =   104.94 MiB
+load_tensors:        CUDA1 model buffer size =   105.80 MiB
+.......................................................................
+llama_context: constructing llama_context
+llama_context: n_seq_max     = 1
+llama_context: n_ctx         = 2048
+llama_context: n_ctx_seq     = 2048
+llama_context: n_batch       = 2048
+llama_context: n_ubatch      = 512
+llama_context: causal_attn   = 1
+llama_context: flash_attn    = auto
+llama_context: kv_unified    = false
+llama_context: freq_base     = 10000.0
+llama_context: freq_scale    = 1
+llama_context: n_ctx_seq (2048) < n_ctx_train (1048576) -- the full capacity of the model will not be utilized
+llama_context:        CPU  output buffer size =     0.38 MiB
+llama_kv_cache:        CPU KV buffer size =     2.00 MiB
+llama_kv_cache:      CUDA0 KV buffer size =     4.00 MiB
+llama_kv_cache:      CUDA1 KV buffer size =     2.00 MiB
+llama_kv_cache: size =    8.00 MiB (  2048 cells,   4 layers,  1/1 seqs), K (f16):    4.00 MiB, V (f16):    4.00 MiB
+llama_memory_recurrent:        CPU RS buffer size =     8.48 MiB
+llama_memory_recurrent:      CUDA0 RS buffer size =     6.16 MiB
+llama_memory_recurrent:      CUDA1 RS buffer size =     6.93 MiB
+llama_memory_recurrent: size =   21.57 MiB (     1 cells,  32 layers,  1 seqs), R (f32):    0.57 MiB, S (f32):   21.00 MiB
+llama_context: Flash Attention was auto, set to enabled
+llama_context:      CUDA0 compute buffer size =   360.52 MiB
+llama_context:      CUDA1 compute buffer size =    22.39 MiB
+llama_context:  CUDA_Host compute buffer size =    18.34 MiB
+llama_context: graph nodes  = 1815
+llama_context: graph splits = 182 (with bs=512), 41 (with bs=1)
+common_init_from_params: added <|end_of_text|> logit bias = -inf
+common_init_from_params: added <|fim_pad|> logit bias = -inf
+common_init_from_params: setting dry_penalty_last_n to ctx_size = 2048
+common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable)
+
+system_info: n_threads = 16 (n_threads_batch = 16) / 32 | CUDA : ARCHS = 860 | USE_GRAPHS = 1 | PEER_MAX_BATCH_SIZE = 128 | CPU : SSE3 = 1 | SSSE3 = 1 | AVX = 1 | AVX2 = 1 | F16C = 1 | FMA = 1 | BMI2 = 1 | AVX512 = 1 | AVX512_VBMI = 1 | AVX512_VNNI = 1 | AVX512_BF16 = 1 | LLAMAFILE = 1 | OPENMP = 1 | REPACK = 1 | 
+perplexity: tokenizing the input ..
+perplexity: tokenization took 39.891 ms
+perplexity: calculating perplexity over 14 chunks, n_ctx=2048, batch_size=2048, n_seq=1
+perplexity: 0.60 seconds per pass - ETA 0.13 minutes
+[1]24.0889,[2]32.9260,[3]34.1823,[4]30.4142,[5]30.7771,[6]27.6059,[7]27.5764,[8]27.5099,[9]28.8098,[10]28.7305,[11]28.2909,[12]27.8772,[13]27.6976,[14]27.3444,
+Final estimate: PPL = 27.3444 +/- 0.76026
+
+llama_perf_context_print:        load time =     232.91 ms
+llama_perf_context_print: prompt eval time =    5319.17 ms / 28672 tokens (    0.19 ms per token,  5390.31 tokens per second)
+llama_perf_context_print:        eval time =       0.00 ms /     1 runs   (    0.00 ms per token,      inf tokens per second)
+llama_perf_context_print:       total time =    5588.48 ms / 28673 tokens
+llama_perf_context_print:    graphs reused =          0
+llama_memory_breakdown_print: | memory breakdown [MiB] | total    free    self   model   context   compute    unaccounted |
+llama_memory_breakdown_print: |   - CUDA0 (RTX 3090)   | 24115 = 20337 + ( 475 =   104 +      10 +     360) +        3302 |
+llama_memory_breakdown_print: |   - CUDA1 (RTX 3090)   | 24124 = 23361 + ( 137 =   105 +       8 +      22) +         625 |
+llama_memory_breakdown_print: |   - Host               |                   302 =   274 +      10 +      18                |

Benchmarks/DataCollection/granite-4.0-h-350m-unsloth-mxfp4_moe-attn_kv_BF16-attn_output_BF16-attn_q_BF16-embeddings_BF16-ffn_down_MXFP4-ffn_up_gate_BF16/perplexity_math.log

@@ -0,0 +1,190 @@
+ggml_cuda_init: GGML_CUDA_FORCE_MMQ:    no
+ggml_cuda_init: GGML_CUDA_FORCE_CUBLAS: no
+ggml_cuda_init: found 2 CUDA devices:
+  Device 0: NVIDIA GeForce RTX 3090, compute capability 8.6, VMM: yes
+  Device 1: NVIDIA GeForce RTX 3090, compute capability 8.6, VMM: yes
+build: 7040 (92bb442ad) with cc (Ubuntu 13.3.0-6ubuntu2~24.04) 13.3.0 for x86_64-linux-gnu
+llama_model_load_from_file_impl: using device CUDA0 (NVIDIA GeForce RTX 3090) (0000:01:00.0) - 20966 MiB free
+llama_model_load_from_file_impl: using device CUDA1 (NVIDIA GeForce RTX 3090) (0000:03:00.0) - 23581 MiB free
+llama_model_loader: loaded meta data with 48 key-value pairs and 402 tensors from /mnt/world8/AI/ToBench/granite-4.0-h-350m-unsloth/Magic_Quant/GGUF/dc_round-1_granite-4.0-h-350m-unsloth-mxfp4_moe-attn_kv_BF16-attn_output_BF16-attn_q_BF16-embeddings_BF16-ffn_down_MXFP4-ffn_up_gate_BF16.gguf (version GGUF V3 (latest))
+llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output.
+llama_model_loader: - kv   0:                       general.architecture str              = granitehybrid
+llama_model_loader: - kv   1:                               general.type str              = model
+llama_model_loader: - kv   2:                               general.name str              = Granite 4.0 H 350m Unsloth
+llama_model_loader: - kv   3:                           general.finetune str              = unsloth
+llama_model_loader: - kv   4:                           general.basename str              = granite-4.0-h
+llama_model_loader: - kv   5:                         general.size_label str              = 350M
+llama_model_loader: - kv   6:                            general.license str              = apache-2.0
+llama_model_loader: - kv   7:                   general.base_model.count u32              = 1
+llama_model_loader: - kv   8:                  general.base_model.0.name str              = Granite 4.0 H 350m
+llama_model_loader: - kv   9:          general.base_model.0.organization str              = Ibm Granite
+llama_model_loader: - kv  10:              general.base_model.0.repo_url str              = https://huggingface.co/ibm-granite/gr...
+llama_model_loader: - kv  11:                               general.tags arr[str,3]       = ["language", "unsloth", "granite-4.0"]
+llama_model_loader: - kv  12:                  granitehybrid.block_count u32              = 32
+llama_model_loader: - kv  13:               granitehybrid.context_length u32              = 1048576
+llama_model_loader: - kv  14:             granitehybrid.embedding_length u32              = 768
+llama_model_loader: - kv  15:          granitehybrid.feed_forward_length u32              = 2048
+llama_model_loader: - kv  16:         granitehybrid.attention.head_count u32              = 12
+llama_model_loader: - kv  17:      granitehybrid.attention.head_count_kv arr[i32,32]      = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, ...
+llama_model_loader: - kv  18:               granitehybrid.rope.freq_base f32              = 10000.000000
+llama_model_loader: - kv  19: granitehybrid.attention.layer_norm_rms_epsilon f32              = 0.000010
+llama_model_loader: - kv  20:                 granitehybrid.expert_count u32              = 0
+llama_model_loader: - kv  21:            granitehybrid.expert_used_count u32              = 0
+llama_model_loader: - kv  22:                   granitehybrid.vocab_size u32              = 100352
+llama_model_loader: - kv  23:         granitehybrid.rope.dimension_count u32              = 64
+llama_model_loader: - kv  24:              granitehybrid.attention.scale f32              = 0.015625
+llama_model_loader: - kv  25:              granitehybrid.embedding_scale f32              = 12.000000
+llama_model_loader: - kv  26:               granitehybrid.residual_scale f32              = 0.246000
+llama_model_loader: - kv  27:                  granitehybrid.logit_scale f32              = 3.000000
+llama_model_loader: - kv  28: granitehybrid.expert_shared_feed_forward_length u32              = 2048
+llama_model_loader: - kv  29:              granitehybrid.ssm.conv_kernel u32              = 4
+llama_model_loader: - kv  30:               granitehybrid.ssm.state_size u32              = 128
+llama_model_loader: - kv  31:              granitehybrid.ssm.group_count u32              = 1
+llama_model_loader: - kv  32:               granitehybrid.ssm.inner_size u32              = 1536
+llama_model_loader: - kv  33:           granitehybrid.ssm.time_step_rank u32              = 48
+llama_model_loader: - kv  34:       granitehybrid.rope.scaling.finetuned bool             = false
+llama_model_loader: - kv  35:                       tokenizer.ggml.model str              = gpt2
+llama_model_loader: - kv  36:                         tokenizer.ggml.pre str              = dbrx
+llama_model_loader: - kv  37:                      tokenizer.ggml.tokens arr[str,100352]  = ["!", "\"", "#", "$", "%", "&", "'", ...
+llama_model_loader: - kv  38:                  tokenizer.ggml.token_type arr[i32,100352]  = [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...
+llama_model_loader: - kv  39:                      tokenizer.ggml.merges arr[str,100000]  = ["Ġ Ġ", "ĠĠ ĠĠ", "i n", "Ġ t",...
+llama_model_loader: - kv  40:                tokenizer.ggml.bos_token_id u32              = 100257
+llama_model_loader: - kv  41:                tokenizer.ggml.eos_token_id u32              = 100257
+llama_model_loader: - kv  42:            tokenizer.ggml.unknown_token_id u32              = 100269
+llama_model_loader: - kv  43:            tokenizer.ggml.padding_token_id u32              = 100256
+llama_model_loader: - kv  44:               tokenizer.ggml.add_bos_token bool             = false
+llama_model_loader: - kv  45:                    tokenizer.chat_template str              = {%- set tools_system_message_prefix =...
+llama_model_loader: - kv  46:               general.quantization_version u32              = 2
+llama_model_loader: - kv  47:                          general.file_type u32              = 38
+llama_model_loader: - type  f32:  233 tensors
+llama_model_loader: - type q8_0:   56 tensors
+llama_model_loader: - type bf16:   81 tensors
+llama_model_loader: - type mxfp4:   32 tensors
+print_info: file format = GGUF V3 (latest)
+print_info: file type   = MXFP4 MoE
+print_info: file size   = 484.86 MiB (11.95 BPW) 
+load: printing all EOG tokens:
+load:   - 100257 ('<|end_of_text|>')
+load:   - 100261 ('<|fim_pad|>')
+load: special tokens cache size = 96
+load: token to piece cache size = 0.6152 MB
+print_info: arch             = granitehybrid
+print_info: vocab_only       = 0
+print_info: n_ctx_train      = 1048576
+print_info: n_embd           = 768
+print_info: n_embd_inp       = 768
+print_info: n_layer          = 32
+print_info: n_head           = 12
+print_info: n_head_kv        = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0]
+print_info: n_rot            = 64
+print_info: n_swa            = 0
+print_info: is_swa_any       = 0
+print_info: n_embd_head_k    = 64
+print_info: n_embd_head_v    = 64
+print_info: n_gqa            = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 3, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0]
+print_info: n_embd_k_gqa     = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 256, 0, 0, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 0, 0]
+print_info: n_embd_v_gqa     = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 256, 0, 0, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 0, 0]
+print_info: f_norm_eps       = 0.0e+00
+print_info: f_norm_rms_eps   = 1.0e-05
+print_info: f_clamp_kqv      = 0.0e+00
+print_info: f_max_alibi_bias = 0.0e+00
+print_info: f_logit_scale    = 3.0e+00
+print_info: f_attn_scale     = 1.6e-02
+print_info: n_ff             = 2048
+print_info: n_expert         = 0
+print_info: n_expert_used    = 0
+print_info: n_expert_groups  = 0
+print_info: n_group_used     = 0
+print_info: causal attn      = 1
+print_info: pooling type     = 0
+print_info: rope type        = 0
+print_info: rope scaling     = linear
+print_info: freq_base_train  = 10000.0
+print_info: freq_scale_train = 1
+print_info: n_ctx_orig_yarn  = 1048576
+print_info: rope_finetuned   = unknown
+print_info: ssm_d_conv       = 4
+print_info: ssm_d_inner      = 1536
+print_info: ssm_d_state      = 128
+print_info: ssm_dt_rank      = 48
+print_info: ssm_n_group      = 1
+print_info: ssm_dt_b_c_rms   = 0
+print_info: model type       = 350M
+print_info: model params     = 340.33 M
+print_info: general.name     = Granite 4.0 H 350m Unsloth
+print_info: f_embedding_scale = 12.000000
+print_info: f_residual_scale  = 0.246000
+print_info: f_attention_scale = 0.015625
+print_info: n_ff_shexp        = 2048
+print_info: vocab type       = BPE
+print_info: n_vocab          = 100352
+print_info: n_merges         = 100000
+print_info: BOS token        = 100257 '<|end_of_text|>'
+print_info: EOS token        = 100257 '<|end_of_text|>'
+print_info: EOT token        = 100257 '<|end_of_text|>'
+print_info: UNK token        = 100269 '<|unk|>'
+print_info: PAD token        = 100256 '<|pad|>'
+print_info: LF token         = 198 'Ċ'
+print_info: FIM PRE token    = 100258 '<|fim_prefix|>'
+print_info: FIM SUF token    = 100260 '<|fim_suffix|>'
+print_info: FIM MID token    = 100259 '<|fim_middle|>'
+print_info: FIM PAD token    = 100261 '<|fim_pad|>'
+print_info: EOG token        = 100257 '<|end_of_text|>'
+print_info: EOG token        = 100261 '<|fim_pad|>'
+print_info: max token length = 256
+load_tensors: loading model tensors, this can take a while... (mmap = true)
+load_tensors: offloading 20 repeating layers to GPU
+load_tensors: offloaded 20/33 layers to GPU
+load_tensors:   CPU_Mapped model buffer size =   274.13 MiB
+load_tensors:        CUDA0 model buffer size =   104.94 MiB
+load_tensors:        CUDA1 model buffer size =   105.80 MiB
+.......................................................................
+llama_context: constructing llama_context
+llama_context: n_seq_max     = 1
+llama_context: n_ctx         = 2048
+llama_context: n_ctx_seq     = 2048
+llama_context: n_batch       = 2048
+llama_context: n_ubatch      = 512
+llama_context: causal_attn   = 1
+llama_context: flash_attn    = auto
+llama_context: kv_unified    = false
+llama_context: freq_base     = 10000.0
+llama_context: freq_scale    = 1
+llama_context: n_ctx_seq (2048) < n_ctx_train (1048576) -- the full capacity of the model will not be utilized
+llama_context:        CPU  output buffer size =     0.38 MiB
+llama_kv_cache:        CPU KV buffer size =     2.00 MiB
+llama_kv_cache:      CUDA0 KV buffer size =     4.00 MiB
+llama_kv_cache:      CUDA1 KV buffer size =     2.00 MiB
+llama_kv_cache: size =    8.00 MiB (  2048 cells,   4 layers,  1/1 seqs), K (f16):    4.00 MiB, V (f16):    4.00 MiB
+llama_memory_recurrent:        CPU RS buffer size =     8.48 MiB
+llama_memory_recurrent:      CUDA0 RS buffer size =     6.16 MiB
+llama_memory_recurrent:      CUDA1 RS buffer size =     6.93 MiB
+llama_memory_recurrent: size =   21.57 MiB (     1 cells,  32 layers,  1 seqs), R (f32):    0.57 MiB, S (f32):   21.00 MiB
+llama_context: Flash Attention was auto, set to enabled
+llama_context:      CUDA0 compute buffer size =   360.52 MiB
+llama_context:      CUDA1 compute buffer size =    22.39 MiB
+llama_context:  CUDA_Host compute buffer size =    18.34 MiB
+llama_context: graph nodes  = 1815
+llama_context: graph splits = 182 (with bs=512), 41 (with bs=1)
+common_init_from_params: added <|end_of_text|> logit bias = -inf
+common_init_from_params: added <|fim_pad|> logit bias = -inf
+common_init_from_params: setting dry_penalty_last_n to ctx_size = 2048
+common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable)
+
+system_info: n_threads = 16 (n_threads_batch = 16) / 32 | CUDA : ARCHS = 860 | USE_GRAPHS = 1 | PEER_MAX_BATCH_SIZE = 128 | CPU : SSE3 = 1 | SSSE3 = 1 | AVX = 1 | AVX2 = 1 | F16C = 1 | FMA = 1 | BMI2 = 1 | AVX512 = 1 | AVX512_VBMI = 1 | AVX512_VNNI = 1 | AVX512_BF16 = 1 | LLAMAFILE = 1 | OPENMP = 1 | REPACK = 1 | 
+perplexity: tokenizing the input ..
+perplexity: tokenization took 33.356 ms
+perplexity: calculating perplexity over 15 chunks, n_ctx=2048, batch_size=2048, n_seq=1
+perplexity: 0.60 seconds per pass - ETA 0.13 minutes
+[1]18.1961,[2]20.7443,[3]19.8126,[4]20.3533,[5]20.8104,[6]20.5046,[7]20.9239,[8]19.8798,[9]19.9944,[10]20.4483,[11]21.0701,[12]21.3482,[13]21.8303,[14]21.5728,[15]21.3446,
+Final estimate: PPL = 21.3446 +/- 0.56540
+
+llama_perf_context_print:        load time =     234.34 ms
+llama_perf_context_print: prompt eval time =    5525.90 ms / 30720 tokens (    0.18 ms per token,  5559.27 tokens per second)
+llama_perf_context_print:        eval time =       0.00 ms /     1 runs   (    0.00 ms per token,      inf tokens per second)
+llama_perf_context_print:       total time =    5799.36 ms / 30721 tokens
+llama_perf_context_print:    graphs reused =          0
+llama_memory_breakdown_print: | memory breakdown [MiB] | total    free    self   model   context   compute    unaccounted |
+llama_memory_breakdown_print: |   - CUDA0 (RTX 3090)   | 24115 = 20329 + ( 475 =   104 +      10 +     360) +        3309 |
+llama_memory_breakdown_print: |   - CUDA1 (RTX 3090)   | 24124 = 23361 + ( 137 =   105 +       8 +      22) +         625 |
+llama_memory_breakdown_print: |   - Host               |                   302 =   274 +      10 +      18                |

Benchmarks/DataCollection/granite-4.0-h-350m-unsloth-mxfp4_moe-attn_kv_BF16-attn_output_BF16-attn_q_BF16-embeddings_MXFP4-ffn_down_BF16-ffn_up_gate_BF16/bench_metrics.json

@@ -0,0 +1,44 @@
+{
+  "raw_metrics": {
+    "llamabench": {
+      "backend": "CUDA",
+      "log_path": "granite-4.0-h-350m-unsloth-mxfp4_moe-attn_kv_BF16-attn_output_BF16-attn_q_BF16-embeddings_MXFP4-ffn_down_BF16-ffn_up_gate_BF16/llamabench.md",
+      "ngl": "35",
+      "raw_row": {
+        "backend": "CUDA",
+        "model": "granitehybrid 350M MXFP4 MoE",
+        "ngl": "35",
+        "params": "340.33 M",
+        "size": "447.40 MiB",
+        "t/s": "1874.21 \u00b1 56.91",
+        "test": "pp8",
+        "tps_value": 1874.21
+      },
+      "test": "pp8",
+      "tps": 1874.21
+    },
+    "perplexity": {
+      "code": {
+        "log_path": "granite-4.0-h-350m-unsloth-mxfp4_moe-attn_kv_BF16-attn_output_BF16-attn_q_BF16-embeddings_MXFP4-ffn_down_BF16-ffn_up_gate_BF16/perplexity_code.log",
+        "ppl": 2.1057,
+        "ppl_error": 0.01995
+      },
+      "general": {
+        "log_path": "granite-4.0-h-350m-unsloth-mxfp4_moe-attn_kv_BF16-attn_output_BF16-attn_q_BF16-embeddings_MXFP4-ffn_down_BF16-ffn_up_gate_BF16/perplexity_general.log",
+        "ppl": 22.1016,
+        "ppl_error": 0.5804
+      },
+      "math": {
+        "log_path": "granite-4.0-h-350m-unsloth-mxfp4_moe-attn_kv_BF16-attn_output_BF16-attn_q_BF16-embeddings_MXFP4-ffn_down_BF16-ffn_up_gate_BF16/perplexity_math.log",
+        "ppl": 11.4772,
+        "ppl_error": 0.26023
+      }
+    }
+  },
+  "summary": {
+    "avg_prec_loss_pct": 13.7237,
+    "bench_tps": 1874.21,
+    "file_size_bytes": 472708032,
+    "file_size_gb": 0.44
+  }
+}

Benchmarks/DataCollection/granite-4.0-h-350m-unsloth-mxfp4_moe-attn_kv_BF16-attn_output_BF16-attn_q_BF16-embeddings_MXFP4-ffn_down_BF16-ffn_up_gate_BF16/llamabench.md

@@ -0,0 +1,11 @@
+ggml_cuda_init: GGML_CUDA_FORCE_MMQ:    no
+ggml_cuda_init: GGML_CUDA_FORCE_CUBLAS: no
+ggml_cuda_init: found 2 CUDA devices:
+  Device 0: NVIDIA GeForce RTX 3090, compute capability 8.6, VMM: yes
+  Device 1: NVIDIA GeForce RTX 3090, compute capability 8.6, VMM: yes
+| model                          |       size |     params | backend    | ngl |            test |                  t/s |
+| ------------------------------ | ---------: | ---------: | ---------- | --: | --------------: | -------------------: |
+| granitehybrid 350M MXFP4 MoE   | 447.40 MiB |   340.33 M | CUDA       |  35 |             pp8 |      1874.21 ± 56.91 |
+| granitehybrid 350M MXFP4 MoE   | 447.40 MiB |   340.33 M | CUDA       |  35 |           tg128 |       318.32 ± 10.34 |
+
+build: 92bb442ad (7040)

Benchmarks/DataCollection/granite-4.0-h-350m-unsloth-mxfp4_moe-attn_kv_BF16-attn_output_BF16-attn_q_BF16-embeddings_MXFP4-ffn_down_BF16-ffn_up_gate_BF16/perplexity_code.log

@@ -0,0 +1,190 @@
+ggml_cuda_init: GGML_CUDA_FORCE_MMQ:    no
+ggml_cuda_init: GGML_CUDA_FORCE_CUBLAS: no
+ggml_cuda_init: found 2 CUDA devices:
+  Device 0: NVIDIA GeForce RTX 3090, compute capability 8.6, VMM: yes
+  Device 1: NVIDIA GeForce RTX 3090, compute capability 8.6, VMM: yes
+build: 7040 (92bb442ad) with cc (Ubuntu 13.3.0-6ubuntu2~24.04) 13.3.0 for x86_64-linux-gnu
+llama_model_load_from_file_impl: using device CUDA0 (NVIDIA GeForce RTX 3090) (0000:01:00.0) - 20830 MiB free
+llama_model_load_from_file_impl: using device CUDA1 (NVIDIA GeForce RTX 3090) (0000:03:00.0) - 23581 MiB free
+llama_model_loader: loaded meta data with 48 key-value pairs and 402 tensors from /mnt/world8/AI/ToBench/granite-4.0-h-350m-unsloth/Magic_Quant/GGUF/dc_round-1_granite-4.0-h-350m-unsloth-mxfp4_moe-attn_kv_BF16-attn_output_BF16-attn_q_BF16-embeddings_MXFP4-ffn_down_BF16-ffn_up_gate_BF16.gguf (version GGUF V3 (latest))
+llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output.
+llama_model_loader: - kv   0:                       general.architecture str              = granitehybrid
+llama_model_loader: - kv   1:                               general.type str              = model
+llama_model_loader: - kv   2:                               general.name str              = Granite 4.0 H 350m Unsloth
+llama_model_loader: - kv   3:                           general.finetune str              = unsloth
+llama_model_loader: - kv   4:                           general.basename str              = granite-4.0-h
+llama_model_loader: - kv   5:                         general.size_label str              = 350M
+llama_model_loader: - kv   6:                            general.license str              = apache-2.0
+llama_model_loader: - kv   7:                   general.base_model.count u32              = 1
+llama_model_loader: - kv   8:                  general.base_model.0.name str              = Granite 4.0 H 350m
+llama_model_loader: - kv   9:          general.base_model.0.organization str              = Ibm Granite
+llama_model_loader: - kv  10:              general.base_model.0.repo_url str              = https://huggingface.co/ibm-granite/gr...
+llama_model_loader: - kv  11:                               general.tags arr[str,3]       = ["language", "unsloth", "granite-4.0"]
+llama_model_loader: - kv  12:                  granitehybrid.block_count u32              = 32
+llama_model_loader: - kv  13:               granitehybrid.context_length u32              = 1048576
+llama_model_loader: - kv  14:             granitehybrid.embedding_length u32              = 768
+llama_model_loader: - kv  15:          granitehybrid.feed_forward_length u32              = 2048
+llama_model_loader: - kv  16:         granitehybrid.attention.head_count u32              = 12
+llama_model_loader: - kv  17:      granitehybrid.attention.head_count_kv arr[i32,32]      = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, ...
+llama_model_loader: - kv  18:               granitehybrid.rope.freq_base f32              = 10000.000000
+llama_model_loader: - kv  19: granitehybrid.attention.layer_norm_rms_epsilon f32              = 0.000010
+llama_model_loader: - kv  20:                 granitehybrid.expert_count u32              = 0
+llama_model_loader: - kv  21:            granitehybrid.expert_used_count u32              = 0
+llama_model_loader: - kv  22:                   granitehybrid.vocab_size u32              = 100352
+llama_model_loader: - kv  23:         granitehybrid.rope.dimension_count u32              = 64
+llama_model_loader: - kv  24:              granitehybrid.attention.scale f32              = 0.015625
+llama_model_loader: - kv  25:              granitehybrid.embedding_scale f32              = 12.000000
+llama_model_loader: - kv  26:               granitehybrid.residual_scale f32              = 0.246000
+llama_model_loader: - kv  27:                  granitehybrid.logit_scale f32              = 3.000000
+llama_model_loader: - kv  28: granitehybrid.expert_shared_feed_forward_length u32              = 2048
+llama_model_loader: - kv  29:              granitehybrid.ssm.conv_kernel u32              = 4
+llama_model_loader: - kv  30:               granitehybrid.ssm.state_size u32              = 128
+llama_model_loader: - kv  31:              granitehybrid.ssm.group_count u32              = 1
+llama_model_loader: - kv  32:               granitehybrid.ssm.inner_size u32              = 1536
+llama_model_loader: - kv  33:           granitehybrid.ssm.time_step_rank u32              = 48
+llama_model_loader: - kv  34:       granitehybrid.rope.scaling.finetuned bool             = false
+llama_model_loader: - kv  35:                       tokenizer.ggml.model str              = gpt2
+llama_model_loader: - kv  36:                         tokenizer.ggml.pre str              = dbrx
+llama_model_loader: - kv  37:                      tokenizer.ggml.tokens arr[str,100352]  = ["!", "\"", "#", "$", "%", "&", "'", ...
+llama_model_loader: - kv  38:                  tokenizer.ggml.token_type arr[i32,100352]  = [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...
+llama_model_loader: - kv  39:                      tokenizer.ggml.merges arr[str,100000]  = ["Ġ Ġ", "ĠĠ ĠĠ", "i n", "Ġ t",...
+llama_model_loader: - kv  40:                tokenizer.ggml.bos_token_id u32              = 100257
+llama_model_loader: - kv  41:                tokenizer.ggml.eos_token_id u32              = 100257
+llama_model_loader: - kv  42:            tokenizer.ggml.unknown_token_id u32              = 100269
+llama_model_loader: - kv  43:            tokenizer.ggml.padding_token_id u32              = 100256
+llama_model_loader: - kv  44:               tokenizer.ggml.add_bos_token bool             = false
+llama_model_loader: - kv  45:                    tokenizer.chat_template str              = {%- set tools_system_message_prefix =...
+llama_model_loader: - kv  46:               general.quantization_version u32              = 2
+llama_model_loader: - kv  47:                          general.file_type u32              = 38
+llama_model_loader: - type  f32:  233 tensors
+llama_model_loader: - type q8_0:   56 tensors
+llama_model_loader: - type bf16:  112 tensors
+llama_model_loader: - type mxfp4:    1 tensors
+print_info: file format = GGUF V3 (latest)
+print_info: file type   = MXFP4 MoE
+print_info: file size   = 447.40 MiB (11.03 BPW) 
+load: printing all EOG tokens:
+load:   - 100257 ('<|end_of_text|>')
+load:   - 100261 ('<|fim_pad|>')
+load: special tokens cache size = 96
+load: token to piece cache size = 0.6152 MB
+print_info: arch             = granitehybrid
+print_info: vocab_only       = 0
+print_info: n_ctx_train      = 1048576
+print_info: n_embd           = 768
+print_info: n_embd_inp       = 768
+print_info: n_layer          = 32
+print_info: n_head           = 12
+print_info: n_head_kv        = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0]
+print_info: n_rot            = 64
+print_info: n_swa            = 0
+print_info: is_swa_any       = 0
+print_info: n_embd_head_k    = 64
+print_info: n_embd_head_v    = 64
+print_info: n_gqa            = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 3, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0]
+print_info: n_embd_k_gqa     = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 256, 0, 0, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 0, 0]
+print_info: n_embd_v_gqa     = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 256, 0, 0, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 0, 0]
+print_info: f_norm_eps       = 0.0e+00
+print_info: f_norm_rms_eps   = 1.0e-05
+print_info: f_clamp_kqv      = 0.0e+00
+print_info: f_max_alibi_bias = 0.0e+00
+print_info: f_logit_scale    = 3.0e+00
+print_info: f_attn_scale     = 1.6e-02
+print_info: n_ff             = 2048
+print_info: n_expert         = 0
+print_info: n_expert_used    = 0
+print_info: n_expert_groups  = 0
+print_info: n_group_used     = 0
+print_info: causal attn      = 1
+print_info: pooling type     = 0
+print_info: rope type        = 0
+print_info: rope scaling     = linear
+print_info: freq_base_train  = 10000.0
+print_info: freq_scale_train = 1
+print_info: n_ctx_orig_yarn  = 1048576
+print_info: rope_finetuned   = unknown
+print_info: ssm_d_conv       = 4
+print_info: ssm_d_inner      = 1536
+print_info: ssm_d_state      = 128
+print_info: ssm_dt_rank      = 48
+print_info: ssm_n_group      = 1
+print_info: ssm_dt_b_c_rms   = 0
+print_info: model type       = 350M
+print_info: model params     = 340.33 M
+print_info: general.name     = Granite 4.0 H 350m Unsloth
+print_info: f_embedding_scale = 12.000000
+print_info: f_residual_scale  = 0.246000
+print_info: f_attention_scale = 0.015625
+print_info: n_ff_shexp        = 2048
+print_info: vocab type       = BPE
+print_info: n_vocab          = 100352
+print_info: n_merges         = 100000
+print_info: BOS token        = 100257 '<|end_of_text|>'
+print_info: EOS token        = 100257 '<|end_of_text|>'
+print_info: EOT token        = 100257 '<|end_of_text|>'
+print_info: UNK token        = 100269 '<|unk|>'
+print_info: PAD token        = 100256 '<|pad|>'
+print_info: LF token         = 198 'Ċ'
+print_info: FIM PRE token    = 100258 '<|fim_prefix|>'
+print_info: FIM SUF token    = 100260 '<|fim_suffix|>'
+print_info: FIM MID token    = 100259 '<|fim_middle|>'
+print_info: FIM PAD token    = 100261 '<|fim_pad|>'
+print_info: EOG token        = 100257 '<|end_of_text|>'
+print_info: EOG token        = 100261 '<|fim_pad|>'
+print_info: max token length = 256
+load_tensors: loading model tensors, this can take a while... (mmap = true)
+load_tensors: offloading 20 repeating layers to GPU
+load_tensors: offloaded 20/33 layers to GPU
+load_tensors:   CPU_Mapped model buffer size =   192.61 MiB
+load_tensors:        CUDA0 model buffer size =   126.97 MiB
+load_tensors:        CUDA1 model buffer size =   127.83 MiB
+.............................................................................................
+llama_context: constructing llama_context
+llama_context: n_seq_max     = 1
+llama_context: n_ctx         = 2048
+llama_context: n_ctx_seq     = 2048
+llama_context: n_batch       = 2048
+llama_context: n_ubatch      = 512
+llama_context: causal_attn   = 1
+llama_context: flash_attn    = auto
+llama_context: kv_unified    = false
+llama_context: freq_base     = 10000.0
+llama_context: freq_scale    = 1
+llama_context: n_ctx_seq (2048) < n_ctx_train (1048576) -- the full capacity of the model will not be utilized
+llama_context:        CPU  output buffer size =     0.38 MiB
+llama_kv_cache:        CPU KV buffer size =     2.00 MiB
+llama_kv_cache:      CUDA0 KV buffer size =     4.00 MiB
+llama_kv_cache:      CUDA1 KV buffer size =     2.00 MiB
+llama_kv_cache: size =    8.00 MiB (  2048 cells,   4 layers,  1/1 seqs), K (f16):    4.00 MiB, V (f16):    4.00 MiB
+llama_memory_recurrent:        CPU RS buffer size =     8.48 MiB
+llama_memory_recurrent:      CUDA0 RS buffer size =     6.16 MiB
+llama_memory_recurrent:      CUDA1 RS buffer size =     6.93 MiB
+llama_memory_recurrent: size =   21.57 MiB (     1 cells,  32 layers,  1 seqs), R (f32):    0.57 MiB, S (f32):   21.00 MiB
+llama_context: Flash Attention was auto, set to enabled
+llama_context:      CUDA0 compute buffer size =   244.64 MiB
+llama_context:      CUDA1 compute buffer size =    22.39 MiB
+llama_context:  CUDA_Host compute buffer size =    18.34 MiB
+llama_context: graph nodes  = 1815
+llama_context: graph splits = 182 (with bs=512), 41 (with bs=1)
+common_init_from_params: added <|end_of_text|> logit bias = -inf
+common_init_from_params: added <|fim_pad|> logit bias = -inf
+common_init_from_params: setting dry_penalty_last_n to ctx_size = 2048
+common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable)
+
+system_info: n_threads = 16 (n_threads_batch = 16) / 32 | CUDA : ARCHS = 860 | USE_GRAPHS = 1 | PEER_MAX_BATCH_SIZE = 128 | CPU : SSE3 = 1 | SSSE3 = 1 | AVX = 1 | AVX2 = 1 | F16C = 1 | FMA = 1 | BMI2 = 1 | AVX512 = 1 | AVX512_VBMI = 1 | AVX512_VNNI = 1 | AVX512_BF16 = 1 | LLAMAFILE = 1 | OPENMP = 1 | REPACK = 1 | 
+perplexity: tokenizing the input ..
+perplexity: tokenization took 93.913 ms
+perplexity: calculating perplexity over 44 chunks, n_ctx=2048, batch_size=2048, n_seq=1
+perplexity: 0.61 seconds per pass - ETA 0.45 minutes
+[1]4.9471,[2]4.5201,[3]2.8084,[4]2.5576,[5]2.8301,[6]3.1267,[7]2.9493,[8]2.7195,[9]2.4827,[10]2.2864,[11]2.2709,[12]2.3069,[13]2.2021,[14]2.1747,[15]2.2264,[16]2.1502,[17]2.1256,[18]2.1518,[19]2.1065,[20]2.0656,[21]2.0282,[22]2.0129,[23]2.0472,[24]2.0167,[25]2.0429,[26]2.0040,[27]1.9894,[28]1.9804,[29]2.0353,[30]2.0560,[31]2.0533,[32]2.0254,[33]2.0531,[34]2.0440,[35]2.0227,[36]2.0569,[37]2.0659,[38]2.0633,[39]2.0880,[40]2.0858,[41]2.0778,[42]2.1065,[43]2.1181,[44]2.1057,
+Final estimate: PPL = 2.1057 +/- 0.01995
+
+llama_perf_context_print:        load time =     233.65 ms
+llama_perf_context_print: prompt eval time =   15369.34 ms / 90112 tokens (    0.17 ms per token,  5863.10 tokens per second)
+llama_perf_context_print:        eval time =       0.00 ms /     1 runs   (    0.00 ms per token,      inf tokens per second)
+llama_perf_context_print:       total time =   16159.82 ms / 90113 tokens
+llama_perf_context_print:    graphs reused =          0
+llama_memory_breakdown_print: | memory breakdown [MiB] | total    free    self   model   context   compute    unaccounted |
+llama_memory_breakdown_print: |   - CUDA0 (RTX 3090)   | 24115 = 20502 + ( 382 =   126 +      10 +     245) +        3230 |
+llama_memory_breakdown_print: |   - CUDA1 (RTX 3090)   | 24124 = 23339 + ( 159 =   127 +       8 +      22) +         625 |
+llama_memory_breakdown_print: |   - Host               |                   221 =   192 +      10 +      18                |

Benchmarks/DataCollection/granite-4.0-h-350m-unsloth-mxfp4_moe-attn_kv_BF16-attn_output_BF16-attn_q_BF16-embeddings_MXFP4-ffn_down_BF16-ffn_up_gate_BF16/perplexity_general.log

@@ -0,0 +1,190 @@
+ggml_cuda_init: GGML_CUDA_FORCE_MMQ:    no
+ggml_cuda_init: GGML_CUDA_FORCE_CUBLAS: no
+ggml_cuda_init: found 2 CUDA devices:
+  Device 0: NVIDIA GeForce RTX 3090, compute capability 8.6, VMM: yes
+  Device 1: NVIDIA GeForce RTX 3090, compute capability 8.6, VMM: yes
+build: 7040 (92bb442ad) with cc (Ubuntu 13.3.0-6ubuntu2~24.04) 13.3.0 for x86_64-linux-gnu
+llama_model_load_from_file_impl: using device CUDA0 (NVIDIA GeForce RTX 3090) (0000:01:00.0) - 20829 MiB free
+llama_model_load_from_file_impl: using device CUDA1 (NVIDIA GeForce RTX 3090) (0000:03:00.0) - 23581 MiB free
+llama_model_loader: loaded meta data with 48 key-value pairs and 402 tensors from /mnt/world8/AI/ToBench/granite-4.0-h-350m-unsloth/Magic_Quant/GGUF/dc_round-1_granite-4.0-h-350m-unsloth-mxfp4_moe-attn_kv_BF16-attn_output_BF16-attn_q_BF16-embeddings_MXFP4-ffn_down_BF16-ffn_up_gate_BF16.gguf (version GGUF V3 (latest))
+llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output.
+llama_model_loader: - kv   0:                       general.architecture str              = granitehybrid
+llama_model_loader: - kv   1:                               general.type str              = model
+llama_model_loader: - kv   2:                               general.name str              = Granite 4.0 H 350m Unsloth
+llama_model_loader: - kv   3:                           general.finetune str              = unsloth
+llama_model_loader: - kv   4:                           general.basename str              = granite-4.0-h
+llama_model_loader: - kv   5:                         general.size_label str              = 350M
+llama_model_loader: - kv   6:                            general.license str              = apache-2.0
+llama_model_loader: - kv   7:                   general.base_model.count u32              = 1
+llama_model_loader: - kv   8:                  general.base_model.0.name str              = Granite 4.0 H 350m
+llama_model_loader: - kv   9:          general.base_model.0.organization str              = Ibm Granite
+llama_model_loader: - kv  10:              general.base_model.0.repo_url str              = https://huggingface.co/ibm-granite/gr...
+llama_model_loader: - kv  11:                               general.tags arr[str,3]       = ["language", "unsloth", "granite-4.0"]
+llama_model_loader: - kv  12:                  granitehybrid.block_count u32              = 32
+llama_model_loader: - kv  13:               granitehybrid.context_length u32              = 1048576
+llama_model_loader: - kv  14:             granitehybrid.embedding_length u32              = 768
+llama_model_loader: - kv  15:          granitehybrid.feed_forward_length u32              = 2048
+llama_model_loader: - kv  16:         granitehybrid.attention.head_count u32              = 12
+llama_model_loader: - kv  17:      granitehybrid.attention.head_count_kv arr[i32,32]      = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, ...
+llama_model_loader: - kv  18:               granitehybrid.rope.freq_base f32              = 10000.000000
+llama_model_loader: - kv  19: granitehybrid.attention.layer_norm_rms_epsilon f32              = 0.000010
+llama_model_loader: - kv  20:                 granitehybrid.expert_count u32              = 0
+llama_model_loader: - kv  21:            granitehybrid.expert_used_count u32              = 0
+llama_model_loader: - kv  22:                   granitehybrid.vocab_size u32              = 100352
+llama_model_loader: - kv  23:         granitehybrid.rope.dimension_count u32              = 64
+llama_model_loader: - kv  24:              granitehybrid.attention.scale f32              = 0.015625
+llama_model_loader: - kv  25:              granitehybrid.embedding_scale f32              = 12.000000
+llama_model_loader: - kv  26:               granitehybrid.residual_scale f32              = 0.246000
+llama_model_loader: - kv  27:                  granitehybrid.logit_scale f32              = 3.000000
+llama_model_loader: - kv  28: granitehybrid.expert_shared_feed_forward_length u32              = 2048
+llama_model_loader: - kv  29:              granitehybrid.ssm.conv_kernel u32              = 4
+llama_model_loader: - kv  30:               granitehybrid.ssm.state_size u32              = 128
+llama_model_loader: - kv  31:              granitehybrid.ssm.group_count u32              = 1
+llama_model_loader: - kv  32:               granitehybrid.ssm.inner_size u32              = 1536
+llama_model_loader: - kv  33:           granitehybrid.ssm.time_step_rank u32              = 48
+llama_model_loader: - kv  34:       granitehybrid.rope.scaling.finetuned bool             = false
+llama_model_loader: - kv  35:                       tokenizer.ggml.model str              = gpt2
+llama_model_loader: - kv  36:                         tokenizer.ggml.pre str              = dbrx
+llama_model_loader: - kv  37:                      tokenizer.ggml.tokens arr[str,100352]  = ["!", "\"", "#", "$", "%", "&", "'", ...
+llama_model_loader: - kv  38:                  tokenizer.ggml.token_type arr[i32,100352]  = [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...
+llama_model_loader: - kv  39:                      tokenizer.ggml.merges arr[str,100000]  = ["Ġ Ġ", "ĠĠ ĠĠ", "i n", "Ġ t",...
+llama_model_loader: - kv  40:                tokenizer.ggml.bos_token_id u32              = 100257
+llama_model_loader: - kv  41:                tokenizer.ggml.eos_token_id u32              = 100257
+llama_model_loader: - kv  42:            tokenizer.ggml.unknown_token_id u32              = 100269
+llama_model_loader: - kv  43:            tokenizer.ggml.padding_token_id u32              = 100256
+llama_model_loader: - kv  44:               tokenizer.ggml.add_bos_token bool             = false
+llama_model_loader: - kv  45:                    tokenizer.chat_template str              = {%- set tools_system_message_prefix =...
+llama_model_loader: - kv  46:               general.quantization_version u32              = 2
+llama_model_loader: - kv  47:                          general.file_type u32              = 38
+llama_model_loader: - type  f32:  233 tensors
+llama_model_loader: - type q8_0:   56 tensors
+llama_model_loader: - type bf16:  112 tensors
+llama_model_loader: - type mxfp4:    1 tensors
+print_info: file format = GGUF V3 (latest)
+print_info: file type   = MXFP4 MoE
+print_info: file size   = 447.40 MiB (11.03 BPW) 
+load: printing all EOG tokens:
+load:   - 100257 ('<|end_of_text|>')
+load:   - 100261 ('<|fim_pad|>')
+load: special tokens cache size = 96
+load: token to piece cache size = 0.6152 MB
+print_info: arch             = granitehybrid
+print_info: vocab_only       = 0
+print_info: n_ctx_train      = 1048576
+print_info: n_embd           = 768
+print_info: n_embd_inp       = 768
+print_info: n_layer          = 32
+print_info: n_head           = 12
+print_info: n_head_kv        = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0]
+print_info: n_rot            = 64
+print_info: n_swa            = 0
+print_info: is_swa_any       = 0
+print_info: n_embd_head_k    = 64
+print_info: n_embd_head_v    = 64
+print_info: n_gqa            = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 3, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0]
+print_info: n_embd_k_gqa     = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 256, 0, 0, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 0, 0]
+print_info: n_embd_v_gqa     = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 256, 0, 0, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 0, 0]
+print_info: f_norm_eps       = 0.0e+00
+print_info: f_norm_rms_eps   = 1.0e-05
+print_info: f_clamp_kqv      = 0.0e+00
+print_info: f_max_alibi_bias = 0.0e+00
+print_info: f_logit_scale    = 3.0e+00
+print_info: f_attn_scale     = 1.6e-02
+print_info: n_ff             = 2048
+print_info: n_expert         = 0
+print_info: n_expert_used    = 0
+print_info: n_expert_groups  = 0
+print_info: n_group_used     = 0
+print_info: causal attn      = 1
+print_info: pooling type     = 0
+print_info: rope type        = 0
+print_info: rope scaling     = linear
+print_info: freq_base_train  = 10000.0
+print_info: freq_scale_train = 1
+print_info: n_ctx_orig_yarn  = 1048576
+print_info: rope_finetuned   = unknown
+print_info: ssm_d_conv       = 4
+print_info: ssm_d_inner      = 1536
+print_info: ssm_d_state      = 128
+print_info: ssm_dt_rank      = 48
+print_info: ssm_n_group      = 1
+print_info: ssm_dt_b_c_rms   = 0
+print_info: model type       = 350M
+print_info: model params     = 340.33 M
+print_info: general.name     = Granite 4.0 H 350m Unsloth
+print_info: f_embedding_scale = 12.000000
+print_info: f_residual_scale  = 0.246000
+print_info: f_attention_scale = 0.015625
+print_info: n_ff_shexp        = 2048
+print_info: vocab type       = BPE
+print_info: n_vocab          = 100352
+print_info: n_merges         = 100000
+print_info: BOS token        = 100257 '<|end_of_text|>'
+print_info: EOS token        = 100257 '<|end_of_text|>'
+print_info: EOT token        = 100257 '<|end_of_text|>'
+print_info: UNK token        = 100269 '<|unk|>'
+print_info: PAD token        = 100256 '<|pad|>'
+print_info: LF token         = 198 'Ċ'
+print_info: FIM PRE token    = 100258 '<|fim_prefix|>'
+print_info: FIM SUF token    = 100260 '<|fim_suffix|>'
+print_info: FIM MID token    = 100259 '<|fim_middle|>'
+print_info: FIM PAD token    = 100261 '<|fim_pad|>'
+print_info: EOG token        = 100257 '<|end_of_text|>'
+print_info: EOG token        = 100261 '<|fim_pad|>'
+print_info: max token length = 256
+load_tensors: loading model tensors, this can take a while... (mmap = true)
+load_tensors: offloading 20 repeating layers to GPU
+load_tensors: offloaded 20/33 layers to GPU
+load_tensors:   CPU_Mapped model buffer size =   192.61 MiB
+load_tensors:        CUDA0 model buffer size =   126.97 MiB
+load_tensors:        CUDA1 model buffer size =   127.83 MiB
+.............................................................................................
+llama_context: constructing llama_context
+llama_context: n_seq_max     = 1
+llama_context: n_ctx         = 2048
+llama_context: n_ctx_seq     = 2048
+llama_context: n_batch       = 2048
+llama_context: n_ubatch      = 512
+llama_context: causal_attn   = 1
+llama_context: flash_attn    = auto
+llama_context: kv_unified    = false
+llama_context: freq_base     = 10000.0
+llama_context: freq_scale    = 1
+llama_context: n_ctx_seq (2048) < n_ctx_train (1048576) -- the full capacity of the model will not be utilized
+llama_context:        CPU  output buffer size =     0.38 MiB
+llama_kv_cache:        CPU KV buffer size =     2.00 MiB
+llama_kv_cache:      CUDA0 KV buffer size =     4.00 MiB
+llama_kv_cache:      CUDA1 KV buffer size =     2.00 MiB
+llama_kv_cache: size =    8.00 MiB (  2048 cells,   4 layers,  1/1 seqs), K (f16):    4.00 MiB, V (f16):    4.00 MiB
+llama_memory_recurrent:        CPU RS buffer size =     8.48 MiB
+llama_memory_recurrent:      CUDA0 RS buffer size =     6.16 MiB
+llama_memory_recurrent:      CUDA1 RS buffer size =     6.93 MiB
+llama_memory_recurrent: size =   21.57 MiB (     1 cells,  32 layers,  1 seqs), R (f32):    0.57 MiB, S (f32):   21.00 MiB
+llama_context: Flash Attention was auto, set to enabled
+llama_context:      CUDA0 compute buffer size =   244.64 MiB
+llama_context:      CUDA1 compute buffer size =    22.39 MiB
+llama_context:  CUDA_Host compute buffer size =    18.34 MiB
+llama_context: graph nodes  = 1815
+llama_context: graph splits = 182 (with bs=512), 41 (with bs=1)
+common_init_from_params: added <|end_of_text|> logit bias = -inf
+common_init_from_params: added <|fim_pad|> logit bias = -inf
+common_init_from_params: setting dry_penalty_last_n to ctx_size = 2048
+common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable)
+
+system_info: n_threads = 16 (n_threads_batch = 16) / 32 | CUDA : ARCHS = 860 | USE_GRAPHS = 1 | PEER_MAX_BATCH_SIZE = 128 | CPU : SSE3 = 1 | SSSE3 = 1 | AVX = 1 | AVX2 = 1 | F16C = 1 | FMA = 1 | BMI2 = 1 | AVX512 = 1 | AVX512_VBMI = 1 | AVX512_VNNI = 1 | AVX512_BF16 = 1 | LLAMAFILE = 1 | OPENMP = 1 | REPACK = 1 | 
+perplexity: tokenizing the input ..
+perplexity: tokenization took 39.722 ms
+perplexity: calculating perplexity over 14 chunks, n_ctx=2048, batch_size=2048, n_seq=1
+perplexity: 0.58 seconds per pass - ETA 0.13 minutes
+[1]21.9676,[2]26.3752,[3]27.9004,[4]24.9540,[5]24.5701,[6]21.7264,[7]21.3481,[8]21.2797,[9]21.9842,[10]22.0063,[11]21.7919,[12]21.9882,[13]22.0670,[14]22.1016,
+Final estimate: PPL = 22.1016 +/- 0.58040
+
+llama_perf_context_print:        load time =     260.10 ms
+llama_perf_context_print: prompt eval time =    5108.65 ms / 28672 tokens (    0.18 ms per token,  5612.44 tokens per second)
+llama_perf_context_print:        eval time =       0.00 ms /     1 runs   (    0.00 ms per token,      inf tokens per second)
+llama_perf_context_print:       total time =    5376.36 ms / 28673 tokens
+llama_perf_context_print:    graphs reused =          0
+llama_memory_breakdown_print: | memory breakdown [MiB] | total    free    self   model   context   compute    unaccounted |
+llama_memory_breakdown_print: |   - CUDA0 (RTX 3090)   | 24115 = 20364 + ( 382 =   126 +      10 +     245) +        3368 |
+llama_memory_breakdown_print: |   - CUDA1 (RTX 3090)   | 24124 = 23339 + ( 159 =   127 +       8 +      22) +         625 |
+llama_memory_breakdown_print: |   - Host               |                   221 =   192 +      10 +      18                |

Benchmarks/DataCollection/granite-4.0-h-350m-unsloth-mxfp4_moe-attn_kv_BF16-attn_output_BF16-attn_q_BF16-embeddings_MXFP4-ffn_down_BF16-ffn_up_gate_BF16/perplexity_math.log

@@ -0,0 +1,190 @@
+ggml_cuda_init: GGML_CUDA_FORCE_MMQ:    no
+ggml_cuda_init: GGML_CUDA_FORCE_CUBLAS: no
+ggml_cuda_init: found 2 CUDA devices:
+  Device 0: NVIDIA GeForce RTX 3090, compute capability 8.6, VMM: yes
+  Device 1: NVIDIA GeForce RTX 3090, compute capability 8.6, VMM: yes
+build: 7040 (92bb442ad) with cc (Ubuntu 13.3.0-6ubuntu2~24.04) 13.3.0 for x86_64-linux-gnu
+llama_model_load_from_file_impl: using device CUDA0 (NVIDIA GeForce RTX 3090) (0000:01:00.0) - 20968 MiB free
+llama_model_load_from_file_impl: using device CUDA1 (NVIDIA GeForce RTX 3090) (0000:03:00.0) - 23581 MiB free
+llama_model_loader: loaded meta data with 48 key-value pairs and 402 tensors from /mnt/world8/AI/ToBench/granite-4.0-h-350m-unsloth/Magic_Quant/GGUF/dc_round-1_granite-4.0-h-350m-unsloth-mxfp4_moe-attn_kv_BF16-attn_output_BF16-attn_q_BF16-embeddings_MXFP4-ffn_down_BF16-ffn_up_gate_BF16.gguf (version GGUF V3 (latest))
+llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output.
+llama_model_loader: - kv   0:                       general.architecture str              = granitehybrid
+llama_model_loader: - kv   1:                               general.type str              = model
+llama_model_loader: - kv   2:                               general.name str              = Granite 4.0 H 350m Unsloth
+llama_model_loader: - kv   3:                           general.finetune str              = unsloth
+llama_model_loader: - kv   4:                           general.basename str              = granite-4.0-h
+llama_model_loader: - kv   5:                         general.size_label str              = 350M
+llama_model_loader: - kv   6:                            general.license str              = apache-2.0
+llama_model_loader: - kv   7:                   general.base_model.count u32              = 1
+llama_model_loader: - kv   8:                  general.base_model.0.name str              = Granite 4.0 H 350m
+llama_model_loader: - kv   9:          general.base_model.0.organization str              = Ibm Granite
+llama_model_loader: - kv  10:              general.base_model.0.repo_url str              = https://huggingface.co/ibm-granite/gr...
+llama_model_loader: - kv  11:                               general.tags arr[str,3]       = ["language", "unsloth", "granite-4.0"]
+llama_model_loader: - kv  12:                  granitehybrid.block_count u32              = 32
+llama_model_loader: - kv  13:               granitehybrid.context_length u32              = 1048576
+llama_model_loader: - kv  14:             granitehybrid.embedding_length u32              = 768
+llama_model_loader: - kv  15:          granitehybrid.feed_forward_length u32              = 2048
+llama_model_loader: - kv  16:         granitehybrid.attention.head_count u32              = 12
+llama_model_loader: - kv  17:      granitehybrid.attention.head_count_kv arr[i32,32]      = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, ...
+llama_model_loader: - kv  18:               granitehybrid.rope.freq_base f32              = 10000.000000
+llama_model_loader: - kv  19: granitehybrid.attention.layer_norm_rms_epsilon f32              = 0.000010
+llama_model_loader: - kv  20:                 granitehybrid.expert_count u32              = 0
+llama_model_loader: - kv  21:            granitehybrid.expert_used_count u32              = 0
+llama_model_loader: - kv  22:                   granitehybrid.vocab_size u32              = 100352
+llama_model_loader: - kv  23:         granitehybrid.rope.dimension_count u32              = 64
+llama_model_loader: - kv  24:              granitehybrid.attention.scale f32              = 0.015625
+llama_model_loader: - kv  25:              granitehybrid.embedding_scale f32              = 12.000000
+llama_model_loader: - kv  26:               granitehybrid.residual_scale f32              = 0.246000
+llama_model_loader: - kv  27:                  granitehybrid.logit_scale f32              = 3.000000
+llama_model_loader: - kv  28: granitehybrid.expert_shared_feed_forward_length u32              = 2048
+llama_model_loader: - kv  29:              granitehybrid.ssm.conv_kernel u32              = 4
+llama_model_loader: - kv  30:               granitehybrid.ssm.state_size u32              = 128
+llama_model_loader: - kv  31:              granitehybrid.ssm.group_count u32              = 1
+llama_model_loader: - kv  32:               granitehybrid.ssm.inner_size u32              = 1536
+llama_model_loader: - kv  33:           granitehybrid.ssm.time_step_rank u32              = 48
+llama_model_loader: - kv  34:       granitehybrid.rope.scaling.finetuned bool             = false
+llama_model_loader: - kv  35:                       tokenizer.ggml.model str              = gpt2
+llama_model_loader: - kv  36:                         tokenizer.ggml.pre str              = dbrx
+llama_model_loader: - kv  37:                      tokenizer.ggml.tokens arr[str,100352]  = ["!", "\"", "#", "$", "%", "&", "'", ...
+llama_model_loader: - kv  38:                  tokenizer.ggml.token_type arr[i32,100352]  = [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...
+llama_model_loader: - kv  39:                      tokenizer.ggml.merges arr[str,100000]  = ["Ġ Ġ", "ĠĠ ĠĠ", "i n", "Ġ t",...
+llama_model_loader: - kv  40:                tokenizer.ggml.bos_token_id u32              = 100257
+llama_model_loader: - kv  41:                tokenizer.ggml.eos_token_id u32              = 100257
+llama_model_loader: - kv  42:            tokenizer.ggml.unknown_token_id u32              = 100269
+llama_model_loader: - kv  43:            tokenizer.ggml.padding_token_id u32              = 100256
+llama_model_loader: - kv  44:               tokenizer.ggml.add_bos_token bool             = false
+llama_model_loader: - kv  45:                    tokenizer.chat_template str              = {%- set tools_system_message_prefix =...
+llama_model_loader: - kv  46:               general.quantization_version u32              = 2
+llama_model_loader: - kv  47:                          general.file_type u32              = 38
+llama_model_loader: - type  f32:  233 tensors
+llama_model_loader: - type q8_0:   56 tensors
+llama_model_loader: - type bf16:  112 tensors
+llama_model_loader: - type mxfp4:    1 tensors
+print_info: file format = GGUF V3 (latest)
+print_info: file type   = MXFP4 MoE
+print_info: file size   = 447.40 MiB (11.03 BPW) 
+load: printing all EOG tokens:
+load:   - 100257 ('<|end_of_text|>')
+load:   - 100261 ('<|fim_pad|>')
+load: special tokens cache size = 96
+load: token to piece cache size = 0.6152 MB
+print_info: arch             = granitehybrid
+print_info: vocab_only       = 0
+print_info: n_ctx_train      = 1048576
+print_info: n_embd           = 768
+print_info: n_embd_inp       = 768
+print_info: n_layer          = 32
+print_info: n_head           = 12
+print_info: n_head_kv        = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0]
+print_info: n_rot            = 64
+print_info: n_swa            = 0
+print_info: is_swa_any       = 0
+print_info: n_embd_head_k    = 64
+print_info: n_embd_head_v    = 64
+print_info: n_gqa            = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 3, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0]
+print_info: n_embd_k_gqa     = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 256, 0, 0, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 0, 0]
+print_info: n_embd_v_gqa     = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 256, 0, 0, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 0, 0]
+print_info: f_norm_eps       = 0.0e+00
+print_info: f_norm_rms_eps   = 1.0e-05
+print_info: f_clamp_kqv      = 0.0e+00
+print_info: f_max_alibi_bias = 0.0e+00
+print_info: f_logit_scale    = 3.0e+00
+print_info: f_attn_scale     = 1.6e-02
+print_info: n_ff             = 2048
+print_info: n_expert         = 0
+print_info: n_expert_used    = 0
+print_info: n_expert_groups  = 0
+print_info: n_group_used     = 0
+print_info: causal attn      = 1
+print_info: pooling type     = 0
+print_info: rope type        = 0
+print_info: rope scaling     = linear
+print_info: freq_base_train  = 10000.0
+print_info: freq_scale_train = 1
+print_info: n_ctx_orig_yarn  = 1048576
+print_info: rope_finetuned   = unknown
+print_info: ssm_d_conv       = 4
+print_info: ssm_d_inner      = 1536
+print_info: ssm_d_state      = 128
+print_info: ssm_dt_rank      = 48
+print_info: ssm_n_group      = 1
+print_info: ssm_dt_b_c_rms   = 0
+print_info: model type       = 350M
+print_info: model params     = 340.33 M
+print_info: general.name     = Granite 4.0 H 350m Unsloth
+print_info: f_embedding_scale = 12.000000
+print_info: f_residual_scale  = 0.246000
+print_info: f_attention_scale = 0.015625
+print_info: n_ff_shexp        = 2048
+print_info: vocab type       = BPE
+print_info: n_vocab          = 100352
+print_info: n_merges         = 100000
+print_info: BOS token        = 100257 '<|end_of_text|>'
+print_info: EOS token        = 100257 '<|end_of_text|>'
+print_info: EOT token        = 100257 '<|end_of_text|>'
+print_info: UNK token        = 100269 '<|unk|>'
+print_info: PAD token        = 100256 '<|pad|>'
+print_info: LF token         = 198 'Ċ'
+print_info: FIM PRE token    = 100258 '<|fim_prefix|>'
+print_info: FIM SUF token    = 100260 '<|fim_suffix|>'
+print_info: FIM MID token    = 100259 '<|fim_middle|>'
+print_info: FIM PAD token    = 100261 '<|fim_pad|>'
+print_info: EOG token        = 100257 '<|end_of_text|>'
+print_info: EOG token        = 100261 '<|fim_pad|>'
+print_info: max token length = 256
+load_tensors: loading model tensors, this can take a while... (mmap = true)
+load_tensors: offloading 20 repeating layers to GPU
+load_tensors: offloaded 20/33 layers to GPU
+load_tensors:   CPU_Mapped model buffer size =   192.61 MiB
+load_tensors:        CUDA0 model buffer size =   126.97 MiB
+load_tensors:        CUDA1 model buffer size =   127.83 MiB
+.............................................................................................
+llama_context: constructing llama_context
+llama_context: n_seq_max     = 1
+llama_context: n_ctx         = 2048
+llama_context: n_ctx_seq     = 2048
+llama_context: n_batch       = 2048
+llama_context: n_ubatch      = 512
+llama_context: causal_attn   = 1
+llama_context: flash_attn    = auto
+llama_context: kv_unified    = false
+llama_context: freq_base     = 10000.0
+llama_context: freq_scale    = 1
+llama_context: n_ctx_seq (2048) < n_ctx_train (1048576) -- the full capacity of the model will not be utilized
+llama_context:        CPU  output buffer size =     0.38 MiB
+llama_kv_cache:        CPU KV buffer size =     2.00 MiB
+llama_kv_cache:      CUDA0 KV buffer size =     4.00 MiB
+llama_kv_cache:      CUDA1 KV buffer size =     2.00 MiB
+llama_kv_cache: size =    8.00 MiB (  2048 cells,   4 layers,  1/1 seqs), K (f16):    4.00 MiB, V (f16):    4.00 MiB
+llama_memory_recurrent:        CPU RS buffer size =     8.48 MiB
+llama_memory_recurrent:      CUDA0 RS buffer size =     6.16 MiB
+llama_memory_recurrent:      CUDA1 RS buffer size =     6.93 MiB
+llama_memory_recurrent: size =   21.57 MiB (     1 cells,  32 layers,  1 seqs), R (f32):    0.57 MiB, S (f32):   21.00 MiB
+llama_context: Flash Attention was auto, set to enabled
+llama_context:      CUDA0 compute buffer size =   244.64 MiB
+llama_context:      CUDA1 compute buffer size =    22.39 MiB
+llama_context:  CUDA_Host compute buffer size =    18.34 MiB
+llama_context: graph nodes  = 1815
+llama_context: graph splits = 182 (with bs=512), 41 (with bs=1)
+common_init_from_params: added <|end_of_text|> logit bias = -inf
+common_init_from_params: added <|fim_pad|> logit bias = -inf
+common_init_from_params: setting dry_penalty_last_n to ctx_size = 2048
+common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable)
+
+system_info: n_threads = 16 (n_threads_batch = 16) / 32 | CUDA : ARCHS = 860 | USE_GRAPHS = 1 | PEER_MAX_BATCH_SIZE = 128 | CPU : SSE3 = 1 | SSSE3 = 1 | AVX = 1 | AVX2 = 1 | F16C = 1 | FMA = 1 | BMI2 = 1 | AVX512 = 1 | AVX512_VBMI = 1 | AVX512_VNNI = 1 | AVX512_BF16 = 1 | LLAMAFILE = 1 | OPENMP = 1 | REPACK = 1 | 
+perplexity: tokenizing the input ..
+perplexity: tokenization took 34.49 ms
+perplexity: calculating perplexity over 15 chunks, n_ctx=2048, batch_size=2048, n_seq=1
+perplexity: 0.60 seconds per pass - ETA 0.13 minutes
+[1]9.4103,[2]10.7747,[3]10.4827,[4]10.8852,[5]11.1206,[6]11.1851,[7]11.3119,[8]10.9990,[9]11.0531,[10]11.0515,[11]11.3187,[12]11.4412,[13]11.5848,[14]11.5855,[15]11.4772,
+Final estimate: PPL = 11.4772 +/- 0.26023
+
+llama_perf_context_print:        load time =     235.81 ms
+llama_perf_context_print: prompt eval time =    5415.14 ms / 30720 tokens (    0.18 ms per token,  5672.99 tokens per second)
+llama_perf_context_print:        eval time =       0.00 ms /     1 runs   (    0.00 ms per token,      inf tokens per second)
+llama_perf_context_print:       total time =    5689.05 ms / 30721 tokens
+llama_perf_context_print:    graphs reused =          0
+llama_memory_breakdown_print: | memory breakdown [MiB] | total    free    self   model   context   compute    unaccounted |
+llama_memory_breakdown_print: |   - CUDA0 (RTX 3090)   | 24115 = 20502 + ( 382 =   126 +      10 +     245) +        3230 |
+llama_memory_breakdown_print: |   - CUDA1 (RTX 3090)   | 24124 = 23339 + ( 159 =   127 +       8 +      22) +         625 |
+llama_memory_breakdown_print: |   - Host               |                   221 =   192 +      10 +      18                |

Benchmarks/DataCollection/granite-4.0-h-350m-unsloth-mxfp4_moe-attn_kv_BF16-attn_output_BF16-attn_q_BF16-embeddings_MXFP4-ffn_down_MXFP4-ffn_up_gate_MXFP4/bench_metrics.json

@@ -0,0 +1,44 @@
+{
+  "raw_metrics": {
+    "llamabench": {
+      "backend": "CUDA",
+      "log_path": "granite-4.0-h-350m-unsloth-mxfp4_moe-attn_kv_BF16-attn_output_BF16-attn_q_BF16-embeddings_MXFP4-ffn_down_MXFP4-ffn_up_gate_MXFP4/llamabench.md",
+      "ngl": "35",
+      "raw_row": {
+        "backend": "CUDA",
+        "model": "granitehybrid 350M MXFP4 MoE",
+        "ngl": "35",
+        "params": "340.33 M",
+        "size": "235.90 MiB",
+        "t/s": "1689.40 \u00b1 37.45",
+        "test": "pp8",
+        "tps_value": 1689.4
+      },
+      "test": "pp8",
+      "tps": 1689.4
+    },
+    "perplexity": {
+      "code": {
+        "log_path": "granite-4.0-h-350m-unsloth-mxfp4_moe-attn_kv_BF16-attn_output_BF16-attn_q_BF16-embeddings_MXFP4-ffn_down_MXFP4-ffn_up_gate_MXFP4/perplexity_code.log",
+        "ppl": 525.8288,
+        "ppl_error": 14.018
+      },
+      "general": {
+        "log_path": "granite-4.0-h-350m-unsloth-mxfp4_moe-attn_kv_BF16-attn_output_BF16-attn_q_BF16-embeddings_MXFP4-ffn_down_MXFP4-ffn_up_gate_MXFP4/perplexity_general.log",
+        "ppl": 1147.1684,
+        "ppl_error": 45.7373
+      },
+      "math": {
+        "log_path": "granite-4.0-h-350m-unsloth-mxfp4_moe-attn_kv_BF16-attn_output_BF16-attn_q_BF16-embeddings_MXFP4-ffn_down_MXFP4-ffn_up_gate_MXFP4/perplexity_math.log",
+        "ppl": 197.3954,
+        "ppl_error": 6.91272
+      }
+    }
+  },
+  "summary": {
+    "avg_prec_loss_pct": 11614.5753,
+    "bench_tps": 1689.4,
+    "file_size_bytes": 250934208,
+    "file_size_gb": 0.23
+  }
+}

Benchmarks/DataCollection/granite-4.0-h-350m-unsloth-mxfp4_moe-attn_kv_BF16-attn_output_BF16-attn_q_BF16-embeddings_MXFP4-ffn_down_MXFP4-ffn_up_gate_MXFP4/llamabench.md

@@ -0,0 +1,11 @@
+ggml_cuda_init: GGML_CUDA_FORCE_MMQ:    no
+ggml_cuda_init: GGML_CUDA_FORCE_CUBLAS: no
+ggml_cuda_init: found 2 CUDA devices:
+  Device 0: NVIDIA GeForce RTX 3090, compute capability 8.6, VMM: yes
+  Device 1: NVIDIA GeForce RTX 3090, compute capability 8.6, VMM: yes
+| model                          |       size |     params | backend    | ngl |            test |                  t/s |
+| ------------------------------ | ---------: | ---------: | ---------- | --: | --------------: | -------------------: |
+| granitehybrid 350M MXFP4 MoE   | 235.90 MiB |   340.33 M | CUDA       |  35 |             pp8 |      1689.40 ± 37.45 |
+| granitehybrid 350M MXFP4 MoE   | 235.90 MiB |   340.33 M | CUDA       |  35 |           tg128 |       334.33 ± 12.38 |
+
+build: 92bb442ad (7040)

Benchmarks/DataCollection/granite-4.0-h-350m-unsloth-mxfp4_moe-attn_kv_BF16-attn_output_BF16-attn_q_BF16-embeddings_MXFP4-ffn_down_MXFP4-ffn_up_gate_MXFP4/perplexity_code.log

@@ -0,0 +1,190 @@
+ggml_cuda_init: GGML_CUDA_FORCE_MMQ:    no
+ggml_cuda_init: GGML_CUDA_FORCE_CUBLAS: no
+ggml_cuda_init: found 2 CUDA devices:
+  Device 0: NVIDIA GeForce RTX 3090, compute capability 8.6, VMM: yes
+  Device 1: NVIDIA GeForce RTX 3090, compute capability 8.6, VMM: yes
+build: 7040 (92bb442ad) with cc (Ubuntu 13.3.0-6ubuntu2~24.04) 13.3.0 for x86_64-linux-gnu
+llama_model_load_from_file_impl: using device CUDA0 (NVIDIA GeForce RTX 3090) (0000:01:00.0) - 20923 MiB free
+llama_model_load_from_file_impl: using device CUDA1 (NVIDIA GeForce RTX 3090) (0000:03:00.0) - 23581 MiB free
+llama_model_loader: loaded meta data with 48 key-value pairs and 402 tensors from /mnt/world8/AI/ToBench/granite-4.0-h-350m-unsloth/Magic_Quant/GGUF/dc_round1_granite-4.0-h-350m-unsloth-mxfp4_moe-attn_kv_BF16-attn_output_BF16-attn_q_BF16-embeddings_MXFP4-ffn_down_MXFP4-ffn_up_gate_MXFP4.gguf (version GGUF V3 (latest))
+llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output.
+llama_model_loader: - kv   0:                       general.architecture str              = granitehybrid
+llama_model_loader: - kv   1:                               general.type str              = model
+llama_model_loader: - kv   2:                               general.name str              = Granite 4.0 H 350m Unsloth
+llama_model_loader: - kv   3:                           general.finetune str              = unsloth
+llama_model_loader: - kv   4:                           general.basename str              = granite-4.0-h
+llama_model_loader: - kv   5:                         general.size_label str              = 350M
+llama_model_loader: - kv   6:                            general.license str              = apache-2.0
+llama_model_loader: - kv   7:                   general.base_model.count u32              = 1
+llama_model_loader: - kv   8:                  general.base_model.0.name str              = Granite 4.0 H 350m
+llama_model_loader: - kv   9:          general.base_model.0.organization str              = Ibm Granite
+llama_model_loader: - kv  10:              general.base_model.0.repo_url str              = https://huggingface.co/ibm-granite/gr...
+llama_model_loader: - kv  11:                               general.tags arr[str,3]       = ["language", "unsloth", "granite-4.0"]
+llama_model_loader: - kv  12:                  granitehybrid.block_count u32              = 32
+llama_model_loader: - kv  13:               granitehybrid.context_length u32              = 1048576
+llama_model_loader: - kv  14:             granitehybrid.embedding_length u32              = 768
+llama_model_loader: - kv  15:          granitehybrid.feed_forward_length u32              = 2048
+llama_model_loader: - kv  16:         granitehybrid.attention.head_count u32              = 12
+llama_model_loader: - kv  17:      granitehybrid.attention.head_count_kv arr[i32,32]      = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, ...
+llama_model_loader: - kv  18:               granitehybrid.rope.freq_base f32              = 10000.000000
+llama_model_loader: - kv  19: granitehybrid.attention.layer_norm_rms_epsilon f32              = 0.000010
+llama_model_loader: - kv  20:                 granitehybrid.expert_count u32              = 0
+llama_model_loader: - kv  21:            granitehybrid.expert_used_count u32              = 0
+llama_model_loader: - kv  22:                   granitehybrid.vocab_size u32              = 100352
+llama_model_loader: - kv  23:         granitehybrid.rope.dimension_count u32              = 64
+llama_model_loader: - kv  24:              granitehybrid.attention.scale f32              = 0.015625
+llama_model_loader: - kv  25:              granitehybrid.embedding_scale f32              = 12.000000
+llama_model_loader: - kv  26:               granitehybrid.residual_scale f32              = 0.246000
+llama_model_loader: - kv  27:                  granitehybrid.logit_scale f32              = 3.000000
+llama_model_loader: - kv  28: granitehybrid.expert_shared_feed_forward_length u32              = 2048
+llama_model_loader: - kv  29:              granitehybrid.ssm.conv_kernel u32              = 4
+llama_model_loader: - kv  30:               granitehybrid.ssm.state_size u32              = 128
+llama_model_loader: - kv  31:              granitehybrid.ssm.group_count u32              = 1
+llama_model_loader: - kv  32:               granitehybrid.ssm.inner_size u32              = 1536
+llama_model_loader: - kv  33:           granitehybrid.ssm.time_step_rank u32              = 48
+llama_model_loader: - kv  34:       granitehybrid.rope.scaling.finetuned bool             = false
+llama_model_loader: - kv  35:                       tokenizer.ggml.model str              = gpt2
+llama_model_loader: - kv  36:                         tokenizer.ggml.pre str              = dbrx
+llama_model_loader: - kv  37:                      tokenizer.ggml.tokens arr[str,100352]  = ["!", "\"", "#", "$", "%", "&", "'", ...
+llama_model_loader: - kv  38:                  tokenizer.ggml.token_type arr[i32,100352]  = [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...
+llama_model_loader: - kv  39:                      tokenizer.ggml.merges arr[str,100000]  = ["Ġ Ġ", "ĠĠ ĠĠ", "i n", "Ġ t",...
+llama_model_loader: - kv  40:                tokenizer.ggml.bos_token_id u32              = 100257
+llama_model_loader: - kv  41:                tokenizer.ggml.eos_token_id u32              = 100257
+llama_model_loader: - kv  42:            tokenizer.ggml.unknown_token_id u32              = 100269
+llama_model_loader: - kv  43:            tokenizer.ggml.padding_token_id u32              = 100256
+llama_model_loader: - kv  44:               tokenizer.ggml.add_bos_token bool             = false
+llama_model_loader: - kv  45:                    tokenizer.chat_template str              = {%- set tools_system_message_prefix =...
+llama_model_loader: - kv  46:               general.quantization_version u32              = 2
+llama_model_loader: - kv  47:                          general.file_type u32              = 38
+llama_model_loader: - type  f32:  233 tensors
+llama_model_loader: - type q8_0:   56 tensors
+llama_model_loader: - type bf16:   16 tensors
+llama_model_loader: - type mxfp4:   97 tensors
+print_info: file format = GGUF V3 (latest)
+print_info: file type   = MXFP4 MoE
+print_info: file size   = 235.90 MiB (5.81 BPW) 
+load: printing all EOG tokens:
+load:   - 100257 ('<|end_of_text|>')
+load:   - 100261 ('<|fim_pad|>')
+load: special tokens cache size = 96
+load: token to piece cache size = 0.6152 MB
+print_info: arch             = granitehybrid
+print_info: vocab_only       = 0
+print_info: n_ctx_train      = 1048576
+print_info: n_embd           = 768
+print_info: n_embd_inp       = 768
+print_info: n_layer          = 32
+print_info: n_head           = 12
+print_info: n_head_kv        = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0]
+print_info: n_rot            = 64
+print_info: n_swa            = 0
+print_info: is_swa_any       = 0
+print_info: n_embd_head_k    = 64
+print_info: n_embd_head_v    = 64
+print_info: n_gqa            = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 3, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0]
+print_info: n_embd_k_gqa     = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 256, 0, 0, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 0, 0]
+print_info: n_embd_v_gqa     = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 256, 0, 0, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 0, 0]
+print_info: f_norm_eps       = 0.0e+00
+print_info: f_norm_rms_eps   = 1.0e-05
+print_info: f_clamp_kqv      = 0.0e+00
+print_info: f_max_alibi_bias = 0.0e+00
+print_info: f_logit_scale    = 3.0e+00
+print_info: f_attn_scale     = 1.6e-02
+print_info: n_ff             = 2048
+print_info: n_expert         = 0
+print_info: n_expert_used    = 0
+print_info: n_expert_groups  = 0
+print_info: n_group_used     = 0
+print_info: causal attn      = 1
+print_info: pooling type     = 0
+print_info: rope type        = 0
+print_info: rope scaling     = linear
+print_info: freq_base_train  = 10000.0
+print_info: freq_scale_train = 1
+print_info: n_ctx_orig_yarn  = 1048576
+print_info: rope_finetuned   = unknown
+print_info: ssm_d_conv       = 4
+print_info: ssm_d_inner      = 1536
+print_info: ssm_d_state      = 128
+print_info: ssm_dt_rank      = 48
+print_info: ssm_n_group      = 1
+print_info: ssm_dt_b_c_rms   = 0
+print_info: model type       = 350M
+print_info: model params     = 340.33 M
+print_info: general.name     = Granite 4.0 H 350m Unsloth
+print_info: f_embedding_scale = 12.000000
+print_info: f_residual_scale  = 0.246000
+print_info: f_attention_scale = 0.015625
+print_info: n_ff_shexp        = 2048
+print_info: vocab type       = BPE
+print_info: n_vocab          = 100352
+print_info: n_merges         = 100000
+print_info: BOS token        = 100257 '<|end_of_text|>'
+print_info: EOS token        = 100257 '<|end_of_text|>'
+print_info: EOT token        = 100257 '<|end_of_text|>'
+print_info: UNK token        = 100269 '<|unk|>'
+print_info: PAD token        = 100256 '<|pad|>'
+print_info: LF token         = 198 'Ċ'
+print_info: FIM PRE token    = 100258 '<|fim_prefix|>'
+print_info: FIM SUF token    = 100260 '<|fim_suffix|>'
+print_info: FIM MID token    = 100259 '<|fim_middle|>'
+print_info: FIM PAD token    = 100261 '<|fim_pad|>'
+print_info: EOG token        = 100257 '<|end_of_text|>'
+print_info: EOG token        = 100261 '<|fim_pad|>'
+print_info: max token length = 256
+load_tensors: loading model tensors, this can take a while... (mmap = true)
+load_tensors: offloading 20 repeating layers to GPU
+load_tensors: offloaded 20/33 layers to GPU
+load_tensors:   CPU_Mapped model buffer size =   113.30 MiB
+load_tensors:        CUDA0 model buffer size =    60.88 MiB
+load_tensors:        CUDA1 model buffer size =    61.74 MiB
+....................................................................................
+llama_context: constructing llama_context
+llama_context: n_seq_max     = 1
+llama_context: n_ctx         = 2048
+llama_context: n_ctx_seq     = 2048
+llama_context: n_batch       = 2048
+llama_context: n_ubatch      = 512
+llama_context: causal_attn   = 1
+llama_context: flash_attn    = auto
+llama_context: kv_unified    = false
+llama_context: freq_base     = 10000.0
+llama_context: freq_scale    = 1
+llama_context: n_ctx_seq (2048) < n_ctx_train (1048576) -- the full capacity of the model will not be utilized
+llama_context:        CPU  output buffer size =     0.38 MiB
+llama_kv_cache:        CPU KV buffer size =     2.00 MiB
+llama_kv_cache:      CUDA0 KV buffer size =     4.00 MiB
+llama_kv_cache:      CUDA1 KV buffer size =     2.00 MiB
+llama_kv_cache: size =    8.00 MiB (  2048 cells,   4 layers,  1/1 seqs), K (f16):    4.00 MiB, V (f16):    4.00 MiB
+llama_memory_recurrent:        CPU RS buffer size =     8.48 MiB
+llama_memory_recurrent:      CUDA0 RS buffer size =     6.16 MiB
+llama_memory_recurrent:      CUDA1 RS buffer size =     6.93 MiB
+llama_memory_recurrent: size =   21.57 MiB (     1 cells,  32 layers,  1 seqs), R (f32):    0.57 MiB, S (f32):   21.00 MiB
+llama_context: Flash Attention was auto, set to enabled
+llama_context:      CUDA0 compute buffer size =   244.36 MiB
+llama_context:      CUDA1 compute buffer size =    22.39 MiB
+llama_context:  CUDA_Host compute buffer size =    18.34 MiB
+llama_context: graph nodes  = 1815
+llama_context: graph splits = 182 (with bs=512), 41 (with bs=1)
+common_init_from_params: added <|end_of_text|> logit bias = -inf
+common_init_from_params: added <|fim_pad|> logit bias = -inf
+common_init_from_params: setting dry_penalty_last_n to ctx_size = 2048
+common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable)
+
+system_info: n_threads = 16 (n_threads_batch = 16) / 32 | CUDA : ARCHS = 860 | USE_GRAPHS = 1 | PEER_MAX_BATCH_SIZE = 128 | CPU : SSE3 = 1 | SSSE3 = 1 | AVX = 1 | AVX2 = 1 | F16C = 1 | FMA = 1 | BMI2 = 1 | AVX512 = 1 | AVX512_VBMI = 1 | AVX512_VNNI = 1 | AVX512_BF16 = 1 | LLAMAFILE = 1 | OPENMP = 1 | REPACK = 1 | 
+perplexity: tokenizing the input ..
+perplexity: tokenization took 94.428 ms
+perplexity: calculating perplexity over 44 chunks, n_ctx=2048, batch_size=2048, n_seq=1
+perplexity: 0.60 seconds per pass - ETA 0.43 minutes
+[1]423.7369,[2]463.8921,[3]251.0199,[4]159.8058,[5]210.8161,[6]297.7708,[7]273.0106,[8]246.1219,[9]194.0425,[10]168.0539,[11]171.8333,[12]233.8148,[13]269.3656,[14]304.8786,[15]320.2936,[16]295.3445,[17]291.8964,[18]313.4164,[19]323.1056,[20]306.9655,[21]300.7201,[22]312.1789,[23]338.1783,[24]365.7625,[25]396.0509,[26]414.4291,[27]448.9114,[28]473.4455,[29]501.8479,[30]516.6659,[31]521.4756,[32]561.2891,[33]573.4863,[34]593.1837,[35]586.7498,[36]577.2415,[37]571.3231,[38]553.5320,[39]567.5331,[40]556.9408,[41]541.1400,[42]533.1267,[43]534.6802,[44]525.8288,
+Final estimate: PPL = 525.8288 +/- 14.01800
+
+llama_perf_context_print:        load time =     199.26 ms
+llama_perf_context_print: prompt eval time =   14361.38 ms / 90112 tokens (    0.16 ms per token,  6274.61 tokens per second)
+llama_perf_context_print:        eval time =       0.00 ms /     1 runs   (    0.00 ms per token,      inf tokens per second)
+llama_perf_context_print:       total time =   15166.65 ms / 90113 tokens
+llama_perf_context_print:    graphs reused =          0
+llama_memory_breakdown_print: | memory breakdown [MiB] | total    free    self   model   context   compute    unaccounted |
+llama_memory_breakdown_print: |   - CUDA0 (RTX 3090)   | 24115 = 20529 + ( 315 =    60 +      10 +     244) +        3270 |
+llama_memory_breakdown_print: |   - CUDA1 (RTX 3090)   | 24124 = 23405 + (  93 =    61 +       8 +      22) +         625 |
+llama_memory_breakdown_print: |   - Host               |                   142 =   113 +      10 +      18                |

Benchmarks/DataCollection/granite-4.0-h-350m-unsloth-mxfp4_moe-attn_kv_BF16-attn_output_BF16-attn_q_BF16-embeddings_MXFP4-ffn_down_MXFP4-ffn_up_gate_MXFP4/perplexity_general.log

@@ -0,0 +1,190 @@
+ggml_cuda_init: GGML_CUDA_FORCE_MMQ:    no
+ggml_cuda_init: GGML_CUDA_FORCE_CUBLAS: no
+ggml_cuda_init: found 2 CUDA devices:
+  Device 0: NVIDIA GeForce RTX 3090, compute capability 8.6, VMM: yes
+  Device 1: NVIDIA GeForce RTX 3090, compute capability 8.6, VMM: yes
+build: 7040 (92bb442ad) with cc (Ubuntu 13.3.0-6ubuntu2~24.04) 13.3.0 for x86_64-linux-gnu
+llama_model_load_from_file_impl: using device CUDA0 (NVIDIA GeForce RTX 3090) (0000:01:00.0) - 20937 MiB free
+llama_model_load_from_file_impl: using device CUDA1 (NVIDIA GeForce RTX 3090) (0000:03:00.0) - 23581 MiB free
+llama_model_loader: loaded meta data with 48 key-value pairs and 402 tensors from /mnt/world8/AI/ToBench/granite-4.0-h-350m-unsloth/Magic_Quant/GGUF/dc_round1_granite-4.0-h-350m-unsloth-mxfp4_moe-attn_kv_BF16-attn_output_BF16-attn_q_BF16-embeddings_MXFP4-ffn_down_MXFP4-ffn_up_gate_MXFP4.gguf (version GGUF V3 (latest))
+llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output.
+llama_model_loader: - kv   0:                       general.architecture str              = granitehybrid
+llama_model_loader: - kv   1:                               general.type str              = model
+llama_model_loader: - kv   2:                               general.name str              = Granite 4.0 H 350m Unsloth
+llama_model_loader: - kv   3:                           general.finetune str              = unsloth
+llama_model_loader: - kv   4:                           general.basename str              = granite-4.0-h
+llama_model_loader: - kv   5:                         general.size_label str              = 350M
+llama_model_loader: - kv   6:                            general.license str              = apache-2.0
+llama_model_loader: - kv   7:                   general.base_model.count u32              = 1
+llama_model_loader: - kv   8:                  general.base_model.0.name str              = Granite 4.0 H 350m
+llama_model_loader: - kv   9:          general.base_model.0.organization str              = Ibm Granite
+llama_model_loader: - kv  10:              general.base_model.0.repo_url str              = https://huggingface.co/ibm-granite/gr...
+llama_model_loader: - kv  11:                               general.tags arr[str,3]       = ["language", "unsloth", "granite-4.0"]
+llama_model_loader: - kv  12:                  granitehybrid.block_count u32              = 32
+llama_model_loader: - kv  13:               granitehybrid.context_length u32              = 1048576
+llama_model_loader: - kv  14:             granitehybrid.embedding_length u32              = 768
+llama_model_loader: - kv  15:          granitehybrid.feed_forward_length u32              = 2048
+llama_model_loader: - kv  16:         granitehybrid.attention.head_count u32              = 12
+llama_model_loader: - kv  17:      granitehybrid.attention.head_count_kv arr[i32,32]      = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, ...
+llama_model_loader: - kv  18:               granitehybrid.rope.freq_base f32              = 10000.000000
+llama_model_loader: - kv  19: granitehybrid.attention.layer_norm_rms_epsilon f32              = 0.000010
+llama_model_loader: - kv  20:                 granitehybrid.expert_count u32              = 0
+llama_model_loader: - kv  21:            granitehybrid.expert_used_count u32              = 0
+llama_model_loader: - kv  22:                   granitehybrid.vocab_size u32              = 100352
+llama_model_loader: - kv  23:         granitehybrid.rope.dimension_count u32              = 64
+llama_model_loader: - kv  24:              granitehybrid.attention.scale f32              = 0.015625
+llama_model_loader: - kv  25:              granitehybrid.embedding_scale f32              = 12.000000
+llama_model_loader: - kv  26:               granitehybrid.residual_scale f32              = 0.246000
+llama_model_loader: - kv  27:                  granitehybrid.logit_scale f32              = 3.000000
+llama_model_loader: - kv  28: granitehybrid.expert_shared_feed_forward_length u32              = 2048
+llama_model_loader: - kv  29:              granitehybrid.ssm.conv_kernel u32              = 4
+llama_model_loader: - kv  30:               granitehybrid.ssm.state_size u32              = 128
+llama_model_loader: - kv  31:              granitehybrid.ssm.group_count u32              = 1
+llama_model_loader: - kv  32:               granitehybrid.ssm.inner_size u32              = 1536
+llama_model_loader: - kv  33:           granitehybrid.ssm.time_step_rank u32              = 48
+llama_model_loader: - kv  34:       granitehybrid.rope.scaling.finetuned bool             = false
+llama_model_loader: - kv  35:                       tokenizer.ggml.model str              = gpt2
+llama_model_loader: - kv  36:                         tokenizer.ggml.pre str              = dbrx
+llama_model_loader: - kv  37:                      tokenizer.ggml.tokens arr[str,100352]  = ["!", "\"", "#", "$", "%", "&", "'", ...
+llama_model_loader: - kv  38:                  tokenizer.ggml.token_type arr[i32,100352]  = [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...
+llama_model_loader: - kv  39:                      tokenizer.ggml.merges arr[str,100000]  = ["Ġ Ġ", "ĠĠ ĠĠ", "i n", "Ġ t",...
+llama_model_loader: - kv  40:                tokenizer.ggml.bos_token_id u32              = 100257
+llama_model_loader: - kv  41:                tokenizer.ggml.eos_token_id u32              = 100257
+llama_model_loader: - kv  42:            tokenizer.ggml.unknown_token_id u32              = 100269
+llama_model_loader: - kv  43:            tokenizer.ggml.padding_token_id u32              = 100256
+llama_model_loader: - kv  44:               tokenizer.ggml.add_bos_token bool             = false
+llama_model_loader: - kv  45:                    tokenizer.chat_template str              = {%- set tools_system_message_prefix =...
+llama_model_loader: - kv  46:               general.quantization_version u32              = 2
+llama_model_loader: - kv  47:                          general.file_type u32              = 38
+llama_model_loader: - type  f32:  233 tensors
+llama_model_loader: - type q8_0:   56 tensors
+llama_model_loader: - type bf16:   16 tensors
+llama_model_loader: - type mxfp4:   97 tensors
+print_info: file format = GGUF V3 (latest)
+print_info: file type   = MXFP4 MoE
+print_info: file size   = 235.90 MiB (5.81 BPW) 
+load: printing all EOG tokens:
+load:   - 100257 ('<|end_of_text|>')
+load:   - 100261 ('<|fim_pad|>')
+load: special tokens cache size = 96
+load: token to piece cache size = 0.6152 MB
+print_info: arch             = granitehybrid
+print_info: vocab_only       = 0
+print_info: n_ctx_train      = 1048576
+print_info: n_embd           = 768
+print_info: n_embd_inp       = 768
+print_info: n_layer          = 32
+print_info: n_head           = 12
+print_info: n_head_kv        = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0]
+print_info: n_rot            = 64
+print_info: n_swa            = 0
+print_info: is_swa_any       = 0
+print_info: n_embd_head_k    = 64
+print_info: n_embd_head_v    = 64
+print_info: n_gqa            = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 3, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0]
+print_info: n_embd_k_gqa     = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 256, 0, 0, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 0, 0]
+print_info: n_embd_v_gqa     = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 256, 0, 0, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 0, 0]
+print_info: f_norm_eps       = 0.0e+00
+print_info: f_norm_rms_eps   = 1.0e-05
+print_info: f_clamp_kqv      = 0.0e+00
+print_info: f_max_alibi_bias = 0.0e+00
+print_info: f_logit_scale    = 3.0e+00
+print_info: f_attn_scale     = 1.6e-02
+print_info: n_ff             = 2048
+print_info: n_expert         = 0
+print_info: n_expert_used    = 0
+print_info: n_expert_groups  = 0
+print_info: n_group_used     = 0
+print_info: causal attn      = 1
+print_info: pooling type     = 0
+print_info: rope type        = 0
+print_info: rope scaling     = linear
+print_info: freq_base_train  = 10000.0
+print_info: freq_scale_train = 1
+print_info: n_ctx_orig_yarn  = 1048576
+print_info: rope_finetuned   = unknown
+print_info: ssm_d_conv       = 4
+print_info: ssm_d_inner      = 1536
+print_info: ssm_d_state      = 128
+print_info: ssm_dt_rank      = 48
+print_info: ssm_n_group      = 1
+print_info: ssm_dt_b_c_rms   = 0
+print_info: model type       = 350M
+print_info: model params     = 340.33 M
+print_info: general.name     = Granite 4.0 H 350m Unsloth
+print_info: f_embedding_scale = 12.000000
+print_info: f_residual_scale  = 0.246000
+print_info: f_attention_scale = 0.015625
+print_info: n_ff_shexp        = 2048
+print_info: vocab type       = BPE
+print_info: n_vocab          = 100352
+print_info: n_merges         = 100000
+print_info: BOS token        = 100257 '<|end_of_text|>'
+print_info: EOS token        = 100257 '<|end_of_text|>'
+print_info: EOT token        = 100257 '<|end_of_text|>'
+print_info: UNK token        = 100269 '<|unk|>'
+print_info: PAD token        = 100256 '<|pad|>'
+print_info: LF token         = 198 'Ċ'
+print_info: FIM PRE token    = 100258 '<|fim_prefix|>'
+print_info: FIM SUF token    = 100260 '<|fim_suffix|>'
+print_info: FIM MID token    = 100259 '<|fim_middle|>'
+print_info: FIM PAD token    = 100261 '<|fim_pad|>'
+print_info: EOG token        = 100257 '<|end_of_text|>'
+print_info: EOG token        = 100261 '<|fim_pad|>'
+print_info: max token length = 256
+load_tensors: loading model tensors, this can take a while... (mmap = true)
+load_tensors: offloading 20 repeating layers to GPU
+load_tensors: offloaded 20/33 layers to GPU
+load_tensors:   CPU_Mapped model buffer size =   113.30 MiB
+load_tensors:        CUDA0 model buffer size =    60.88 MiB
+load_tensors:        CUDA1 model buffer size =    61.74 MiB
+....................................................................................
+llama_context: constructing llama_context
+llama_context: n_seq_max     = 1
+llama_context: n_ctx         = 2048
+llama_context: n_ctx_seq     = 2048
+llama_context: n_batch       = 2048
+llama_context: n_ubatch      = 512
+llama_context: causal_attn   = 1
+llama_context: flash_attn    = auto
+llama_context: kv_unified    = false
+llama_context: freq_base     = 10000.0
+llama_context: freq_scale    = 1
+llama_context: n_ctx_seq (2048) < n_ctx_train (1048576) -- the full capacity of the model will not be utilized
+llama_context:        CPU  output buffer size =     0.38 MiB
+llama_kv_cache:        CPU KV buffer size =     2.00 MiB
+llama_kv_cache:      CUDA0 KV buffer size =     4.00 MiB
+llama_kv_cache:      CUDA1 KV buffer size =     2.00 MiB
+llama_kv_cache: size =    8.00 MiB (  2048 cells,   4 layers,  1/1 seqs), K (f16):    4.00 MiB, V (f16):    4.00 MiB
+llama_memory_recurrent:        CPU RS buffer size =     8.48 MiB
+llama_memory_recurrent:      CUDA0 RS buffer size =     6.16 MiB
+llama_memory_recurrent:      CUDA1 RS buffer size =     6.93 MiB
+llama_memory_recurrent: size =   21.57 MiB (     1 cells,  32 layers,  1 seqs), R (f32):    0.57 MiB, S (f32):   21.00 MiB
+llama_context: Flash Attention was auto, set to enabled
+llama_context:      CUDA0 compute buffer size =   244.36 MiB
+llama_context:      CUDA1 compute buffer size =    22.39 MiB
+llama_context:  CUDA_Host compute buffer size =    18.34 MiB
+llama_context: graph nodes  = 1815
+llama_context: graph splits = 182 (with bs=512), 41 (with bs=1)
+common_init_from_params: added <|end_of_text|> logit bias = -inf
+common_init_from_params: added <|fim_pad|> logit bias = -inf
+common_init_from_params: setting dry_penalty_last_n to ctx_size = 2048
+common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable)
+
+system_info: n_threads = 16 (n_threads_batch = 16) / 32 | CUDA : ARCHS = 860 | USE_GRAPHS = 1 | PEER_MAX_BATCH_SIZE = 128 | CPU : SSE3 = 1 | SSSE3 = 1 | AVX = 1 | AVX2 = 1 | F16C = 1 | FMA = 1 | BMI2 = 1 | AVX512 = 1 | AVX512_VBMI = 1 | AVX512_VNNI = 1 | AVX512_BF16 = 1 | LLAMAFILE = 1 | OPENMP = 1 | REPACK = 1 | 
+perplexity: tokenizing the input ..
+perplexity: tokenization took 40.438 ms
+perplexity: calculating perplexity over 14 chunks, n_ctx=2048, batch_size=2048, n_seq=1
+perplexity: 0.59 seconds per pass - ETA 0.13 minutes
+[1]453.0437,[2]877.5497,[3]883.5634,[4]761.9689,[5]866.1065,[6]878.7951,[7]1026.7906,[8]1152.6683,[9]1257.3264,[10]1238.7183,[11]1273.4403,[12]1168.7774,[13]1176.6425,[14]1147.1684,
+Final estimate: PPL = 1147.1684 +/- 45.73730
+
+llama_perf_context_print:        load time =     203.69 ms
+llama_perf_context_print: prompt eval time =    4719.56 ms / 28672 tokens (    0.16 ms per token,  6075.15 tokens per second)
+llama_perf_context_print:        eval time =       0.00 ms /     1 runs   (    0.00 ms per token,      inf tokens per second)
+llama_perf_context_print:       total time =    4984.81 ms / 28673 tokens
+llama_perf_context_print:    graphs reused =          0
+llama_memory_breakdown_print: | memory breakdown [MiB] | total    free    self   model   context   compute    unaccounted |
+llama_memory_breakdown_print: |   - CUDA0 (RTX 3090)   | 24115 = 20522 + ( 315 =    60 +      10 +     244) +        3277 |
+llama_memory_breakdown_print: |   - CUDA1 (RTX 3090)   | 24124 = 23405 + (  93 =    61 +       8 +      22) +         625 |
+llama_memory_breakdown_print: |   - Host               |                   142 =   113 +      10 +      18                |

Benchmarks/DataCollection/granite-4.0-h-350m-unsloth-mxfp4_moe-attn_kv_BF16-attn_output_BF16-attn_q_BF16-embeddings_MXFP4-ffn_down_MXFP4-ffn_up_gate_MXFP4/perplexity_math.log

@@ -0,0 +1,190 @@
+ggml_cuda_init: GGML_CUDA_FORCE_MMQ:    no
+ggml_cuda_init: GGML_CUDA_FORCE_CUBLAS: no
+ggml_cuda_init: found 2 CUDA devices:
+  Device 0: NVIDIA GeForce RTX 3090, compute capability 8.6, VMM: yes
+  Device 1: NVIDIA GeForce RTX 3090, compute capability 8.6, VMM: yes
+build: 7040 (92bb442ad) with cc (Ubuntu 13.3.0-6ubuntu2~24.04) 13.3.0 for x86_64-linux-gnu
+llama_model_load_from_file_impl: using device CUDA0 (NVIDIA GeForce RTX 3090) (0000:01:00.0) - 20929 MiB free
+llama_model_load_from_file_impl: using device CUDA1 (NVIDIA GeForce RTX 3090) (0000:03:00.0) - 23581 MiB free
+llama_model_loader: loaded meta data with 48 key-value pairs and 402 tensors from /mnt/world8/AI/ToBench/granite-4.0-h-350m-unsloth/Magic_Quant/GGUF/dc_round1_granite-4.0-h-350m-unsloth-mxfp4_moe-attn_kv_BF16-attn_output_BF16-attn_q_BF16-embeddings_MXFP4-ffn_down_MXFP4-ffn_up_gate_MXFP4.gguf (version GGUF V3 (latest))
+llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output.
+llama_model_loader: - kv   0:                       general.architecture str              = granitehybrid
+llama_model_loader: - kv   1:                               general.type str              = model
+llama_model_loader: - kv   2:                               general.name str              = Granite 4.0 H 350m Unsloth
+llama_model_loader: - kv   3:                           general.finetune str              = unsloth
+llama_model_loader: - kv   4:                           general.basename str              = granite-4.0-h
+llama_model_loader: - kv   5:                         general.size_label str              = 350M
+llama_model_loader: - kv   6:                            general.license str              = apache-2.0
+llama_model_loader: - kv   7:                   general.base_model.count u32              = 1
+llama_model_loader: - kv   8:                  general.base_model.0.name str              = Granite 4.0 H 350m
+llama_model_loader: - kv   9:          general.base_model.0.organization str              = Ibm Granite
+llama_model_loader: - kv  10:              general.base_model.0.repo_url str              = https://huggingface.co/ibm-granite/gr...
+llama_model_loader: - kv  11:                               general.tags arr[str,3]       = ["language", "unsloth", "granite-4.0"]
+llama_model_loader: - kv  12:                  granitehybrid.block_count u32              = 32
+llama_model_loader: - kv  13:               granitehybrid.context_length u32              = 1048576
+llama_model_loader: - kv  14:             granitehybrid.embedding_length u32              = 768
+llama_model_loader: - kv  15:          granitehybrid.feed_forward_length u32              = 2048
+llama_model_loader: - kv  16:         granitehybrid.attention.head_count u32              = 12
+llama_model_loader: - kv  17:      granitehybrid.attention.head_count_kv arr[i32,32]      = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, ...
+llama_model_loader: - kv  18:               granitehybrid.rope.freq_base f32              = 10000.000000
+llama_model_loader: - kv  19: granitehybrid.attention.layer_norm_rms_epsilon f32              = 0.000010
+llama_model_loader: - kv  20:                 granitehybrid.expert_count u32              = 0
+llama_model_loader: - kv  21:            granitehybrid.expert_used_count u32              = 0
+llama_model_loader: - kv  22:                   granitehybrid.vocab_size u32              = 100352
+llama_model_loader: - kv  23:         granitehybrid.rope.dimension_count u32              = 64
+llama_model_loader: - kv  24:              granitehybrid.attention.scale f32              = 0.015625
+llama_model_loader: - kv  25:              granitehybrid.embedding_scale f32              = 12.000000
+llama_model_loader: - kv  26:               granitehybrid.residual_scale f32              = 0.246000
+llama_model_loader: - kv  27:                  granitehybrid.logit_scale f32              = 3.000000
+llama_model_loader: - kv  28: granitehybrid.expert_shared_feed_forward_length u32              = 2048
+llama_model_loader: - kv  29:              granitehybrid.ssm.conv_kernel u32              = 4
+llama_model_loader: - kv  30:               granitehybrid.ssm.state_size u32              = 128
+llama_model_loader: - kv  31:              granitehybrid.ssm.group_count u32              = 1
+llama_model_loader: - kv  32:               granitehybrid.ssm.inner_size u32              = 1536
+llama_model_loader: - kv  33:           granitehybrid.ssm.time_step_rank u32              = 48
+llama_model_loader: - kv  34:       granitehybrid.rope.scaling.finetuned bool             = false
+llama_model_loader: - kv  35:                       tokenizer.ggml.model str              = gpt2
+llama_model_loader: - kv  36:                         tokenizer.ggml.pre str              = dbrx
+llama_model_loader: - kv  37:                      tokenizer.ggml.tokens arr[str,100352]  = ["!", "\"", "#", "$", "%", "&", "'", ...
+llama_model_loader: - kv  38:                  tokenizer.ggml.token_type arr[i32,100352]  = [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...
+llama_model_loader: - kv  39:                      tokenizer.ggml.merges arr[str,100000]  = ["Ġ Ġ", "ĠĠ ĠĠ", "i n", "Ġ t",...
+llama_model_loader: - kv  40:                tokenizer.ggml.bos_token_id u32              = 100257
+llama_model_loader: - kv  41:                tokenizer.ggml.eos_token_id u32              = 100257
+llama_model_loader: - kv  42:            tokenizer.ggml.unknown_token_id u32              = 100269
+llama_model_loader: - kv  43:            tokenizer.ggml.padding_token_id u32              = 100256
+llama_model_loader: - kv  44:               tokenizer.ggml.add_bos_token bool             = false
+llama_model_loader: - kv  45:                    tokenizer.chat_template str              = {%- set tools_system_message_prefix =...
+llama_model_loader: - kv  46:               general.quantization_version u32              = 2
+llama_model_loader: - kv  47:                          general.file_type u32              = 38
+llama_model_loader: - type  f32:  233 tensors
+llama_model_loader: - type q8_0:   56 tensors
+llama_model_loader: - type bf16:   16 tensors
+llama_model_loader: - type mxfp4:   97 tensors
+print_info: file format = GGUF V3 (latest)
+print_info: file type   = MXFP4 MoE
+print_info: file size   = 235.90 MiB (5.81 BPW) 
+load: printing all EOG tokens:
+load:   - 100257 ('<|end_of_text|>')
+load:   - 100261 ('<|fim_pad|>')
+load: special tokens cache size = 96
+load: token to piece cache size = 0.6152 MB
+print_info: arch             = granitehybrid
+print_info: vocab_only       = 0
+print_info: n_ctx_train      = 1048576
+print_info: n_embd           = 768
+print_info: n_embd_inp       = 768
+print_info: n_layer          = 32
+print_info: n_head           = 12
+print_info: n_head_kv        = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0]
+print_info: n_rot            = 64
+print_info: n_swa            = 0
+print_info: is_swa_any       = 0
+print_info: n_embd_head_k    = 64
+print_info: n_embd_head_v    = 64
+print_info: n_gqa            = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 3, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0]
+print_info: n_embd_k_gqa     = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 256, 0, 0, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 0, 0]
+print_info: n_embd_v_gqa     = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 256, 0, 0, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 0, 0]
+print_info: f_norm_eps       = 0.0e+00
+print_info: f_norm_rms_eps   = 1.0e-05
+print_info: f_clamp_kqv      = 0.0e+00
+print_info: f_max_alibi_bias = 0.0e+00
+print_info: f_logit_scale    = 3.0e+00
+print_info: f_attn_scale     = 1.6e-02
+print_info: n_ff             = 2048
+print_info: n_expert         = 0
+print_info: n_expert_used    = 0
+print_info: n_expert_groups  = 0
+print_info: n_group_used     = 0
+print_info: causal attn      = 1
+print_info: pooling type     = 0
+print_info: rope type        = 0
+print_info: rope scaling     = linear
+print_info: freq_base_train  = 10000.0
+print_info: freq_scale_train = 1
+print_info: n_ctx_orig_yarn  = 1048576
+print_info: rope_finetuned   = unknown
+print_info: ssm_d_conv       = 4
+print_info: ssm_d_inner      = 1536
+print_info: ssm_d_state      = 128
+print_info: ssm_dt_rank      = 48
+print_info: ssm_n_group      = 1
+print_info: ssm_dt_b_c_rms   = 0
+print_info: model type       = 350M
+print_info: model params     = 340.33 M
+print_info: general.name     = Granite 4.0 H 350m Unsloth
+print_info: f_embedding_scale = 12.000000
+print_info: f_residual_scale  = 0.246000
+print_info: f_attention_scale = 0.015625
+print_info: n_ff_shexp        = 2048
+print_info: vocab type       = BPE
+print_info: n_vocab          = 100352
+print_info: n_merges         = 100000
+print_info: BOS token        = 100257 '<|end_of_text|>'
+print_info: EOS token        = 100257 '<|end_of_text|>'
+print_info: EOT token        = 100257 '<|end_of_text|>'
+print_info: UNK token        = 100269 '<|unk|>'
+print_info: PAD token        = 100256 '<|pad|>'
+print_info: LF token         = 198 'Ċ'
+print_info: FIM PRE token    = 100258 '<|fim_prefix|>'
+print_info: FIM SUF token    = 100260 '<|fim_suffix|>'
+print_info: FIM MID token    = 100259 '<|fim_middle|>'
+print_info: FIM PAD token    = 100261 '<|fim_pad|>'
+print_info: EOG token        = 100257 '<|end_of_text|>'
+print_info: EOG token        = 100261 '<|fim_pad|>'
+print_info: max token length = 256
+load_tensors: loading model tensors, this can take a while... (mmap = true)
+load_tensors: offloading 20 repeating layers to GPU
+load_tensors: offloaded 20/33 layers to GPU
+load_tensors:   CPU_Mapped model buffer size =   113.30 MiB
+load_tensors:        CUDA0 model buffer size =    60.88 MiB
+load_tensors:        CUDA1 model buffer size =    61.74 MiB
+....................................................................................
+llama_context: constructing llama_context
+llama_context: n_seq_max     = 1
+llama_context: n_ctx         = 2048
+llama_context: n_ctx_seq     = 2048
+llama_context: n_batch       = 2048
+llama_context: n_ubatch      = 512
+llama_context: causal_attn   = 1
+llama_context: flash_attn    = auto
+llama_context: kv_unified    = false
+llama_context: freq_base     = 10000.0
+llama_context: freq_scale    = 1
+llama_context: n_ctx_seq (2048) < n_ctx_train (1048576) -- the full capacity of the model will not be utilized
+llama_context:        CPU  output buffer size =     0.38 MiB
+llama_kv_cache:        CPU KV buffer size =     2.00 MiB
+llama_kv_cache:      CUDA0 KV buffer size =     4.00 MiB
+llama_kv_cache:      CUDA1 KV buffer size =     2.00 MiB
+llama_kv_cache: size =    8.00 MiB (  2048 cells,   4 layers,  1/1 seqs), K (f16):    4.00 MiB, V (f16):    4.00 MiB
+llama_memory_recurrent:        CPU RS buffer size =     8.48 MiB
+llama_memory_recurrent:      CUDA0 RS buffer size =     6.16 MiB
+llama_memory_recurrent:      CUDA1 RS buffer size =     6.93 MiB
+llama_memory_recurrent: size =   21.57 MiB (     1 cells,  32 layers,  1 seqs), R (f32):    0.57 MiB, S (f32):   21.00 MiB
+llama_context: Flash Attention was auto, set to enabled
+llama_context:      CUDA0 compute buffer size =   244.36 MiB
+llama_context:      CUDA1 compute buffer size =    22.39 MiB
+llama_context:  CUDA_Host compute buffer size =    18.34 MiB
+llama_context: graph nodes  = 1815
+llama_context: graph splits = 182 (with bs=512), 41 (with bs=1)
+common_init_from_params: added <|end_of_text|> logit bias = -inf
+common_init_from_params: added <|fim_pad|> logit bias = -inf
+common_init_from_params: setting dry_penalty_last_n to ctx_size = 2048
+common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable)
+
+system_info: n_threads = 16 (n_threads_batch = 16) / 32 | CUDA : ARCHS = 860 | USE_GRAPHS = 1 | PEER_MAX_BATCH_SIZE = 128 | CPU : SSE3 = 1 | SSSE3 = 1 | AVX = 1 | AVX2 = 1 | F16C = 1 | FMA = 1 | BMI2 = 1 | AVX512 = 1 | AVX512_VBMI = 1 | AVX512_VNNI = 1 | AVX512_BF16 = 1 | LLAMAFILE = 1 | OPENMP = 1 | REPACK = 1 | 
+perplexity: tokenizing the input ..
+perplexity: tokenization took 33.939 ms
+perplexity: calculating perplexity over 15 chunks, n_ctx=2048, batch_size=2048, n_seq=1
+perplexity: 0.56 seconds per pass - ETA 0.13 minutes
+[1]187.5569,[2]219.8043,[3]194.0576,[4]181.2363,[5]223.8031,[6]224.3280,[7]197.5362,[8]176.9783,[9]179.7542,[10]197.3073,[11]200.3855,[12]200.4161,[13]209.6261,[14]201.9376,[15]197.3954,
+Final estimate: PPL = 197.3954 +/- 6.91272
+
+llama_perf_context_print:        load time =     196.92 ms
+llama_perf_context_print: prompt eval time =    5060.48 ms / 30720 tokens (    0.16 ms per token,  6070.57 tokens per second)
+llama_perf_context_print:        eval time =       0.00 ms /     1 runs   (    0.00 ms per token,      inf tokens per second)
+llama_perf_context_print:       total time =    5340.57 ms / 30721 tokens
+llama_perf_context_print:    graphs reused =          0
+llama_memory_breakdown_print: | memory breakdown [MiB] | total    free    self   model   context   compute    unaccounted |
+llama_memory_breakdown_print: |   - CUDA0 (RTX 3090)   | 24115 = 20529 + ( 315 =    60 +      10 +     244) +        3270 |
+llama_memory_breakdown_print: |   - CUDA1 (RTX 3090)   | 24124 = 23405 + (  93 =    61 +       8 +      22) +         625 |
+llama_memory_breakdown_print: |   - Host               |                   142 =   113 +      10 +      18                |

Benchmarks/DataCollection/granite-4.0-h-350m-unsloth-mxfp4_moe-attn_kv_BF16-attn_output_BF16-attn_q_BF16-embeddings_MXFP4-ffn_down_MXFP4-ffn_up_gate_Q8_0/bench_metrics.json

@@ -0,0 +1,44 @@
+{
+  "raw_metrics": {
+    "llamabench": {
+      "backend": "CUDA",
+      "log_path": "granite-4.0-h-350m-unsloth-mxfp4_moe-attn_kv_BF16-attn_output_BF16-attn_q_BF16-embeddings_MXFP4-ffn_down_MXFP4-ffn_up_gate_Q8_0/llamabench.md",
+      "ngl": "35",
+      "raw_row": {
+        "backend": "CUDA",
+        "model": "granitehybrid 350M MXFP4 MoE",
+        "ngl": "35",
+        "params": "340.33 M",
+        "size": "286.90 MiB",
+        "t/s": "1670.78 \u00b1 46.40",
+        "test": "pp8",
+        "tps_value": 1670.78
+      },
+      "test": "pp8",
+      "tps": 1670.78
+    },
+    "perplexity": {
+      "code": {
+        "log_path": "granite-4.0-h-350m-unsloth-mxfp4_moe-attn_kv_BF16-attn_output_BF16-attn_q_BF16-embeddings_MXFP4-ffn_down_MXFP4-ffn_up_gate_Q8_0/perplexity_code.log",
+        "ppl": 2.9192,
+        "ppl_error": 0.03384
+      },
+      "general": {
+        "log_path": "granite-4.0-h-350m-unsloth-mxfp4_moe-attn_kv_BF16-attn_output_BF16-attn_q_BF16-embeddings_MXFP4-ffn_down_MXFP4-ffn_up_gate_Q8_0/perplexity_general.log",
+        "ppl": 38.5296,
+        "ppl_error": 1.10574
+      },
+      "math": {
+        "log_path": "granite-4.0-h-350m-unsloth-mxfp4_moe-attn_kv_BF16-attn_output_BF16-attn_q_BF16-embeddings_MXFP4-ffn_down_MXFP4-ffn_up_gate_Q8_0/perplexity_math.log",
+        "ppl": 28.6447,
+        "ppl_error": 0.77572
+      }
+    }
+  },
+  "summary": {
+    "avg_prec_loss_pct": 113.42,
+    "bench_tps": 1670.78,
+    "file_size_bytes": 304411584,
+    "file_size_gb": 0.28
+  }
+}

Benchmarks/DataCollection/granite-4.0-h-350m-unsloth-mxfp4_moe-attn_kv_BF16-attn_output_BF16-attn_q_BF16-embeddings_MXFP4-ffn_down_MXFP4-ffn_up_gate_Q8_0/llamabench.md

@@ -0,0 +1,11 @@
+ggml_cuda_init: GGML_CUDA_FORCE_MMQ:    no
+ggml_cuda_init: GGML_CUDA_FORCE_CUBLAS: no
+ggml_cuda_init: found 2 CUDA devices:
+  Device 0: NVIDIA GeForce RTX 3090, compute capability 8.6, VMM: yes
+  Device 1: NVIDIA GeForce RTX 3090, compute capability 8.6, VMM: yes
+| model                          |       size |     params | backend    | ngl |            test |                  t/s |
+| ------------------------------ | ---------: | ---------: | ---------- | --: | --------------: | -------------------: |
+| granitehybrid 350M MXFP4 MoE   | 286.90 MiB |   340.33 M | CUDA       |  35 |             pp8 |      1670.78 ± 46.40 |
+| granitehybrid 350M MXFP4 MoE   | 286.90 MiB |   340.33 M | CUDA       |  35 |           tg128 |       330.39 ± 10.80 |
+
+build: 92bb442ad (7040)

Benchmarks/DataCollection/granite-4.0-h-350m-unsloth-mxfp4_moe-attn_kv_BF16-attn_output_BF16-attn_q_BF16-embeddings_MXFP4-ffn_down_MXFP4-ffn_up_gate_Q8_0/perplexity_code.log

@@ -0,0 +1,190 @@
+ggml_cuda_init: GGML_CUDA_FORCE_MMQ:    no
+ggml_cuda_init: GGML_CUDA_FORCE_CUBLAS: no
+ggml_cuda_init: found 2 CUDA devices:
+  Device 0: NVIDIA GeForce RTX 3090, compute capability 8.6, VMM: yes
+  Device 1: NVIDIA GeForce RTX 3090, compute capability 8.6, VMM: yes
+build: 7040 (92bb442ad) with cc (Ubuntu 13.3.0-6ubuntu2~24.04) 13.3.0 for x86_64-linux-gnu
+llama_model_load_from_file_impl: using device CUDA0 (NVIDIA GeForce RTX 3090) (0000:01:00.0) - 20970 MiB free
+llama_model_load_from_file_impl: using device CUDA1 (NVIDIA GeForce RTX 3090) (0000:03:00.0) - 23581 MiB free
+llama_model_loader: loaded meta data with 48 key-value pairs and 402 tensors from /mnt/world8/AI/ToBench/granite-4.0-h-350m-unsloth/Magic_Quant/GGUF/dc_round0_granite-4.0-h-350m-unsloth-mxfp4_moe-attn_kv_BF16-attn_output_BF16-attn_q_BF16-embeddings_MXFP4-ffn_down_MXFP4-ffn_up_gate_Q8_0.gguf (version GGUF V3 (latest))
+llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output.
+llama_model_loader: - kv   0:                       general.architecture str              = granitehybrid
+llama_model_loader: - kv   1:                               general.type str              = model
+llama_model_loader: - kv   2:                               general.name str              = Granite 4.0 H 350m Unsloth
+llama_model_loader: - kv   3:                           general.finetune str              = unsloth
+llama_model_loader: - kv   4:                           general.basename str              = granite-4.0-h
+llama_model_loader: - kv   5:                         general.size_label str              = 350M
+llama_model_loader: - kv   6:                            general.license str              = apache-2.0
+llama_model_loader: - kv   7:                   general.base_model.count u32              = 1
+llama_model_loader: - kv   8:                  general.base_model.0.name str              = Granite 4.0 H 350m
+llama_model_loader: - kv   9:          general.base_model.0.organization str              = Ibm Granite
+llama_model_loader: - kv  10:              general.base_model.0.repo_url str              = https://huggingface.co/ibm-granite/gr...
+llama_model_loader: - kv  11:                               general.tags arr[str,3]       = ["language", "unsloth", "granite-4.0"]
+llama_model_loader: - kv  12:                  granitehybrid.block_count u32              = 32
+llama_model_loader: - kv  13:               granitehybrid.context_length u32              = 1048576
+llama_model_loader: - kv  14:             granitehybrid.embedding_length u32              = 768
+llama_model_loader: - kv  15:          granitehybrid.feed_forward_length u32              = 2048
+llama_model_loader: - kv  16:         granitehybrid.attention.head_count u32              = 12
+llama_model_loader: - kv  17:      granitehybrid.attention.head_count_kv arr[i32,32]      = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, ...
+llama_model_loader: - kv  18:               granitehybrid.rope.freq_base f32              = 10000.000000
+llama_model_loader: - kv  19: granitehybrid.attention.layer_norm_rms_epsilon f32              = 0.000010
+llama_model_loader: - kv  20:                 granitehybrid.expert_count u32              = 0
+llama_model_loader: - kv  21:            granitehybrid.expert_used_count u32              = 0
+llama_model_loader: - kv  22:                   granitehybrid.vocab_size u32              = 100352
+llama_model_loader: - kv  23:         granitehybrid.rope.dimension_count u32              = 64
+llama_model_loader: - kv  24:              granitehybrid.attention.scale f32              = 0.015625
+llama_model_loader: - kv  25:              granitehybrid.embedding_scale f32              = 12.000000
+llama_model_loader: - kv  26:               granitehybrid.residual_scale f32              = 0.246000
+llama_model_loader: - kv  27:                  granitehybrid.logit_scale f32              = 3.000000
+llama_model_loader: - kv  28: granitehybrid.expert_shared_feed_forward_length u32              = 2048
+llama_model_loader: - kv  29:              granitehybrid.ssm.conv_kernel u32              = 4
+llama_model_loader: - kv  30:               granitehybrid.ssm.state_size u32              = 128
+llama_model_loader: - kv  31:              granitehybrid.ssm.group_count u32              = 1
+llama_model_loader: - kv  32:               granitehybrid.ssm.inner_size u32              = 1536
+llama_model_loader: - kv  33:           granitehybrid.ssm.time_step_rank u32              = 48
+llama_model_loader: - kv  34:       granitehybrid.rope.scaling.finetuned bool             = false
+llama_model_loader: - kv  35:                       tokenizer.ggml.model str              = gpt2
+llama_model_loader: - kv  36:                         tokenizer.ggml.pre str              = dbrx
+llama_model_loader: - kv  37:                      tokenizer.ggml.tokens arr[str,100352]  = ["!", "\"", "#", "$", "%", "&", "'", ...
+llama_model_loader: - kv  38:                  tokenizer.ggml.token_type arr[i32,100352]  = [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...
+llama_model_loader: - kv  39:                      tokenizer.ggml.merges arr[str,100000]  = ["Ġ Ġ", "ĠĠ ĠĠ", "i n", "Ġ t",...
+llama_model_loader: - kv  40:                tokenizer.ggml.bos_token_id u32              = 100257
+llama_model_loader: - kv  41:                tokenizer.ggml.eos_token_id u32              = 100257
+llama_model_loader: - kv  42:            tokenizer.ggml.unknown_token_id u32              = 100269
+llama_model_loader: - kv  43:            tokenizer.ggml.padding_token_id u32              = 100256
+llama_model_loader: - kv  44:               tokenizer.ggml.add_bos_token bool             = false
+llama_model_loader: - kv  45:                    tokenizer.chat_template str              = {%- set tools_system_message_prefix =...
+llama_model_loader: - kv  46:               general.quantization_version u32              = 2
+llama_model_loader: - kv  47:                          general.file_type u32              = 38
+llama_model_loader: - type  f32:  233 tensors
+llama_model_loader: - type q8_0:  120 tensors
+llama_model_loader: - type bf16:   16 tensors
+llama_model_loader: - type mxfp4:   33 tensors
+print_info: file format = GGUF V3 (latest)
+print_info: file type   = MXFP4 MoE
+print_info: file size   = 286.90 MiB (7.07 BPW) 
+load: printing all EOG tokens:
+load:   - 100257 ('<|end_of_text|>')
+load:   - 100261 ('<|fim_pad|>')
+load: special tokens cache size = 96
+load: token to piece cache size = 0.6152 MB
+print_info: arch             = granitehybrid
+print_info: vocab_only       = 0
+print_info: n_ctx_train      = 1048576
+print_info: n_embd           = 768
+print_info: n_embd_inp       = 768
+print_info: n_layer          = 32
+print_info: n_head           = 12
+print_info: n_head_kv        = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0]
+print_info: n_rot            = 64
+print_info: n_swa            = 0
+print_info: is_swa_any       = 0
+print_info: n_embd_head_k    = 64
+print_info: n_embd_head_v    = 64
+print_info: n_gqa            = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 3, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0]
+print_info: n_embd_k_gqa     = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 256, 0, 0, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 0, 0]
+print_info: n_embd_v_gqa     = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 256, 0, 0, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 0, 0]
+print_info: f_norm_eps       = 0.0e+00
+print_info: f_norm_rms_eps   = 1.0e-05
+print_info: f_clamp_kqv      = 0.0e+00
+print_info: f_max_alibi_bias = 0.0e+00
+print_info: f_logit_scale    = 3.0e+00
+print_info: f_attn_scale     = 1.6e-02
+print_info: n_ff             = 2048
+print_info: n_expert         = 0
+print_info: n_expert_used    = 0
+print_info: n_expert_groups  = 0
+print_info: n_group_used     = 0
+print_info: causal attn      = 1
+print_info: pooling type     = 0
+print_info: rope type        = 0
+print_info: rope scaling     = linear
+print_info: freq_base_train  = 10000.0
+print_info: freq_scale_train = 1
+print_info: n_ctx_orig_yarn  = 1048576
+print_info: rope_finetuned   = unknown
+print_info: ssm_d_conv       = 4
+print_info: ssm_d_inner      = 1536
+print_info: ssm_d_state      = 128
+print_info: ssm_dt_rank      = 48
+print_info: ssm_n_group      = 1
+print_info: ssm_dt_b_c_rms   = 0
+print_info: model type       = 350M
+print_info: model params     = 340.33 M
+print_info: general.name     = Granite 4.0 H 350m Unsloth
+print_info: f_embedding_scale = 12.000000
+print_info: f_residual_scale  = 0.246000
+print_info: f_attention_scale = 0.015625
+print_info: n_ff_shexp        = 2048
+print_info: vocab type       = BPE
+print_info: n_vocab          = 100352
+print_info: n_merges         = 100000
+print_info: BOS token        = 100257 '<|end_of_text|>'
+print_info: EOS token        = 100257 '<|end_of_text|>'
+print_info: EOT token        = 100257 '<|end_of_text|>'
+print_info: UNK token        = 100269 '<|unk|>'
+print_info: PAD token        = 100256 '<|pad|>'
+print_info: LF token         = 198 'Ċ'
+print_info: FIM PRE token    = 100258 '<|fim_prefix|>'
+print_info: FIM SUF token    = 100260 '<|fim_suffix|>'
+print_info: FIM MID token    = 100259 '<|fim_middle|>'
+print_info: FIM PAD token    = 100261 '<|fim_pad|>'
+print_info: EOG token        = 100257 '<|end_of_text|>'
+print_info: EOG token        = 100261 '<|fim_pad|>'
+print_info: max token length = 256
+load_tensors: loading model tensors, this can take a while... (mmap = true)
+load_tensors: offloading 20 repeating layers to GPU
+load_tensors: offloaded 20/33 layers to GPU
+load_tensors:   CPU_Mapped model buffer size =   132.43 MiB
+load_tensors:        CUDA0 model buffer size =    76.82 MiB
+load_tensors:        CUDA1 model buffer size =    77.68 MiB
+........................................................................................
+llama_context: constructing llama_context
+llama_context: n_seq_max     = 1
+llama_context: n_ctx         = 2048
+llama_context: n_ctx_seq     = 2048
+llama_context: n_batch       = 2048
+llama_context: n_ubatch      = 512
+llama_context: causal_attn   = 1
+llama_context: flash_attn    = auto
+llama_context: kv_unified    = false
+llama_context: freq_base     = 10000.0
+llama_context: freq_scale    = 1
+llama_context: n_ctx_seq (2048) < n_ctx_train (1048576) -- the full capacity of the model will not be utilized
+llama_context:        CPU  output buffer size =     0.38 MiB
+llama_kv_cache:        CPU KV buffer size =     2.00 MiB
+llama_kv_cache:      CUDA0 KV buffer size =     4.00 MiB
+llama_kv_cache:      CUDA1 KV buffer size =     2.00 MiB
+llama_kv_cache: size =    8.00 MiB (  2048 cells,   4 layers,  1/1 seqs), K (f16):    4.00 MiB, V (f16):    4.00 MiB
+llama_memory_recurrent:        CPU RS buffer size =     8.48 MiB
+llama_memory_recurrent:      CUDA0 RS buffer size =     6.16 MiB
+llama_memory_recurrent:      CUDA1 RS buffer size =     6.93 MiB
+llama_memory_recurrent: size =   21.57 MiB (     1 cells,  32 layers,  1 seqs), R (f32):    0.57 MiB, S (f32):   21.00 MiB
+llama_context: Flash Attention was auto, set to enabled
+llama_context:      CUDA0 compute buffer size =   245.16 MiB
+llama_context:      CUDA1 compute buffer size =    22.39 MiB
+llama_context:  CUDA_Host compute buffer size =    18.34 MiB
+llama_context: graph nodes  = 1815
+llama_context: graph splits = 182 (with bs=512), 41 (with bs=1)
+common_init_from_params: added <|end_of_text|> logit bias = -inf
+common_init_from_params: added <|fim_pad|> logit bias = -inf
+common_init_from_params: setting dry_penalty_last_n to ctx_size = 2048
+common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable)
+
+system_info: n_threads = 16 (n_threads_batch = 16) / 32 | CUDA : ARCHS = 860 | USE_GRAPHS = 1 | PEER_MAX_BATCH_SIZE = 128 | CPU : SSE3 = 1 | SSSE3 = 1 | AVX = 1 | AVX2 = 1 | F16C = 1 | FMA = 1 | BMI2 = 1 | AVX512 = 1 | AVX512_VBMI = 1 | AVX512_VNNI = 1 | AVX512_BF16 = 1 | LLAMAFILE = 1 | OPENMP = 1 | REPACK = 1 | 
+perplexity: tokenizing the input ..
+perplexity: tokenization took 96.4 ms
+perplexity: calculating perplexity over 44 chunks, n_ctx=2048, batch_size=2048, n_seq=1
+perplexity: 0.57 seconds per pass - ETA 0.40 minutes
+[1]8.0593,[2]7.9795,[3]4.3196,[4]3.8159,[5]4.5867,[6]5.2062,[7]4.7698,[8]4.2420,[9]3.7184,[10]3.3188,[11]3.2609,[12]3.2922,[13]3.1011,[14]3.0448,[15]3.1506,[16]3.0063,[17]2.9327,[18]2.9610,[19]2.8765,[20]2.8071,[21]2.7559,[22]2.7215,[23]2.8159,[24]2.7771,[25]2.8121,[26]2.7537,[27]2.7353,[28]2.7203,[29]2.8187,[30]2.8588,[31]2.8487,[32]2.8122,[33]2.8454,[34]2.8287,[35]2.7895,[36]2.8394,[37]2.8486,[38]2.8396,[39]2.8891,[40]2.9088,[41]2.8947,[42]2.9436,[43]2.9459,[44]2.9192,
+Final estimate: PPL = 2.9192 +/- 0.03384
+
+llama_perf_context_print:        load time =     200.81 ms
+llama_perf_context_print: prompt eval time =   14335.06 ms / 90112 tokens (    0.16 ms per token,  6286.13 tokens per second)
+llama_perf_context_print:        eval time =       0.00 ms /     1 runs   (    0.00 ms per token,      inf tokens per second)
+llama_perf_context_print:       total time =   15131.42 ms / 90113 tokens
+llama_perf_context_print:    graphs reused =          0
+llama_memory_breakdown_print: | memory breakdown [MiB] | total    free    self   model   context   compute    unaccounted |
+llama_memory_breakdown_print: |   - CUDA0 (RTX 3090)   | 24115 = 20554 + ( 332 =    76 +      10 +     245) +        3228 |
+llama_memory_breakdown_print: |   - CUDA1 (RTX 3090)   | 24124 = 23389 + ( 109 =    77 +       8 +      22) +         625 |
+llama_memory_breakdown_print: |   - Host               |                   161 =   132 +      10 +      18                |

Benchmarks/DataCollection/granite-4.0-h-350m-unsloth-mxfp4_moe-attn_kv_BF16-attn_output_BF16-attn_q_BF16-embeddings_MXFP4-ffn_down_MXFP4-ffn_up_gate_Q8_0/perplexity_general.log

@@ -0,0 +1,190 @@
+ggml_cuda_init: GGML_CUDA_FORCE_MMQ:    no
+ggml_cuda_init: GGML_CUDA_FORCE_CUBLAS: no
+ggml_cuda_init: found 2 CUDA devices:
+  Device 0: NVIDIA GeForce RTX 3090, compute capability 8.6, VMM: yes
+  Device 1: NVIDIA GeForce RTX 3090, compute capability 8.6, VMM: yes
+build: 7040 (92bb442ad) with cc (Ubuntu 13.3.0-6ubuntu2~24.04) 13.3.0 for x86_64-linux-gnu
+llama_model_load_from_file_impl: using device CUDA0 (NVIDIA GeForce RTX 3090) (0000:01:00.0) - 20967 MiB free
+llama_model_load_from_file_impl: using device CUDA1 (NVIDIA GeForce RTX 3090) (0000:03:00.0) - 23581 MiB free
+llama_model_loader: loaded meta data with 48 key-value pairs and 402 tensors from /mnt/world8/AI/ToBench/granite-4.0-h-350m-unsloth/Magic_Quant/GGUF/dc_round0_granite-4.0-h-350m-unsloth-mxfp4_moe-attn_kv_BF16-attn_output_BF16-attn_q_BF16-embeddings_MXFP4-ffn_down_MXFP4-ffn_up_gate_Q8_0.gguf (version GGUF V3 (latest))
+llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output.
+llama_model_loader: - kv   0:                       general.architecture str              = granitehybrid
+llama_model_loader: - kv   1:                               general.type str              = model
+llama_model_loader: - kv   2:                               general.name str              = Granite 4.0 H 350m Unsloth
+llama_model_loader: - kv   3:                           general.finetune str              = unsloth
+llama_model_loader: - kv   4:                           general.basename str              = granite-4.0-h
+llama_model_loader: - kv   5:                         general.size_label str              = 350M
+llama_model_loader: - kv   6:                            general.license str              = apache-2.0
+llama_model_loader: - kv   7:                   general.base_model.count u32              = 1
+llama_model_loader: - kv   8:                  general.base_model.0.name str              = Granite 4.0 H 350m
+llama_model_loader: - kv   9:          general.base_model.0.organization str              = Ibm Granite
+llama_model_loader: - kv  10:              general.base_model.0.repo_url str              = https://huggingface.co/ibm-granite/gr...
+llama_model_loader: - kv  11:                               general.tags arr[str,3]       = ["language", "unsloth", "granite-4.0"]
+llama_model_loader: - kv  12:                  granitehybrid.block_count u32              = 32
+llama_model_loader: - kv  13:               granitehybrid.context_length u32              = 1048576
+llama_model_loader: - kv  14:             granitehybrid.embedding_length u32              = 768
+llama_model_loader: - kv  15:          granitehybrid.feed_forward_length u32              = 2048
+llama_model_loader: - kv  16:         granitehybrid.attention.head_count u32              = 12
+llama_model_loader: - kv  17:      granitehybrid.attention.head_count_kv arr[i32,32]      = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, ...
+llama_model_loader: - kv  18:               granitehybrid.rope.freq_base f32              = 10000.000000
+llama_model_loader: - kv  19: granitehybrid.attention.layer_norm_rms_epsilon f32              = 0.000010
+llama_model_loader: - kv  20:                 granitehybrid.expert_count u32              = 0
+llama_model_loader: - kv  21:            granitehybrid.expert_used_count u32              = 0
+llama_model_loader: - kv  22:                   granitehybrid.vocab_size u32              = 100352
+llama_model_loader: - kv  23:         granitehybrid.rope.dimension_count u32              = 64
+llama_model_loader: - kv  24:              granitehybrid.attention.scale f32              = 0.015625
+llama_model_loader: - kv  25:              granitehybrid.embedding_scale f32              = 12.000000
+llama_model_loader: - kv  26:               granitehybrid.residual_scale f32              = 0.246000
+llama_model_loader: - kv  27:                  granitehybrid.logit_scale f32              = 3.000000
+llama_model_loader: - kv  28: granitehybrid.expert_shared_feed_forward_length u32              = 2048
+llama_model_loader: - kv  29:              granitehybrid.ssm.conv_kernel u32              = 4
+llama_model_loader: - kv  30:               granitehybrid.ssm.state_size u32              = 128
+llama_model_loader: - kv  31:              granitehybrid.ssm.group_count u32              = 1
+llama_model_loader: - kv  32:               granitehybrid.ssm.inner_size u32              = 1536
+llama_model_loader: - kv  33:           granitehybrid.ssm.time_step_rank u32              = 48
+llama_model_loader: - kv  34:       granitehybrid.rope.scaling.finetuned bool             = false
+llama_model_loader: - kv  35:                       tokenizer.ggml.model str              = gpt2
+llama_model_loader: - kv  36:                         tokenizer.ggml.pre str              = dbrx
+llama_model_loader: - kv  37:                      tokenizer.ggml.tokens arr[str,100352]  = ["!", "\"", "#", "$", "%", "&", "'", ...
+llama_model_loader: - kv  38:                  tokenizer.ggml.token_type arr[i32,100352]  = [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...
+llama_model_loader: - kv  39:                      tokenizer.ggml.merges arr[str,100000]  = ["Ġ Ġ", "ĠĠ ĠĠ", "i n", "Ġ t",...
+llama_model_loader: - kv  40:                tokenizer.ggml.bos_token_id u32              = 100257
+llama_model_loader: - kv  41:                tokenizer.ggml.eos_token_id u32              = 100257
+llama_model_loader: - kv  42:            tokenizer.ggml.unknown_token_id u32              = 100269
+llama_model_loader: - kv  43:            tokenizer.ggml.padding_token_id u32              = 100256
+llama_model_loader: - kv  44:               tokenizer.ggml.add_bos_token bool             = false
+llama_model_loader: - kv  45:                    tokenizer.chat_template str              = {%- set tools_system_message_prefix =...
+llama_model_loader: - kv  46:               general.quantization_version u32              = 2
+llama_model_loader: - kv  47:                          general.file_type u32              = 38
+llama_model_loader: - type  f32:  233 tensors
+llama_model_loader: - type q8_0:  120 tensors
+llama_model_loader: - type bf16:   16 tensors
+llama_model_loader: - type mxfp4:   33 tensors
+print_info: file format = GGUF V3 (latest)
+print_info: file type   = MXFP4 MoE
+print_info: file size   = 286.90 MiB (7.07 BPW) 
+load: printing all EOG tokens:
+load:   - 100257 ('<|end_of_text|>')
+load:   - 100261 ('<|fim_pad|>')
+load: special tokens cache size = 96
+load: token to piece cache size = 0.6152 MB
+print_info: arch             = granitehybrid
+print_info: vocab_only       = 0
+print_info: n_ctx_train      = 1048576
+print_info: n_embd           = 768
+print_info: n_embd_inp       = 768
+print_info: n_layer          = 32
+print_info: n_head           = 12
+print_info: n_head_kv        = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0]
+print_info: n_rot            = 64
+print_info: n_swa            = 0
+print_info: is_swa_any       = 0
+print_info: n_embd_head_k    = 64
+print_info: n_embd_head_v    = 64
+print_info: n_gqa            = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 3, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0]
+print_info: n_embd_k_gqa     = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 256, 0, 0, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 0, 0]
+print_info: n_embd_v_gqa     = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 256, 0, 0, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 0, 0]
+print_info: f_norm_eps       = 0.0e+00
+print_info: f_norm_rms_eps   = 1.0e-05
+print_info: f_clamp_kqv      = 0.0e+00
+print_info: f_max_alibi_bias = 0.0e+00
+print_info: f_logit_scale    = 3.0e+00
+print_info: f_attn_scale     = 1.6e-02
+print_info: n_ff             = 2048
+print_info: n_expert         = 0
+print_info: n_expert_used    = 0
+print_info: n_expert_groups  = 0
+print_info: n_group_used     = 0
+print_info: causal attn      = 1
+print_info: pooling type     = 0
+print_info: rope type        = 0
+print_info: rope scaling     = linear
+print_info: freq_base_train  = 10000.0
+print_info: freq_scale_train = 1
+print_info: n_ctx_orig_yarn  = 1048576
+print_info: rope_finetuned   = unknown
+print_info: ssm_d_conv       = 4
+print_info: ssm_d_inner      = 1536
+print_info: ssm_d_state      = 128
+print_info: ssm_dt_rank      = 48
+print_info: ssm_n_group      = 1
+print_info: ssm_dt_b_c_rms   = 0
+print_info: model type       = 350M
+print_info: model params     = 340.33 M
+print_info: general.name     = Granite 4.0 H 350m Unsloth
+print_info: f_embedding_scale = 12.000000
+print_info: f_residual_scale  = 0.246000
+print_info: f_attention_scale = 0.015625
+print_info: n_ff_shexp        = 2048
+print_info: vocab type       = BPE
+print_info: n_vocab          = 100352
+print_info: n_merges         = 100000
+print_info: BOS token        = 100257 '<|end_of_text|>'
+print_info: EOS token        = 100257 '<|end_of_text|>'
+print_info: EOT token        = 100257 '<|end_of_text|>'
+print_info: UNK token        = 100269 '<|unk|>'
+print_info: PAD token        = 100256 '<|pad|>'
+print_info: LF token         = 198 'Ċ'
+print_info: FIM PRE token    = 100258 '<|fim_prefix|>'
+print_info: FIM SUF token    = 100260 '<|fim_suffix|>'
+print_info: FIM MID token    = 100259 '<|fim_middle|>'
+print_info: FIM PAD token    = 100261 '<|fim_pad|>'
+print_info: EOG token        = 100257 '<|end_of_text|>'
+print_info: EOG token        = 100261 '<|fim_pad|>'
+print_info: max token length = 256
+load_tensors: loading model tensors, this can take a while... (mmap = true)
+load_tensors: offloading 20 repeating layers to GPU
+load_tensors: offloaded 20/33 layers to GPU
+load_tensors:   CPU_Mapped model buffer size =   132.43 MiB
+load_tensors:        CUDA0 model buffer size =    76.82 MiB
+load_tensors:        CUDA1 model buffer size =    77.68 MiB
+........................................................................................
+llama_context: constructing llama_context
+llama_context: n_seq_max     = 1
+llama_context: n_ctx         = 2048
+llama_context: n_ctx_seq     = 2048
+llama_context: n_batch       = 2048
+llama_context: n_ubatch      = 512
+llama_context: causal_attn   = 1
+llama_context: flash_attn    = auto
+llama_context: kv_unified    = false
+llama_context: freq_base     = 10000.0
+llama_context: freq_scale    = 1
+llama_context: n_ctx_seq (2048) < n_ctx_train (1048576) -- the full capacity of the model will not be utilized
+llama_context:        CPU  output buffer size =     0.38 MiB
+llama_kv_cache:        CPU KV buffer size =     2.00 MiB
+llama_kv_cache:      CUDA0 KV buffer size =     4.00 MiB
+llama_kv_cache:      CUDA1 KV buffer size =     2.00 MiB
+llama_kv_cache: size =    8.00 MiB (  2048 cells,   4 layers,  1/1 seqs), K (f16):    4.00 MiB, V (f16):    4.00 MiB
+llama_memory_recurrent:        CPU RS buffer size =     8.48 MiB
+llama_memory_recurrent:      CUDA0 RS buffer size =     6.16 MiB
+llama_memory_recurrent:      CUDA1 RS buffer size =     6.93 MiB
+llama_memory_recurrent: size =   21.57 MiB (     1 cells,  32 layers,  1 seqs), R (f32):    0.57 MiB, S (f32):   21.00 MiB
+llama_context: Flash Attention was auto, set to enabled
+llama_context:      CUDA0 compute buffer size =   245.16 MiB
+llama_context:      CUDA1 compute buffer size =    22.39 MiB
+llama_context:  CUDA_Host compute buffer size =    18.34 MiB
+llama_context: graph nodes  = 1815
+llama_context: graph splits = 182 (with bs=512), 41 (with bs=1)
+common_init_from_params: added <|end_of_text|> logit bias = -inf
+common_init_from_params: added <|fim_pad|> logit bias = -inf
+common_init_from_params: setting dry_penalty_last_n to ctx_size = 2048
+common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable)
+
+system_info: n_threads = 16 (n_threads_batch = 16) / 32 | CUDA : ARCHS = 860 | USE_GRAPHS = 1 | PEER_MAX_BATCH_SIZE = 128 | CPU : SSE3 = 1 | SSSE3 = 1 | AVX = 1 | AVX2 = 1 | F16C = 1 | FMA = 1 | BMI2 = 1 | AVX512 = 1 | AVX512_VBMI = 1 | AVX512_VNNI = 1 | AVX512_BF16 = 1 | LLAMAFILE = 1 | OPENMP = 1 | REPACK = 1 | 
+perplexity: tokenizing the input ..
+perplexity: tokenization took 38.896 ms
+perplexity: calculating perplexity over 14 chunks, n_ctx=2048, batch_size=2048, n_seq=1
+perplexity: 0.61 seconds per pass - ETA 0.13 minutes
+[1]31.5240,[2]45.7066,[3]48.3023,[4]41.9670,[5]42.6444,[6]37.7527,[7]38.7840,[8]39.0558,[9]41.6015,[10]41.4375,[11]40.8831,[12]40.1259,[13]39.5731,[14]38.5296,
+Final estimate: PPL = 38.5296 +/- 1.10574
+
+llama_perf_context_print:        load time =     214.78 ms
+llama_perf_context_print: prompt eval time =    4800.95 ms / 28672 tokens (    0.17 ms per token,  5972.15 tokens per second)
+llama_perf_context_print:        eval time =       0.00 ms /     1 runs   (    0.00 ms per token,      inf tokens per second)
+llama_perf_context_print:       total time =    5061.68 ms / 28673 tokens
+llama_perf_context_print:    graphs reused =          0
+llama_memory_breakdown_print: | memory breakdown [MiB] | total    free    self   model   context   compute    unaccounted |
+llama_memory_breakdown_print: |   - CUDA0 (RTX 3090)   | 24115 = 20551 + ( 332 =    76 +      10 +     245) +        3231 |
+llama_memory_breakdown_print: |   - CUDA1 (RTX 3090)   | 24124 = 23389 + ( 109 =    77 +       8 +      22) +         625 |
+llama_memory_breakdown_print: |   - Host               |                   161 =   132 +      10 +      18                |

Benchmarks/DataCollection/granite-4.0-h-350m-unsloth-mxfp4_moe-attn_kv_BF16-attn_output_BF16-attn_q_BF16-embeddings_MXFP4-ffn_down_MXFP4-ffn_up_gate_Q8_0/perplexity_math.log

@@ -0,0 +1,190 @@
+ggml_cuda_init: GGML_CUDA_FORCE_MMQ:    no
+ggml_cuda_init: GGML_CUDA_FORCE_CUBLAS: no
+ggml_cuda_init: found 2 CUDA devices:
+  Device 0: NVIDIA GeForce RTX 3090, compute capability 8.6, VMM: yes
+  Device 1: NVIDIA GeForce RTX 3090, compute capability 8.6, VMM: yes
+build: 7040 (92bb442ad) with cc (Ubuntu 13.3.0-6ubuntu2~24.04) 13.3.0 for x86_64-linux-gnu
+llama_model_load_from_file_impl: using device CUDA0 (NVIDIA GeForce RTX 3090) (0000:01:00.0) - 20970 MiB free
+llama_model_load_from_file_impl: using device CUDA1 (NVIDIA GeForce RTX 3090) (0000:03:00.0) - 23581 MiB free
+llama_model_loader: loaded meta data with 48 key-value pairs and 402 tensors from /mnt/world8/AI/ToBench/granite-4.0-h-350m-unsloth/Magic_Quant/GGUF/dc_round0_granite-4.0-h-350m-unsloth-mxfp4_moe-attn_kv_BF16-attn_output_BF16-attn_q_BF16-embeddings_MXFP4-ffn_down_MXFP4-ffn_up_gate_Q8_0.gguf (version GGUF V3 (latest))
+llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output.
+llama_model_loader: - kv   0:                       general.architecture str              = granitehybrid
+llama_model_loader: - kv   1:                               general.type str              = model
+llama_model_loader: - kv   2:                               general.name str              = Granite 4.0 H 350m Unsloth
+llama_model_loader: - kv   3:                           general.finetune str              = unsloth
+llama_model_loader: - kv   4:                           general.basename str              = granite-4.0-h
+llama_model_loader: - kv   5:                         general.size_label str              = 350M
+llama_model_loader: - kv   6:                            general.license str              = apache-2.0
+llama_model_loader: - kv   7:                   general.base_model.count u32              = 1
+llama_model_loader: - kv   8:                  general.base_model.0.name str              = Granite 4.0 H 350m
+llama_model_loader: - kv   9:          general.base_model.0.organization str              = Ibm Granite
+llama_model_loader: - kv  10:              general.base_model.0.repo_url str              = https://huggingface.co/ibm-granite/gr...
+llama_model_loader: - kv  11:                               general.tags arr[str,3]       = ["language", "unsloth", "granite-4.0"]
+llama_model_loader: - kv  12:                  granitehybrid.block_count u32              = 32
+llama_model_loader: - kv  13:               granitehybrid.context_length u32              = 1048576
+llama_model_loader: - kv  14:             granitehybrid.embedding_length u32              = 768
+llama_model_loader: - kv  15:          granitehybrid.feed_forward_length u32              = 2048
+llama_model_loader: - kv  16:         granitehybrid.attention.head_count u32              = 12
+llama_model_loader: - kv  17:      granitehybrid.attention.head_count_kv arr[i32,32]      = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, ...
+llama_model_loader: - kv  18:               granitehybrid.rope.freq_base f32              = 10000.000000
+llama_model_loader: - kv  19: granitehybrid.attention.layer_norm_rms_epsilon f32              = 0.000010
+llama_model_loader: - kv  20:                 granitehybrid.expert_count u32              = 0
+llama_model_loader: - kv  21:            granitehybrid.expert_used_count u32              = 0
+llama_model_loader: - kv  22:                   granitehybrid.vocab_size u32              = 100352
+llama_model_loader: - kv  23:         granitehybrid.rope.dimension_count u32              = 64
+llama_model_loader: - kv  24:              granitehybrid.attention.scale f32              = 0.015625
+llama_model_loader: - kv  25:              granitehybrid.embedding_scale f32              = 12.000000
+llama_model_loader: - kv  26:               granitehybrid.residual_scale f32              = 0.246000
+llama_model_loader: - kv  27:                  granitehybrid.logit_scale f32              = 3.000000
+llama_model_loader: - kv  28: granitehybrid.expert_shared_feed_forward_length u32              = 2048
+llama_model_loader: - kv  29:              granitehybrid.ssm.conv_kernel u32              = 4
+llama_model_loader: - kv  30:               granitehybrid.ssm.state_size u32              = 128
+llama_model_loader: - kv  31:              granitehybrid.ssm.group_count u32              = 1
+llama_model_loader: - kv  32:               granitehybrid.ssm.inner_size u32              = 1536
+llama_model_loader: - kv  33:           granitehybrid.ssm.time_step_rank u32              = 48
+llama_model_loader: - kv  34:       granitehybrid.rope.scaling.finetuned bool             = false
+llama_model_loader: - kv  35:                       tokenizer.ggml.model str              = gpt2
+llama_model_loader: - kv  36:                         tokenizer.ggml.pre str              = dbrx
+llama_model_loader: - kv  37:                      tokenizer.ggml.tokens arr[str,100352]  = ["!", "\"", "#", "$", "%", "&", "'", ...
+llama_model_loader: - kv  38:                  tokenizer.ggml.token_type arr[i32,100352]  = [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...
+llama_model_loader: - kv  39:                      tokenizer.ggml.merges arr[str,100000]  = ["Ġ Ġ", "ĠĠ ĠĠ", "i n", "Ġ t",...
+llama_model_loader: - kv  40:                tokenizer.ggml.bos_token_id u32              = 100257
+llama_model_loader: - kv  41:                tokenizer.ggml.eos_token_id u32              = 100257
+llama_model_loader: - kv  42:            tokenizer.ggml.unknown_token_id u32              = 100269
+llama_model_loader: - kv  43:            tokenizer.ggml.padding_token_id u32              = 100256
+llama_model_loader: - kv  44:               tokenizer.ggml.add_bos_token bool             = false
+llama_model_loader: - kv  45:                    tokenizer.chat_template str              = {%- set tools_system_message_prefix =...
+llama_model_loader: - kv  46:               general.quantization_version u32              = 2
+llama_model_loader: - kv  47:                          general.file_type u32              = 38
+llama_model_loader: - type  f32:  233 tensors
+llama_model_loader: - type q8_0:  120 tensors
+llama_model_loader: - type bf16:   16 tensors
+llama_model_loader: - type mxfp4:   33 tensors
+print_info: file format = GGUF V3 (latest)
+print_info: file type   = MXFP4 MoE
+print_info: file size   = 286.90 MiB (7.07 BPW) 
+load: printing all EOG tokens:
+load:   - 100257 ('<|end_of_text|>')
+load:   - 100261 ('<|fim_pad|>')
+load: special tokens cache size = 96
+load: token to piece cache size = 0.6152 MB
+print_info: arch             = granitehybrid
+print_info: vocab_only       = 0
+print_info: n_ctx_train      = 1048576
+print_info: n_embd           = 768
+print_info: n_embd_inp       = 768
+print_info: n_layer          = 32
+print_info: n_head           = 12
+print_info: n_head_kv        = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0]
+print_info: n_rot            = 64
+print_info: n_swa            = 0
+print_info: is_swa_any       = 0
+print_info: n_embd_head_k    = 64
+print_info: n_embd_head_v    = 64
+print_info: n_gqa            = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 3, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0]
+print_info: n_embd_k_gqa     = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 256, 0, 0, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 0, 0]
+print_info: n_embd_v_gqa     = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 256, 0, 0, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 0, 0]
+print_info: f_norm_eps       = 0.0e+00
+print_info: f_norm_rms_eps   = 1.0e-05
+print_info: f_clamp_kqv      = 0.0e+00
+print_info: f_max_alibi_bias = 0.0e+00
+print_info: f_logit_scale    = 3.0e+00
+print_info: f_attn_scale     = 1.6e-02
+print_info: n_ff             = 2048
+print_info: n_expert         = 0
+print_info: n_expert_used    = 0
+print_info: n_expert_groups  = 0
+print_info: n_group_used     = 0
+print_info: causal attn      = 1
+print_info: pooling type     = 0
+print_info: rope type        = 0
+print_info: rope scaling     = linear
+print_info: freq_base_train  = 10000.0
+print_info: freq_scale_train = 1
+print_info: n_ctx_orig_yarn  = 1048576
+print_info: rope_finetuned   = unknown
+print_info: ssm_d_conv       = 4
+print_info: ssm_d_inner      = 1536
+print_info: ssm_d_state      = 128
+print_info: ssm_dt_rank      = 48
+print_info: ssm_n_group      = 1
+print_info: ssm_dt_b_c_rms   = 0
+print_info: model type       = 350M
+print_info: model params     = 340.33 M
+print_info: general.name     = Granite 4.0 H 350m Unsloth
+print_info: f_embedding_scale = 12.000000
+print_info: f_residual_scale  = 0.246000
+print_info: f_attention_scale = 0.015625
+print_info: n_ff_shexp        = 2048
+print_info: vocab type       = BPE
+print_info: n_vocab          = 100352
+print_info: n_merges         = 100000
+print_info: BOS token        = 100257 '<|end_of_text|>'
+print_info: EOS token        = 100257 '<|end_of_text|>'
+print_info: EOT token        = 100257 '<|end_of_text|>'
+print_info: UNK token        = 100269 '<|unk|>'
+print_info: PAD token        = 100256 '<|pad|>'
+print_info: LF token         = 198 'Ċ'
+print_info: FIM PRE token    = 100258 '<|fim_prefix|>'
+print_info: FIM SUF token    = 100260 '<|fim_suffix|>'
+print_info: FIM MID token    = 100259 '<|fim_middle|>'
+print_info: FIM PAD token    = 100261 '<|fim_pad|>'
+print_info: EOG token        = 100257 '<|end_of_text|>'
+print_info: EOG token        = 100261 '<|fim_pad|>'
+print_info: max token length = 256
+load_tensors: loading model tensors, this can take a while... (mmap = true)
+load_tensors: offloading 20 repeating layers to GPU
+load_tensors: offloaded 20/33 layers to GPU
+load_tensors:   CPU_Mapped model buffer size =   132.43 MiB
+load_tensors:        CUDA0 model buffer size =    76.82 MiB
+load_tensors:        CUDA1 model buffer size =    77.68 MiB
+........................................................................................
+llama_context: constructing llama_context
+llama_context: n_seq_max     = 1
+llama_context: n_ctx         = 2048
+llama_context: n_ctx_seq     = 2048
+llama_context: n_batch       = 2048
+llama_context: n_ubatch      = 512
+llama_context: causal_attn   = 1
+llama_context: flash_attn    = auto
+llama_context: kv_unified    = false
+llama_context: freq_base     = 10000.0
+llama_context: freq_scale    = 1
+llama_context: n_ctx_seq (2048) < n_ctx_train (1048576) -- the full capacity of the model will not be utilized
+llama_context:        CPU  output buffer size =     0.38 MiB
+llama_kv_cache:        CPU KV buffer size =     2.00 MiB
+llama_kv_cache:      CUDA0 KV buffer size =     4.00 MiB
+llama_kv_cache:      CUDA1 KV buffer size =     2.00 MiB
+llama_kv_cache: size =    8.00 MiB (  2048 cells,   4 layers,  1/1 seqs), K (f16):    4.00 MiB, V (f16):    4.00 MiB
+llama_memory_recurrent:        CPU RS buffer size =     8.48 MiB
+llama_memory_recurrent:      CUDA0 RS buffer size =     6.16 MiB
+llama_memory_recurrent:      CUDA1 RS buffer size =     6.93 MiB
+llama_memory_recurrent: size =   21.57 MiB (     1 cells,  32 layers,  1 seqs), R (f32):    0.57 MiB, S (f32):   21.00 MiB
+llama_context: Flash Attention was auto, set to enabled
+llama_context:      CUDA0 compute buffer size =   245.16 MiB
+llama_context:      CUDA1 compute buffer size =    22.39 MiB
+llama_context:  CUDA_Host compute buffer size =    18.34 MiB
+llama_context: graph nodes  = 1815
+llama_context: graph splits = 182 (with bs=512), 41 (with bs=1)
+common_init_from_params: added <|end_of_text|> logit bias = -inf
+common_init_from_params: added <|fim_pad|> logit bias = -inf
+common_init_from_params: setting dry_penalty_last_n to ctx_size = 2048
+common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable)
+
+system_info: n_threads = 16 (n_threads_batch = 16) / 32 | CUDA : ARCHS = 860 | USE_GRAPHS = 1 | PEER_MAX_BATCH_SIZE = 128 | CPU : SSE3 = 1 | SSSE3 = 1 | AVX = 1 | AVX2 = 1 | F16C = 1 | FMA = 1 | BMI2 = 1 | AVX512 = 1 | AVX512_VBMI = 1 | AVX512_VNNI = 1 | AVX512_BF16 = 1 | LLAMAFILE = 1 | OPENMP = 1 | REPACK = 1 | 
+perplexity: tokenizing the input ..
+perplexity: tokenization took 33.959 ms
+perplexity: calculating perplexity over 15 chunks, n_ctx=2048, batch_size=2048, n_seq=1
+perplexity: 0.57 seconds per pass - ETA 0.13 minutes
+[1]25.7186,[2]28.5748,[3]26.3380,[4]27.3421,[5]28.1277,[6]27.8836,[7]27.8618,[8]26.3901,[9]26.6020,[10]27.2272,[11]28.0583,[12]28.3542,[13]29.1743,[14]28.9948,[15]28.6447,
+Final estimate: PPL = 28.6447 +/- 0.77572
+
+llama_perf_context_print:        load time =     203.92 ms
+llama_perf_context_print: prompt eval time =    5059.67 ms / 30720 tokens (    0.16 ms per token,  6071.55 tokens per second)
+llama_perf_context_print:        eval time =       0.00 ms /     1 runs   (    0.00 ms per token,      inf tokens per second)
+llama_perf_context_print:       total time =    5330.85 ms / 30721 tokens
+llama_perf_context_print:    graphs reused =          0
+llama_memory_breakdown_print: | memory breakdown [MiB] | total    free    self   model   context   compute    unaccounted |
+llama_memory_breakdown_print: |   - CUDA0 (RTX 3090)   | 24115 = 20554 + ( 332 =    76 +      10 +     245) +        3228 |
+llama_memory_breakdown_print: |   - CUDA1 (RTX 3090)   | 24124 = 23389 + ( 109 =    77 +       8 +      22) +         625 |
+llama_memory_breakdown_print: |   - Host               |                   161 =   132 +      10 +      18                |

Benchmarks/DataCollection/granite-4.0-h-350m-unsloth-mxfp4_moe-attn_kv_BF16-attn_output_BF16-attn_q_BF16-embeddings_Q8_0-ffn_down_BF16-ffn_up_gate_Q8_0/bench_metrics.json

@@ -0,0 +1,44 @@
+{
+  "raw_metrics": {
+    "llamabench": {
+      "backend": "CUDA",
+      "log_path": "granite-4.0-h-350m-unsloth-mxfp4_moe-attn_kv_BF16-attn_output_BF16-attn_q_BF16-embeddings_Q8_0-ffn_down_BF16-ffn_up_gate_Q8_0/llamabench.md",
+      "ngl": "35",
+      "raw_row": {
+        "backend": "CUDA",
+        "model": "granitehybrid 350M MXFP4 MoE",
+        "ngl": "35",
+        "params": "340.33 M",
+        "size": "396.45 MiB",
+        "t/s": "1741.52 \u00b1 50.12",
+        "test": "pp8",
+        "tps_value": 1741.52
+      },
+      "test": "pp8",
+      "tps": 1741.52
+    },
+    "perplexity": {
+      "code": {
+        "log_path": "granite-4.0-h-350m-unsloth-mxfp4_moe-attn_kv_BF16-attn_output_BF16-attn_q_BF16-embeddings_Q8_0-ffn_down_BF16-ffn_up_gate_Q8_0/perplexity_code.log",
+        "ppl": 1.9567,
+        "ppl_error": 0.01755
+      },
+      "general": {
+        "log_path": "granite-4.0-h-350m-unsloth-mxfp4_moe-attn_kv_BF16-attn_output_BF16-attn_q_BF16-embeddings_Q8_0-ffn_down_BF16-ffn_up_gate_Q8_0/perplexity_general.log",
+        "ppl": 18.2447,
+        "ppl_error": 0.4697
+      },
+      "math": {
+        "log_path": "granite-4.0-h-350m-unsloth-mxfp4_moe-attn_kv_BF16-attn_output_BF16-attn_q_BF16-embeddings_Q8_0-ffn_down_BF16-ffn_up_gate_Q8_0/perplexity_math.log",
+        "ppl": 10.3383,
+        "ppl_error": 0.23316
+      }
+    }
+  },
+  "summary": {
+    "avg_prec_loss_pct": 0.4023,
+    "bench_tps": 1741.52,
+    "file_size_bytes": 419279808,
+    "file_size_gb": 0.39
+  }
+}

Benchmarks/DataCollection/granite-4.0-h-350m-unsloth-mxfp4_moe-attn_kv_BF16-attn_output_BF16-attn_q_BF16-embeddings_Q8_0-ffn_down_BF16-ffn_up_gate_Q8_0/llamabench.md

@@ -0,0 +1,11 @@
+ggml_cuda_init: GGML_CUDA_FORCE_MMQ:    no
+ggml_cuda_init: GGML_CUDA_FORCE_CUBLAS: no
+ggml_cuda_init: found 2 CUDA devices:
+  Device 0: NVIDIA GeForce RTX 3090, compute capability 8.6, VMM: yes
+  Device 1: NVIDIA GeForce RTX 3090, compute capability 8.6, VMM: yes
+| model                          |       size |     params | backend    | ngl |            test |                  t/s |
+| ------------------------------ | ---------: | ---------: | ---------- | --: | --------------: | -------------------: |
+| granitehybrid 350M MXFP4 MoE   | 396.45 MiB |   340.33 M | CUDA       |  35 |             pp8 |      1741.52 ± 50.12 |
+| granitehybrid 350M MXFP4 MoE   | 396.45 MiB |   340.33 M | CUDA       |  35 |           tg128 |        325.04 ± 8.26 |
+
+build: 92bb442ad (7040)

Benchmarks/DataCollection/granite-4.0-h-350m-unsloth-mxfp4_moe-attn_kv_BF16-attn_output_BF16-attn_q_BF16-embeddings_Q8_0-ffn_down_BF16-ffn_up_gate_Q8_0/perplexity_code.log

@@ -0,0 +1,189 @@
+ggml_cuda_init: GGML_CUDA_FORCE_MMQ:    no
+ggml_cuda_init: GGML_CUDA_FORCE_CUBLAS: no
+ggml_cuda_init: found 2 CUDA devices:
+  Device 0: NVIDIA GeForce RTX 3090, compute capability 8.6, VMM: yes
+  Device 1: NVIDIA GeForce RTX 3090, compute capability 8.6, VMM: yes
+build: 7040 (92bb442ad) with cc (Ubuntu 13.3.0-6ubuntu2~24.04) 13.3.0 for x86_64-linux-gnu
+llama_model_load_from_file_impl: using device CUDA0 (NVIDIA GeForce RTX 3090) (0000:01:00.0) - 20922 MiB free
+llama_model_load_from_file_impl: using device CUDA1 (NVIDIA GeForce RTX 3090) (0000:03:00.0) - 23581 MiB free
+llama_model_loader: loaded meta data with 48 key-value pairs and 402 tensors from /mnt/world8/AI/ToBench/granite-4.0-h-350m-unsloth/Magic_Quant/GGUF/dc_round1_granite-4.0-h-350m-unsloth-mxfp4_moe-attn_kv_BF16-attn_output_BF16-attn_q_BF16-embeddings_Q8_0-ffn_down_BF16-ffn_up_gate_Q8_0.gguf (version GGUF V3 (latest))
+llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output.
+llama_model_loader: - kv   0:                       general.architecture str              = granitehybrid
+llama_model_loader: - kv   1:                               general.type str              = model
+llama_model_loader: - kv   2:                               general.name str              = Granite 4.0 H 350m Unsloth
+llama_model_loader: - kv   3:                           general.finetune str              = unsloth
+llama_model_loader: - kv   4:                           general.basename str              = granite-4.0-h
+llama_model_loader: - kv   5:                         general.size_label str              = 350M
+llama_model_loader: - kv   6:                            general.license str              = apache-2.0
+llama_model_loader: - kv   7:                   general.base_model.count u32              = 1
+llama_model_loader: - kv   8:                  general.base_model.0.name str              = Granite 4.0 H 350m
+llama_model_loader: - kv   9:          general.base_model.0.organization str              = Ibm Granite
+llama_model_loader: - kv  10:              general.base_model.0.repo_url str              = https://huggingface.co/ibm-granite/gr...
+llama_model_loader: - kv  11:                               general.tags arr[str,3]       = ["language", "unsloth", "granite-4.0"]
+llama_model_loader: - kv  12:                  granitehybrid.block_count u32              = 32
+llama_model_loader: - kv  13:               granitehybrid.context_length u32              = 1048576
+llama_model_loader: - kv  14:             granitehybrid.embedding_length u32              = 768
+llama_model_loader: - kv  15:          granitehybrid.feed_forward_length u32              = 2048
+llama_model_loader: - kv  16:         granitehybrid.attention.head_count u32              = 12
+llama_model_loader: - kv  17:      granitehybrid.attention.head_count_kv arr[i32,32]      = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, ...
+llama_model_loader: - kv  18:               granitehybrid.rope.freq_base f32              = 10000.000000
+llama_model_loader: - kv  19: granitehybrid.attention.layer_norm_rms_epsilon f32              = 0.000010
+llama_model_loader: - kv  20:                 granitehybrid.expert_count u32              = 0
+llama_model_loader: - kv  21:            granitehybrid.expert_used_count u32              = 0
+llama_model_loader: - kv  22:                   granitehybrid.vocab_size u32              = 100352
+llama_model_loader: - kv  23:         granitehybrid.rope.dimension_count u32              = 64
+llama_model_loader: - kv  24:              granitehybrid.attention.scale f32              = 0.015625
+llama_model_loader: - kv  25:              granitehybrid.embedding_scale f32              = 12.000000
+llama_model_loader: - kv  26:               granitehybrid.residual_scale f32              = 0.246000
+llama_model_loader: - kv  27:                  granitehybrid.logit_scale f32              = 3.000000
+llama_model_loader: - kv  28: granitehybrid.expert_shared_feed_forward_length u32              = 2048
+llama_model_loader: - kv  29:              granitehybrid.ssm.conv_kernel u32              = 4
+llama_model_loader: - kv  30:               granitehybrid.ssm.state_size u32              = 128
+llama_model_loader: - kv  31:              granitehybrid.ssm.group_count u32              = 1
+llama_model_loader: - kv  32:               granitehybrid.ssm.inner_size u32              = 1536
+llama_model_loader: - kv  33:           granitehybrid.ssm.time_step_rank u32              = 48
+llama_model_loader: - kv  34:       granitehybrid.rope.scaling.finetuned bool             = false
+llama_model_loader: - kv  35:                       tokenizer.ggml.model str              = gpt2
+llama_model_loader: - kv  36:                         tokenizer.ggml.pre str              = dbrx
+llama_model_loader: - kv  37:                      tokenizer.ggml.tokens arr[str,100352]  = ["!", "\"", "#", "$", "%", "&", "'", ...
+llama_model_loader: - kv  38:                  tokenizer.ggml.token_type arr[i32,100352]  = [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...
+llama_model_loader: - kv  39:                      tokenizer.ggml.merges arr[str,100000]  = ["Ġ Ġ", "ĠĠ ĠĠ", "i n", "Ġ t",...
+llama_model_loader: - kv  40:                tokenizer.ggml.bos_token_id u32              = 100257
+llama_model_loader: - kv  41:                tokenizer.ggml.eos_token_id u32              = 100257
+llama_model_loader: - kv  42:            tokenizer.ggml.unknown_token_id u32              = 100269
+llama_model_loader: - kv  43:            tokenizer.ggml.padding_token_id u32              = 100256
+llama_model_loader: - kv  44:               tokenizer.ggml.add_bos_token bool             = false
+llama_model_loader: - kv  45:                    tokenizer.chat_template str              = {%- set tools_system_message_prefix =...
+llama_model_loader: - kv  46:               general.quantization_version u32              = 2
+llama_model_loader: - kv  47:                          general.file_type u32              = 38
+llama_model_loader: - type  f32:  233 tensors
+llama_model_loader: - type q8_0:  121 tensors
+llama_model_loader: - type bf16:   48 tensors
+print_info: file format = GGUF V3 (latest)
+print_info: file type   = MXFP4 MoE
+print_info: file size   = 396.45 MiB (9.77 BPW) 
+load: printing all EOG tokens:
+load:   - 100257 ('<|end_of_text|>')
+load:   - 100261 ('<|fim_pad|>')
+load: special tokens cache size = 96
+load: token to piece cache size = 0.6152 MB
+print_info: arch             = granitehybrid
+print_info: vocab_only       = 0
+print_info: n_ctx_train      = 1048576
+print_info: n_embd           = 768
+print_info: n_embd_inp       = 768
+print_info: n_layer          = 32
+print_info: n_head           = 12
+print_info: n_head_kv        = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0]
+print_info: n_rot            = 64
+print_info: n_swa            = 0
+print_info: is_swa_any       = 0
+print_info: n_embd_head_k    = 64
+print_info: n_embd_head_v    = 64
+print_info: n_gqa            = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 3, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0]
+print_info: n_embd_k_gqa     = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 256, 0, 0, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 0, 0]
+print_info: n_embd_v_gqa     = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 256, 0, 0, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 0, 0]
+print_info: f_norm_eps       = 0.0e+00
+print_info: f_norm_rms_eps   = 1.0e-05
+print_info: f_clamp_kqv      = 0.0e+00
+print_info: f_max_alibi_bias = 0.0e+00
+print_info: f_logit_scale    = 3.0e+00
+print_info: f_attn_scale     = 1.6e-02
+print_info: n_ff             = 2048
+print_info: n_expert         = 0
+print_info: n_expert_used    = 0
+print_info: n_expert_groups  = 0
+print_info: n_group_used     = 0
+print_info: causal attn      = 1
+print_info: pooling type     = 0
+print_info: rope type        = 0
+print_info: rope scaling     = linear
+print_info: freq_base_train  = 10000.0
+print_info: freq_scale_train = 1
+print_info: n_ctx_orig_yarn  = 1048576
+print_info: rope_finetuned   = unknown
+print_info: ssm_d_conv       = 4
+print_info: ssm_d_inner      = 1536
+print_info: ssm_d_state      = 128
+print_info: ssm_dt_rank      = 48
+print_info: ssm_n_group      = 1
+print_info: ssm_dt_b_c_rms   = 0
+print_info: model type       = 350M
+print_info: model params     = 340.33 M
+print_info: general.name     = Granite 4.0 H 350m Unsloth
+print_info: f_embedding_scale = 12.000000
+print_info: f_residual_scale  = 0.246000
+print_info: f_attention_scale = 0.015625
+print_info: n_ff_shexp        = 2048
+print_info: vocab type       = BPE
+print_info: n_vocab          = 100352
+print_info: n_merges         = 100000
+print_info: BOS token        = 100257 '<|end_of_text|>'
+print_info: EOS token        = 100257 '<|end_of_text|>'
+print_info: EOT token        = 100257 '<|end_of_text|>'
+print_info: UNK token        = 100269 '<|unk|>'
+print_info: PAD token        = 100256 '<|pad|>'
+print_info: LF token         = 198 'Ċ'
+print_info: FIM PRE token    = 100258 '<|fim_prefix|>'
+print_info: FIM SUF token    = 100260 '<|fim_suffix|>'
+print_info: FIM MID token    = 100259 '<|fim_middle|>'
+print_info: FIM PAD token    = 100261 '<|fim_pad|>'
+print_info: EOG token        = 100257 '<|end_of_text|>'
+print_info: EOG token        = 100261 '<|fim_pad|>'
+print_info: max token length = 256
+load_tensors: loading model tensors, this can take a while... (mmap = true)
+load_tensors: offloading 20 repeating layers to GPU
+load_tensors: offloaded 20/33 layers to GPU
+load_tensors:   CPU_Mapped model buffer size =   197.91 MiB
+load_tensors:        CUDA0 model buffer size =    98.85 MiB
+load_tensors:        CUDA1 model buffer size =    99.71 MiB
+..................................................................................
+llama_context: constructing llama_context
+llama_context: n_seq_max     = 1
+llama_context: n_ctx         = 2048
+llama_context: n_ctx_seq     = 2048
+llama_context: n_batch       = 2048
+llama_context: n_ubatch      = 512
+llama_context: causal_attn   = 1
+llama_context: flash_attn    = auto
+llama_context: kv_unified    = false
+llama_context: freq_base     = 10000.0
+llama_context: freq_scale    = 1
+llama_context: n_ctx_seq (2048) < n_ctx_train (1048576) -- the full capacity of the model will not be utilized
+llama_context:        CPU  output buffer size =     0.38 MiB
+llama_kv_cache:        CPU KV buffer size =     2.00 MiB
+llama_kv_cache:      CUDA0 KV buffer size =     4.00 MiB
+llama_kv_cache:      CUDA1 KV buffer size =     2.00 MiB
+llama_kv_cache: size =    8.00 MiB (  2048 cells,   4 layers,  1/1 seqs), K (f16):    4.00 MiB, V (f16):    4.00 MiB
+llama_memory_recurrent:        CPU RS buffer size =     8.48 MiB
+llama_memory_recurrent:      CUDA0 RS buffer size =     6.16 MiB
+llama_memory_recurrent:      CUDA1 RS buffer size =     6.93 MiB
+llama_memory_recurrent: size =   21.57 MiB (     1 cells,  32 layers,  1 seqs), R (f32):    0.57 MiB, S (f32):   21.00 MiB
+llama_context: Flash Attention was auto, set to enabled
+llama_context:      CUDA0 compute buffer size =   283.69 MiB
+llama_context:      CUDA1 compute buffer size =    22.39 MiB
+llama_context:  CUDA_Host compute buffer size =    18.34 MiB
+llama_context: graph nodes  = 1815
+llama_context: graph splits = 182 (with bs=512), 41 (with bs=1)
+common_init_from_params: added <|end_of_text|> logit bias = -inf
+common_init_from_params: added <|fim_pad|> logit bias = -inf
+common_init_from_params: setting dry_penalty_last_n to ctx_size = 2048
+common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable)
+
+system_info: n_threads = 16 (n_threads_batch = 16) / 32 | CUDA : ARCHS = 860 | USE_GRAPHS = 1 | PEER_MAX_BATCH_SIZE = 128 | CPU : SSE3 = 1 | SSSE3 = 1 | AVX = 1 | AVX2 = 1 | F16C = 1 | FMA = 1 | BMI2 = 1 | AVX512 = 1 | AVX512_VBMI = 1 | AVX512_VNNI = 1 | AVX512_BF16 = 1 | LLAMAFILE = 1 | OPENMP = 1 | REPACK = 1 | 
+perplexity: tokenizing the input ..
+perplexity: tokenization took 96.678 ms
+perplexity: calculating perplexity over 44 chunks, n_ctx=2048, batch_size=2048, n_seq=1
+perplexity: 0.58 seconds per pass - ETA 0.42 minutes
+[1]4.3776,[2]3.9915,[3]2.5736,[4]2.3696,[5]2.6082,[6]2.8523,[7]2.7037,[8]2.5124,[9]2.3093,[10]2.1403,[11]2.1224,[12]2.1489,[13]2.0600,[14]2.0395,[15]2.0800,[16]2.0140,[17]1.9887,[18]2.0071,[19]1.9676,[20]1.9325,[21]1.8995,[22]1.8844,[23]1.9134,[24]1.8869,[25]1.9061,[26]1.8741,[27]1.8611,[28]1.8529,[29]1.8984,[30]1.9149,[31]1.9141,[32]1.8897,[33]1.9134,[34]1.9056,[35]1.8869,[36]1.9183,[37]1.9248,[38]1.9228,[39]1.9446,[40]1.9419,[41]1.9348,[42]1.9588,[43]1.9676,[44]1.9567,
+Final estimate: PPL = 1.9567 +/- 0.01755
+
+llama_perf_context_print:        load time =     226.45 ms
+llama_perf_context_print: prompt eval time =   15117.57 ms / 90112 tokens (    0.17 ms per token,  5960.75 tokens per second)
+llama_perf_context_print:        eval time =       0.00 ms /     1 runs   (    0.00 ms per token,      inf tokens per second)
+llama_perf_context_print:       total time =   15919.68 ms / 90113 tokens
+llama_perf_context_print:    graphs reused =          0
+llama_memory_breakdown_print: | memory breakdown [MiB] | total    free    self   model   context   compute    unaccounted |
+llama_memory_breakdown_print: |   - CUDA0 (RTX 3090)   | 24115 = 20450 + ( 393 =    98 +      10 +     284) +        3271 |
+llama_memory_breakdown_print: |   - CUDA1 (RTX 3090)   | 24124 = 23367 + ( 131 =    99 +       8 +      22) +         625 |
+llama_memory_breakdown_print: |   - Host               |                   226 =   197 +      10 +      18                |

Benchmarks/DataCollection/granite-4.0-h-350m-unsloth-mxfp4_moe-attn_kv_BF16-attn_output_BF16-attn_q_BF16-embeddings_Q8_0-ffn_down_BF16-ffn_up_gate_Q8_0/perplexity_general.log

@@ -0,0 +1,189 @@
+ggml_cuda_init: GGML_CUDA_FORCE_MMQ:    no
+ggml_cuda_init: GGML_CUDA_FORCE_CUBLAS: no
+ggml_cuda_init: found 2 CUDA devices:
+  Device 0: NVIDIA GeForce RTX 3090, compute capability 8.6, VMM: yes
+  Device 1: NVIDIA GeForce RTX 3090, compute capability 8.6, VMM: yes
+build: 7040 (92bb442ad) with cc (Ubuntu 13.3.0-6ubuntu2~24.04) 13.3.0 for x86_64-linux-gnu
+llama_model_load_from_file_impl: using device CUDA0 (NVIDIA GeForce RTX 3090) (0000:01:00.0) - 20932 MiB free
+llama_model_load_from_file_impl: using device CUDA1 (NVIDIA GeForce RTX 3090) (0000:03:00.0) - 23581 MiB free
+llama_model_loader: loaded meta data with 48 key-value pairs and 402 tensors from /mnt/world8/AI/ToBench/granite-4.0-h-350m-unsloth/Magic_Quant/GGUF/dc_round1_granite-4.0-h-350m-unsloth-mxfp4_moe-attn_kv_BF16-attn_output_BF16-attn_q_BF16-embeddings_Q8_0-ffn_down_BF16-ffn_up_gate_Q8_0.gguf (version GGUF V3 (latest))
+llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output.
+llama_model_loader: - kv   0:                       general.architecture str              = granitehybrid
+llama_model_loader: - kv   1:                               general.type str              = model
+llama_model_loader: - kv   2:                               general.name str              = Granite 4.0 H 350m Unsloth
+llama_model_loader: - kv   3:                           general.finetune str              = unsloth
+llama_model_loader: - kv   4:                           general.basename str              = granite-4.0-h
+llama_model_loader: - kv   5:                         general.size_label str              = 350M
+llama_model_loader: - kv   6:                            general.license str              = apache-2.0
+llama_model_loader: - kv   7:                   general.base_model.count u32              = 1
+llama_model_loader: - kv   8:                  general.base_model.0.name str              = Granite 4.0 H 350m
+llama_model_loader: - kv   9:          general.base_model.0.organization str              = Ibm Granite
+llama_model_loader: - kv  10:              general.base_model.0.repo_url str              = https://huggingface.co/ibm-granite/gr...
+llama_model_loader: - kv  11:                               general.tags arr[str,3]       = ["language", "unsloth", "granite-4.0"]
+llama_model_loader: - kv  12:                  granitehybrid.block_count u32              = 32
+llama_model_loader: - kv  13:               granitehybrid.context_length u32              = 1048576
+llama_model_loader: - kv  14:             granitehybrid.embedding_length u32              = 768
+llama_model_loader: - kv  15:          granitehybrid.feed_forward_length u32              = 2048
+llama_model_loader: - kv  16:         granitehybrid.attention.head_count u32              = 12
+llama_model_loader: - kv  17:      granitehybrid.attention.head_count_kv arr[i32,32]      = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, ...
+llama_model_loader: - kv  18:               granitehybrid.rope.freq_base f32              = 10000.000000
+llama_model_loader: - kv  19: granitehybrid.attention.layer_norm_rms_epsilon f32              = 0.000010
+llama_model_loader: - kv  20:                 granitehybrid.expert_count u32              = 0
+llama_model_loader: - kv  21:            granitehybrid.expert_used_count u32              = 0
+llama_model_loader: - kv  22:                   granitehybrid.vocab_size u32              = 100352
+llama_model_loader: - kv  23:         granitehybrid.rope.dimension_count u32              = 64
+llama_model_loader: - kv  24:              granitehybrid.attention.scale f32              = 0.015625
+llama_model_loader: - kv  25:              granitehybrid.embedding_scale f32              = 12.000000
+llama_model_loader: - kv  26:               granitehybrid.residual_scale f32              = 0.246000
+llama_model_loader: - kv  27:                  granitehybrid.logit_scale f32              = 3.000000
+llama_model_loader: - kv  28: granitehybrid.expert_shared_feed_forward_length u32              = 2048
+llama_model_loader: - kv  29:              granitehybrid.ssm.conv_kernel u32              = 4
+llama_model_loader: - kv  30:               granitehybrid.ssm.state_size u32              = 128
+llama_model_loader: - kv  31:              granitehybrid.ssm.group_count u32              = 1
+llama_model_loader: - kv  32:               granitehybrid.ssm.inner_size u32              = 1536
+llama_model_loader: - kv  33:           granitehybrid.ssm.time_step_rank u32              = 48
+llama_model_loader: - kv  34:       granitehybrid.rope.scaling.finetuned bool             = false
+llama_model_loader: - kv  35:                       tokenizer.ggml.model str              = gpt2
+llama_model_loader: - kv  36:                         tokenizer.ggml.pre str              = dbrx
+llama_model_loader: - kv  37:                      tokenizer.ggml.tokens arr[str,100352]  = ["!", "\"", "#", "$", "%", "&", "'", ...
+llama_model_loader: - kv  38:                  tokenizer.ggml.token_type arr[i32,100352]  = [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...
+llama_model_loader: - kv  39:                      tokenizer.ggml.merges arr[str,100000]  = ["Ġ Ġ", "ĠĠ ĠĠ", "i n", "Ġ t",...
+llama_model_loader: - kv  40:                tokenizer.ggml.bos_token_id u32              = 100257
+llama_model_loader: - kv  41:                tokenizer.ggml.eos_token_id u32              = 100257
+llama_model_loader: - kv  42:            tokenizer.ggml.unknown_token_id u32              = 100269
+llama_model_loader: - kv  43:            tokenizer.ggml.padding_token_id u32              = 100256
+llama_model_loader: - kv  44:               tokenizer.ggml.add_bos_token bool             = false
+llama_model_loader: - kv  45:                    tokenizer.chat_template str              = {%- set tools_system_message_prefix =...
+llama_model_loader: - kv  46:               general.quantization_version u32              = 2
+llama_model_loader: - kv  47:                          general.file_type u32              = 38
+llama_model_loader: - type  f32:  233 tensors
+llama_model_loader: - type q8_0:  121 tensors
+llama_model_loader: - type bf16:   48 tensors
+print_info: file format = GGUF V3 (latest)
+print_info: file type   = MXFP4 MoE
+print_info: file size   = 396.45 MiB (9.77 BPW) 
+load: printing all EOG tokens:
+load:   - 100257 ('<|end_of_text|>')
+load:   - 100261 ('<|fim_pad|>')
+load: special tokens cache size = 96
+load: token to piece cache size = 0.6152 MB
+print_info: arch             = granitehybrid
+print_info: vocab_only       = 0
+print_info: n_ctx_train      = 1048576
+print_info: n_embd           = 768
+print_info: n_embd_inp       = 768
+print_info: n_layer          = 32
+print_info: n_head           = 12
+print_info: n_head_kv        = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0]
+print_info: n_rot            = 64
+print_info: n_swa            = 0
+print_info: is_swa_any       = 0
+print_info: n_embd_head_k    = 64
+print_info: n_embd_head_v    = 64
+print_info: n_gqa            = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 3, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0]
+print_info: n_embd_k_gqa     = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 256, 0, 0, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 0, 0]
+print_info: n_embd_v_gqa     = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 256, 0, 0, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 0, 0]
+print_info: f_norm_eps       = 0.0e+00
+print_info: f_norm_rms_eps   = 1.0e-05
+print_info: f_clamp_kqv      = 0.0e+00
+print_info: f_max_alibi_bias = 0.0e+00
+print_info: f_logit_scale    = 3.0e+00
+print_info: f_attn_scale     = 1.6e-02
+print_info: n_ff             = 2048
+print_info: n_expert         = 0
+print_info: n_expert_used    = 0
+print_info: n_expert_groups  = 0
+print_info: n_group_used     = 0
+print_info: causal attn      = 1
+print_info: pooling type     = 0
+print_info: rope type        = 0
+print_info: rope scaling     = linear
+print_info: freq_base_train  = 10000.0
+print_info: freq_scale_train = 1
+print_info: n_ctx_orig_yarn  = 1048576
+print_info: rope_finetuned   = unknown
+print_info: ssm_d_conv       = 4
+print_info: ssm_d_inner      = 1536
+print_info: ssm_d_state      = 128
+print_info: ssm_dt_rank      = 48
+print_info: ssm_n_group      = 1
+print_info: ssm_dt_b_c_rms   = 0
+print_info: model type       = 350M
+print_info: model params     = 340.33 M
+print_info: general.name     = Granite 4.0 H 350m Unsloth
+print_info: f_embedding_scale = 12.000000
+print_info: f_residual_scale  = 0.246000
+print_info: f_attention_scale = 0.015625
+print_info: n_ff_shexp        = 2048
+print_info: vocab type       = BPE
+print_info: n_vocab          = 100352
+print_info: n_merges         = 100000
+print_info: BOS token        = 100257 '<|end_of_text|>'
+print_info: EOS token        = 100257 '<|end_of_text|>'
+print_info: EOT token        = 100257 '<|end_of_text|>'
+print_info: UNK token        = 100269 '<|unk|>'
+print_info: PAD token        = 100256 '<|pad|>'
+print_info: LF token         = 198 'Ċ'
+print_info: FIM PRE token    = 100258 '<|fim_prefix|>'
+print_info: FIM SUF token    = 100260 '<|fim_suffix|>'
+print_info: FIM MID token    = 100259 '<|fim_middle|>'
+print_info: FIM PAD token    = 100261 '<|fim_pad|>'
+print_info: EOG token        = 100257 '<|end_of_text|>'
+print_info: EOG token        = 100261 '<|fim_pad|>'
+print_info: max token length = 256
+load_tensors: loading model tensors, this can take a while... (mmap = true)
+load_tensors: offloading 20 repeating layers to GPU
+load_tensors: offloaded 20/33 layers to GPU
+load_tensors:   CPU_Mapped model buffer size =   197.91 MiB
+load_tensors:        CUDA0 model buffer size =    98.85 MiB
+load_tensors:        CUDA1 model buffer size =    99.71 MiB
+..................................................................................
+llama_context: constructing llama_context
+llama_context: n_seq_max     = 1
+llama_context: n_ctx         = 2048
+llama_context: n_ctx_seq     = 2048
+llama_context: n_batch       = 2048
+llama_context: n_ubatch      = 512
+llama_context: causal_attn   = 1
+llama_context: flash_attn    = auto
+llama_context: kv_unified    = false
+llama_context: freq_base     = 10000.0
+llama_context: freq_scale    = 1
+llama_context: n_ctx_seq (2048) < n_ctx_train (1048576) -- the full capacity of the model will not be utilized
+llama_context:        CPU  output buffer size =     0.38 MiB
+llama_kv_cache:        CPU KV buffer size =     2.00 MiB
+llama_kv_cache:      CUDA0 KV buffer size =     4.00 MiB
+llama_kv_cache:      CUDA1 KV buffer size =     2.00 MiB
+llama_kv_cache: size =    8.00 MiB (  2048 cells,   4 layers,  1/1 seqs), K (f16):    4.00 MiB, V (f16):    4.00 MiB
+llama_memory_recurrent:        CPU RS buffer size =     8.48 MiB
+llama_memory_recurrent:      CUDA0 RS buffer size =     6.16 MiB
+llama_memory_recurrent:      CUDA1 RS buffer size =     6.93 MiB
+llama_memory_recurrent: size =   21.57 MiB (     1 cells,  32 layers,  1 seqs), R (f32):    0.57 MiB, S (f32):   21.00 MiB
+llama_context: Flash Attention was auto, set to enabled
+llama_context:      CUDA0 compute buffer size =   283.69 MiB
+llama_context:      CUDA1 compute buffer size =    22.39 MiB
+llama_context:  CUDA_Host compute buffer size =    18.34 MiB
+llama_context: graph nodes  = 1815
+llama_context: graph splits = 182 (with bs=512), 41 (with bs=1)
+common_init_from_params: added <|end_of_text|> logit bias = -inf
+common_init_from_params: added <|fim_pad|> logit bias = -inf
+common_init_from_params: setting dry_penalty_last_n to ctx_size = 2048
+common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable)
+
+system_info: n_threads = 16 (n_threads_batch = 16) / 32 | CUDA : ARCHS = 860 | USE_GRAPHS = 1 | PEER_MAX_BATCH_SIZE = 128 | CPU : SSE3 = 1 | SSSE3 = 1 | AVX = 1 | AVX2 = 1 | F16C = 1 | FMA = 1 | BMI2 = 1 | AVX512 = 1 | AVX512_VBMI = 1 | AVX512_VNNI = 1 | AVX512_BF16 = 1 | LLAMAFILE = 1 | OPENMP = 1 | REPACK = 1 | 
+perplexity: tokenizing the input ..
+perplexity: tokenization took 43.338 ms
+perplexity: calculating perplexity over 14 chunks, n_ctx=2048, batch_size=2048, n_seq=1
+perplexity: 0.59 seconds per pass - ETA 0.13 minutes
+[1]18.6167,[2]21.6814,[3]22.3295,[4]20.2659,[5]20.2652,[6]18.0742,[7]17.7088,[8]17.6684,[9]18.1875,[10]18.1614,[11]18.0033,[12]18.1250,[13]18.1935,[14]18.2447,
+Final estimate: PPL = 18.2447 +/- 0.46970
+
+llama_perf_context_print:        load time =     231.26 ms
+llama_perf_context_print: prompt eval time =    5035.71 ms / 28672 tokens (    0.18 ms per token,  5693.74 tokens per second)
+llama_perf_context_print:        eval time =       0.00 ms /     1 runs   (    0.00 ms per token,      inf tokens per second)
+llama_perf_context_print:       total time =    5306.06 ms / 28673 tokens
+llama_perf_context_print:    graphs reused =          0
+llama_memory_breakdown_print: | memory breakdown [MiB] | total    free    self   model   context   compute    unaccounted |
+llama_memory_breakdown_print: |   - CUDA0 (RTX 3090)   | 24115 = 20444 + ( 393 =    98 +      10 +     284) +        3277 |
+llama_memory_breakdown_print: |   - CUDA1 (RTX 3090)   | 24124 = 23367 + ( 131 =    99 +       8 +      22) +         625 |
+llama_memory_breakdown_print: |   - Host               |                   226 =   197 +      10 +      18                |

Benchmarks/DataCollection/granite-4.0-h-350m-unsloth-mxfp4_moe-attn_kv_BF16-attn_output_BF16-attn_q_BF16-embeddings_Q8_0-ffn_down_BF16-ffn_up_gate_Q8_0/perplexity_math.log

@@ -0,0 +1,189 @@
+ggml_cuda_init: GGML_CUDA_FORCE_MMQ:    no
+ggml_cuda_init: GGML_CUDA_FORCE_CUBLAS: no
+ggml_cuda_init: found 2 CUDA devices:
+  Device 0: NVIDIA GeForce RTX 3090, compute capability 8.6, VMM: yes
+  Device 1: NVIDIA GeForce RTX 3090, compute capability 8.6, VMM: yes
+build: 7040 (92bb442ad) with cc (Ubuntu 13.3.0-6ubuntu2~24.04) 13.3.0 for x86_64-linux-gnu
+llama_model_load_from_file_impl: using device CUDA0 (NVIDIA GeForce RTX 3090) (0000:01:00.0) - 20928 MiB free
+llama_model_load_from_file_impl: using device CUDA1 (NVIDIA GeForce RTX 3090) (0000:03:00.0) - 23581 MiB free
+llama_model_loader: loaded meta data with 48 key-value pairs and 402 tensors from /mnt/world8/AI/ToBench/granite-4.0-h-350m-unsloth/Magic_Quant/GGUF/dc_round1_granite-4.0-h-350m-unsloth-mxfp4_moe-attn_kv_BF16-attn_output_BF16-attn_q_BF16-embeddings_Q8_0-ffn_down_BF16-ffn_up_gate_Q8_0.gguf (version GGUF V3 (latest))
+llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output.
+llama_model_loader: - kv   0:                       general.architecture str              = granitehybrid
+llama_model_loader: - kv   1:                               general.type str              = model
+llama_model_loader: - kv   2:                               general.name str              = Granite 4.0 H 350m Unsloth
+llama_model_loader: - kv   3:                           general.finetune str              = unsloth
+llama_model_loader: - kv   4:                           general.basename str              = granite-4.0-h
+llama_model_loader: - kv   5:                         general.size_label str              = 350M
+llama_model_loader: - kv   6:                            general.license str              = apache-2.0
+llama_model_loader: - kv   7:                   general.base_model.count u32              = 1
+llama_model_loader: - kv   8:                  general.base_model.0.name str              = Granite 4.0 H 350m
+llama_model_loader: - kv   9:          general.base_model.0.organization str              = Ibm Granite
+llama_model_loader: - kv  10:              general.base_model.0.repo_url str              = https://huggingface.co/ibm-granite/gr...
+llama_model_loader: - kv  11:                               general.tags arr[str,3]       = ["language", "unsloth", "granite-4.0"]
+llama_model_loader: - kv  12:                  granitehybrid.block_count u32              = 32
+llama_model_loader: - kv  13:               granitehybrid.context_length u32              = 1048576
+llama_model_loader: - kv  14:             granitehybrid.embedding_length u32              = 768
+llama_model_loader: - kv  15:          granitehybrid.feed_forward_length u32              = 2048
+llama_model_loader: - kv  16:         granitehybrid.attention.head_count u32              = 12
+llama_model_loader: - kv  17:      granitehybrid.attention.head_count_kv arr[i32,32]      = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, ...
+llama_model_loader: - kv  18:               granitehybrid.rope.freq_base f32              = 10000.000000
+llama_model_loader: - kv  19: granitehybrid.attention.layer_norm_rms_epsilon f32              = 0.000010
+llama_model_loader: - kv  20:                 granitehybrid.expert_count u32              = 0
+llama_model_loader: - kv  21:            granitehybrid.expert_used_count u32              = 0
+llama_model_loader: - kv  22:                   granitehybrid.vocab_size u32              = 100352
+llama_model_loader: - kv  23:         granitehybrid.rope.dimension_count u32              = 64
+llama_model_loader: - kv  24:              granitehybrid.attention.scale f32              = 0.015625
+llama_model_loader: - kv  25:              granitehybrid.embedding_scale f32              = 12.000000
+llama_model_loader: - kv  26:               granitehybrid.residual_scale f32              = 0.246000
+llama_model_loader: - kv  27:                  granitehybrid.logit_scale f32              = 3.000000
+llama_model_loader: - kv  28: granitehybrid.expert_shared_feed_forward_length u32              = 2048
+llama_model_loader: - kv  29:              granitehybrid.ssm.conv_kernel u32              = 4
+llama_model_loader: - kv  30:               granitehybrid.ssm.state_size u32              = 128
+llama_model_loader: - kv  31:              granitehybrid.ssm.group_count u32              = 1
+llama_model_loader: - kv  32:               granitehybrid.ssm.inner_size u32              = 1536
+llama_model_loader: - kv  33:           granitehybrid.ssm.time_step_rank u32              = 48
+llama_model_loader: - kv  34:       granitehybrid.rope.scaling.finetuned bool             = false
+llama_model_loader: - kv  35:                       tokenizer.ggml.model str              = gpt2
+llama_model_loader: - kv  36:                         tokenizer.ggml.pre str              = dbrx
+llama_model_loader: - kv  37:                      tokenizer.ggml.tokens arr[str,100352]  = ["!", "\"", "#", "$", "%", "&", "'", ...
+llama_model_loader: - kv  38:                  tokenizer.ggml.token_type arr[i32,100352]  = [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...
+llama_model_loader: - kv  39:                      tokenizer.ggml.merges arr[str,100000]  = ["Ġ Ġ", "ĠĠ ĠĠ", "i n", "Ġ t",...
+llama_model_loader: - kv  40:                tokenizer.ggml.bos_token_id u32              = 100257
+llama_model_loader: - kv  41:                tokenizer.ggml.eos_token_id u32              = 100257
+llama_model_loader: - kv  42:            tokenizer.ggml.unknown_token_id u32              = 100269
+llama_model_loader: - kv  43:            tokenizer.ggml.padding_token_id u32              = 100256
+llama_model_loader: - kv  44:               tokenizer.ggml.add_bos_token bool             = false
+llama_model_loader: - kv  45:                    tokenizer.chat_template str              = {%- set tools_system_message_prefix =...
+llama_model_loader: - kv  46:               general.quantization_version u32              = 2
+llama_model_loader: - kv  47:                          general.file_type u32              = 38
+llama_model_loader: - type  f32:  233 tensors
+llama_model_loader: - type q8_0:  121 tensors
+llama_model_loader: - type bf16:   48 tensors
+print_info: file format = GGUF V3 (latest)
+print_info: file type   = MXFP4 MoE
+print_info: file size   = 396.45 MiB (9.77 BPW) 
+load: printing all EOG tokens:
+load:   - 100257 ('<|end_of_text|>')
+load:   - 100261 ('<|fim_pad|>')
+load: special tokens cache size = 96
+load: token to piece cache size = 0.6152 MB
+print_info: arch             = granitehybrid
+print_info: vocab_only       = 0
+print_info: n_ctx_train      = 1048576
+print_info: n_embd           = 768
+print_info: n_embd_inp       = 768
+print_info: n_layer          = 32
+print_info: n_head           = 12
+print_info: n_head_kv        = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0]
+print_info: n_rot            = 64
+print_info: n_swa            = 0
+print_info: is_swa_any       = 0
+print_info: n_embd_head_k    = 64
+print_info: n_embd_head_v    = 64
+print_info: n_gqa            = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 3, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0]
+print_info: n_embd_k_gqa     = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 256, 0, 0, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 0, 0]
+print_info: n_embd_v_gqa     = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 256, 0, 0, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 0, 0]
+print_info: f_norm_eps       = 0.0e+00
+print_info: f_norm_rms_eps   = 1.0e-05
+print_info: f_clamp_kqv      = 0.0e+00
+print_info: f_max_alibi_bias = 0.0e+00
+print_info: f_logit_scale    = 3.0e+00
+print_info: f_attn_scale     = 1.6e-02
+print_info: n_ff             = 2048
+print_info: n_expert         = 0
+print_info: n_expert_used    = 0
+print_info: n_expert_groups  = 0
+print_info: n_group_used     = 0
+print_info: causal attn      = 1
+print_info: pooling type     = 0
+print_info: rope type        = 0
+print_info: rope scaling     = linear
+print_info: freq_base_train  = 10000.0
+print_info: freq_scale_train = 1
+print_info: n_ctx_orig_yarn  = 1048576
+print_info: rope_finetuned   = unknown
+print_info: ssm_d_conv       = 4
+print_info: ssm_d_inner      = 1536
+print_info: ssm_d_state      = 128
+print_info: ssm_dt_rank      = 48
+print_info: ssm_n_group      = 1
+print_info: ssm_dt_b_c_rms   = 0
+print_info: model type       = 350M
+print_info: model params     = 340.33 M
+print_info: general.name     = Granite 4.0 H 350m Unsloth
+print_info: f_embedding_scale = 12.000000
+print_info: f_residual_scale  = 0.246000
+print_info: f_attention_scale = 0.015625
+print_info: n_ff_shexp        = 2048
+print_info: vocab type       = BPE
+print_info: n_vocab          = 100352
+print_info: n_merges         = 100000
+print_info: BOS token        = 100257 '<|end_of_text|>'
+print_info: EOS token        = 100257 '<|end_of_text|>'
+print_info: EOT token        = 100257 '<|end_of_text|>'
+print_info: UNK token        = 100269 '<|unk|>'
+print_info: PAD token        = 100256 '<|pad|>'
+print_info: LF token         = 198 'Ċ'
+print_info: FIM PRE token    = 100258 '<|fim_prefix|>'
+print_info: FIM SUF token    = 100260 '<|fim_suffix|>'
+print_info: FIM MID token    = 100259 '<|fim_middle|>'
+print_info: FIM PAD token    = 100261 '<|fim_pad|>'
+print_info: EOG token        = 100257 '<|end_of_text|>'
+print_info: EOG token        = 100261 '<|fim_pad|>'
+print_info: max token length = 256
+load_tensors: loading model tensors, this can take a while... (mmap = true)
+load_tensors: offloading 20 repeating layers to GPU
+load_tensors: offloaded 20/33 layers to GPU
+load_tensors:   CPU_Mapped model buffer size =   197.91 MiB
+load_tensors:        CUDA0 model buffer size =    98.85 MiB
+load_tensors:        CUDA1 model buffer size =    99.71 MiB
+..................................................................................
+llama_context: constructing llama_context
+llama_context: n_seq_max     = 1
+llama_context: n_ctx         = 2048
+llama_context: n_ctx_seq     = 2048
+llama_context: n_batch       = 2048
+llama_context: n_ubatch      = 512
+llama_context: causal_attn   = 1
+llama_context: flash_attn    = auto
+llama_context: kv_unified    = false
+llama_context: freq_base     = 10000.0
+llama_context: freq_scale    = 1
+llama_context: n_ctx_seq (2048) < n_ctx_train (1048576) -- the full capacity of the model will not be utilized
+llama_context:        CPU  output buffer size =     0.38 MiB
+llama_kv_cache:        CPU KV buffer size =     2.00 MiB
+llama_kv_cache:      CUDA0 KV buffer size =     4.00 MiB
+llama_kv_cache:      CUDA1 KV buffer size =     2.00 MiB
+llama_kv_cache: size =    8.00 MiB (  2048 cells,   4 layers,  1/1 seqs), K (f16):    4.00 MiB, V (f16):    4.00 MiB
+llama_memory_recurrent:        CPU RS buffer size =     8.48 MiB
+llama_memory_recurrent:      CUDA0 RS buffer size =     6.16 MiB
+llama_memory_recurrent:      CUDA1 RS buffer size =     6.93 MiB
+llama_memory_recurrent: size =   21.57 MiB (     1 cells,  32 layers,  1 seqs), R (f32):    0.57 MiB, S (f32):   21.00 MiB
+llama_context: Flash Attention was auto, set to enabled
+llama_context:      CUDA0 compute buffer size =   283.69 MiB
+llama_context:      CUDA1 compute buffer size =    22.39 MiB
+llama_context:  CUDA_Host compute buffer size =    18.34 MiB
+llama_context: graph nodes  = 1815
+llama_context: graph splits = 182 (with bs=512), 41 (with bs=1)
+common_init_from_params: added <|end_of_text|> logit bias = -inf
+common_init_from_params: added <|fim_pad|> logit bias = -inf
+common_init_from_params: setting dry_penalty_last_n to ctx_size = 2048
+common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable)
+
+system_info: n_threads = 16 (n_threads_batch = 16) / 32 | CUDA : ARCHS = 860 | USE_GRAPHS = 1 | PEER_MAX_BATCH_SIZE = 128 | CPU : SSE3 = 1 | SSSE3 = 1 | AVX = 1 | AVX2 = 1 | F16C = 1 | FMA = 1 | BMI2 = 1 | AVX512 = 1 | AVX512_VBMI = 1 | AVX512_VNNI = 1 | AVX512_BF16 = 1 | LLAMAFILE = 1 | OPENMP = 1 | REPACK = 1 | 
+perplexity: tokenizing the input ..
+perplexity: tokenization took 34.926 ms
+perplexity: calculating perplexity over 15 chunks, n_ctx=2048, batch_size=2048, n_seq=1
+perplexity: 0.58 seconds per pass - ETA 0.13 minutes
+[1]8.7503,[2]9.9529,[3]9.5020,[4]9.8516,[5]9.9959,[6]10.0948,[7]10.2524,[8]9.9473,[9]10.0075,[10]10.0191,[11]10.2583,[12]10.3453,[13]10.4630,[14]10.4358,[15]10.3383,
+Final estimate: PPL = 10.3383 +/- 0.23316
+
+llama_perf_context_print:        load time =     226.46 ms
+llama_perf_context_print: prompt eval time =    5370.16 ms / 30720 tokens (    0.17 ms per token,  5720.50 tokens per second)
+llama_perf_context_print:        eval time =       0.00 ms /     1 runs   (    0.00 ms per token,      inf tokens per second)
+llama_perf_context_print:       total time =    5645.40 ms / 30721 tokens
+llama_perf_context_print:    graphs reused =          0
+llama_memory_breakdown_print: | memory breakdown [MiB] | total    free    self   model   context   compute    unaccounted |
+llama_memory_breakdown_print: |   - CUDA0 (RTX 3090)   | 24115 = 20452 + ( 393 =    98 +      10 +     284) +        3269 |
+llama_memory_breakdown_print: |   - CUDA1 (RTX 3090)   | 24124 = 23367 + ( 131 =    99 +       8 +      22) +         625 |
+llama_memory_breakdown_print: |   - Host               |                   226 =   197 +      10 +      18                |