RedHatAI/Kimi-K3-FP8-BLOCK

This is a variant of moonshotai/Kimi-K3 quantized with FP8_BLOCK quantization applied to MLP and experts layers. Attention layers were not quantized due to shape mismatches, namely with self_attn.b_proj.weight

Creation Process

Quantized using llm-compressor:

from compressed_tensors.entrypoints.convert import CompressedTensorsDequantizer

from llmcompressor import model_free_ptq

MODEL_ID = "moonshotai/Kimi-K3"
SAVE_DIR = MODEL_ID.rstrip("/").split("/")[-1] + "-FP8-BLOCK"

ignore = [
    "re:.*embed_tokens.*",
    "re:.*self_attn.*",  # no attention because (q_proj|k_proj|v_proj|b_proj|f_a_proj) are all fused, and `b_proj` has a weight non-divisible by 128
    "re:.*block_sparse_moe\.gate.*",
    "re:.*self_attention_res_proj.*",
    "re:.*mlp_res_proj.*",
    "re:.*output_attn_res_proj.*",
    "re:.*lm_head.*",
    "re:.*vision_tower.*",
    "re:.*mm_projector.*",
]

model_free_ptq(
    model_stub=MODEL_ID,
    save_directory=SAVE_DIR,
    scheme="FP8_BLOCK",
    ignore=ignore,
    converter=CompressedTensorsDequantizer(
        MODEL_ID,
        ignore=ignore,
    ),
    max_workers=7,
    device=[
        f"cuda:{i}"
        for i in range(7)
    ],
)