File size: 948 Bytes
797a614 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 | # Quantization recipe — harrier-oss-v1-0.6b/fp8
base_model: microsoft/harrier-oss-v1-0.6b
base_revision: f9b9dc8d367d443f2479d27aa5d8d2850c0774ee
variant: fp8
toolchain: llm-compressor==0.13.0 (compressed-tensors==0.18.0)
calibration: collections/harrier-oss-v1-0.6b/calibration/manifest.json (448 samples, seed 20260818973, audit status OK)
recipe: |
# llm-compressor recipe, applied via llmcompressor.oneshot(recipe=recipe, ...)
from llmcompressor.modifiers.quantization import QuantizationModifier
recipe = QuantizationModifier(
targets="Linear",
scheme="FP8_DYNAMIC",
ignore=["lm_head", "re:.*embed_tokens.*", "re:.*norm.*"],
)
# oneshot(model="bf16", recipe=recipe, dataset=calibration_dataset,
# text_column="text", num_calibration_samples=448, max_seq_length=128,
# trust_remote_code_model=True, output_dir="fp8")
driver_script: collections/harrier-oss-v1-0.6b/quantization/quantize_fp8.py
|