File size: 948 Bytes
ea9105e | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 | # Quantization recipe — harrier-oss-v1-270m/fp8
base_model: microsoft/harrier-oss-v1-270m
base_revision: 31de22b673913c7d658c0f03f792d77c2dcf8ebd
variant: fp8
toolchain: llm-compressor==0.13.0 (compressed-tensors==0.18.0)
calibration: collections/harrier-oss-v1-270m/calibration/manifest.json (448 samples, seed 20260818871, audit status OK)
recipe: |
# llm-compressor recipe, applied via llmcompressor.oneshot(recipe=recipe, ...)
from llmcompressor.modifiers.quantization import QuantizationModifier
recipe = QuantizationModifier(
targets="Linear",
scheme="FP8_DYNAMIC",
ignore=["lm_head", "re:.*embed_tokens.*", "re:.*norm.*"],
)
# oneshot(model="bf16", recipe=recipe, dataset=calibration_dataset,
# text_column="text", num_calibration_samples=448, max_seq_length=128,
# trust_remote_code_model=True, output_dir="fp8")
driver_script: collections/harrier-oss-v1-270m/quantization/quantize_fp8.py
|