| # Quantization recipe — harrier-oss-v1-0.6b/fp8 | |
| base_model: microsoft/harrier-oss-v1-0.6b | |
| base_revision: f9b9dc8d367d443f2479d27aa5d8d2850c0774ee | |
| variant: fp8 | |
| toolchain: llm-compressor==0.13.0 (compressed-tensors==0.18.0) | |
| calibration: collections/harrier-oss-v1-0.6b/calibration/manifest.json (448 samples, seed 20260818973, audit status OK) | |
| recipe: | | |
| # llm-compressor recipe, applied via llmcompressor.oneshot(recipe=recipe, ...) | |
| from llmcompressor.modifiers.quantization import QuantizationModifier | |
| recipe = QuantizationModifier( | |
| targets="Linear", | |
| scheme="FP8_DYNAMIC", | |
| ignore=["lm_head", "re:.*embed_tokens.*", "re:.*norm.*"], | |
| ) | |
| # oneshot(model="bf16", recipe=recipe, dataset=calibration_dataset, | |
| # text_column="text", num_calibration_samples=448, max_seq_length=128, | |
| # trust_remote_code_model=True, output_dir="fp8") | |
| driver_script: collections/harrier-oss-v1-0.6b/quantization/quantize_fp8.py | |