Update README.md
Browse files
README.md
CHANGED
|
@@ -18,7 +18,7 @@ pip install git+https://github.com/huggingface/transformers.git
|
|
| 18 |
|
| 19 |
|
| 20 |
~~~bash
|
| 21 |
-
vllm serve Intel/Qwen3.5-35B-A3B-int4-AutoRound --port 8000 --tensor-parallel-size 1 --max-model-len 2048 --reasoning-parser qwen3 --served-model-name qwen
|
| 22 |
~~~
|
| 23 |
|
| 24 |
~~~bash
|
|
@@ -96,7 +96,7 @@ The user wants a short description of the image.
|
|
| 96 |
This pr is required https://github.com/intel/auto-round/pull/1476
|
| 97 |
|
| 98 |
~~~bash
|
| 99 |
-
auto-round "Qwen/Qwen3.5-35B-A3B" --output_dir "./Qwen35-int4" --ignore_layers shared_expert
|
| 100 |
~~~
|
| 101 |
|
| 102 |
|
|
|
|
| 18 |
|
| 19 |
|
| 20 |
~~~bash
|
| 21 |
+
vllm serve Intel/Qwen3.5-35B-A3B-int4-AutoRound --port 8000 --tensor-parallel-size 1 --max-model-len 2048 --reasoning-parser qwen3 --served-model-name qwen --speculative-config '{"method":"qwen3_next_mtp","num_speculative_tokens":2}'
|
| 22 |
~~~
|
| 23 |
|
| 24 |
~~~bash
|
|
|
|
| 96 |
This pr is required https://github.com/intel/auto-round/pull/1476
|
| 97 |
|
| 98 |
~~~bash
|
| 99 |
+
auto-round "Qwen/Qwen3.5-35B-A3B" --output_dir "./Qwen35-int4" --ignore_layers shared_expert,mtp.fc
|
| 100 |
~~~
|
| 101 |
|
| 102 |
|