语言 中文|[English](https://huggingface.co/YCWTG/gemma-4-31B-it-NVFP4A16-GPTQ) ## 模型详情 这是 [google/gemma-4-31B-it](https://huggingface.co/google/gemma-4-31B-it) 的一个 **NVFP4A16 量化** 的版本，由 [llm-compressor](https://github.com/vllm-project/llm-compressor) 生成。请遵循原始模型的 license。 如需使用 thinking mode，请依次执行以下步骤： 1. 启动服务时添加 `--reasoning-parser gemma4`。 2. 通过添加 `--default-chat-template-kwargs '{"enable_thinking": true}'` 启用 thinking，或者直接在 [chat_template.jinja](https://huggingface.co/YCWTG/gemma-4-31B-it-NVFP4A16-GPTQ/blob/main/chat_template.jinja) 中设置 `{%- set enable_thinking = true %}`。 示例请参见 [Thinking Mode](#思考模式)。 ### 量化策略（Intel MoE Recipe） | Layer Type | Bits | Notes | | ---------------- | ------ | ------------------------------------------------------ | | `lm_head` | 16-bit | 保持原始精度，以尽量保留最终 token prediction 的质量，并避免在输出投影阶段引入额外精度损失 | | `vision_tower.*` | 16-bit | 保持原始精度，以更好保留视觉特征提取能力，并减少多模态能力退化 | | `embed_vision.*` | 16-bit | 保持原始精度，以维持 vision embedding 的保真度，并减少视觉特征处理前的量化误差 | ## 快速开始 ### vLLM 使用方式 [vLLM](https://github.com/vllm-project/vllm) 是一个面向 LLM 的高吞吐、内存高效的 inference 与 serving 引擎。 直接与模型对话 ```python import argparse import atexit import json import os import shutil import subprocess import sys import time import urllib.error import urllib.request # --------------------------- # 用户可配置项 # --------------------------- DEFAULTS = { "model": "YCWTG/gemma-4-31B-it-NVFP4A16-GPTQ", "served_model_name": "YCWTG/gemma-4-31B-it-NVFP4A16-GPTQ", "host": "localhost", "port": 8000, "max_model_len": 66464, "enable_auto_tool_choice": True, "async_scheduling": True, "tool_call_parser": "gemma4", "max_num_seqs": 1, "reasoning_parser": "gemma4", "default_chat_template_kwargs": '{"enable_thinking": true}', "allowed_local_media_path": "/home/ycwtg/image", } RUNTIME = { "gpu_memory_utilization": 0.97, "startup_timeout_sec": 180, "healthcheck_timeout_sec": 3, "healthcheck_interval_sec": 1, "chat_timeout_sec": 600, } SERVE_VALUE_OPTIONS = ( ("--served-model-name", "served_model_name"), ("--host", "host"), ("--port", "port"), ("--max-model-len", "max_model_len"), ("--tool-call-parser", "tool_call_parser"), ("--max_num_seqs", "max_num_seqs"), ("--reasoning-parser", "reasoning_parser"), ("--default-chat-template-kwargs", "default_chat_template_kwargs"), ) CLIENT_VALUE_OPTIONS = ( ("--model", "model"), *SERVE_VALUE_OPTIONS, ) SERVE_BOOL_OPTIONS = ( ("--enable-auto-tool-choice", "enable_auto_tool_choice"), ("--async-scheduling", "async_scheduling"), ) CLIENT_BOOL_OPTIONS = ( ("--enable-auto-tool-choice", "--no-enable-auto-tool-choice", "enable_auto_tool_choice"), ("--async-scheduling", "--no-async-scheduling", "async_scheduling"), ) def append_value_options(cmd, args, options): for flag, attr in options: cmd.extend([flag, str(getattr(args, attr))]) def append_true_bool_options(cmd, args, options): for flag, attr in options: if getattr(args, attr): cmd.append(flag) def append_boolean_optional_options(cmd, args, options): for positive_flag, negative_flag, attr in options: cmd.append(positive_flag if getattr(args, attr) else negative_flag) def append_optional_value_option(cmd, args, flag, attr): value = getattr(args, attr) if value is None: return if isinstance(value, str) and not value.strip(): return cmd.extend([flag, str(value)]) def multiline_input(): print('用户（单独一行输入 "END" 发送，输入 "exit" 退出）：') lines = [] while True: line = input() text = line.strip() if text.lower() in {"exit", "quit"}: return None if text == "END": break lines.append(line) return "\n".join(lines) def resolve_client_host(host): return "127.0.0.1" if host in {"0.0.0.0", "::"} else host def launch_vllm(args): cmd = ["vllm", "serve", args.model] append_value_options(cmd, args, SERVE_VALUE_OPTIONS) append_optional_value_option(cmd, args, "--allowed-local-media-path", "allowed_local_media_path") cmd.extend( [ "--gpu-memory-utilization", str(RUNTIME["gpu_memory_utilization"]), ] ) append_true_bool_options(cmd, args, SERVE_BOOL_OPTIONS) print("启动 vLLM：") print(" ".join(cmd)) try: return subprocess.Popen(cmd) except FileNotFoundError as e: raise RuntimeError("未找到 vllm 命令，请先激活包含 vllm 的 Python 环境。") from e def stop_vllm(proc): if proc and proc.poll() is None: proc.terminate() try: proc.wait(timeout=10) except subprocess.TimeoutExpired: proc.kill() def wait_vllm_ready(base_url, timeout_sec=RUNTIME["startup_timeout_sec"]): deadline = time.time() + timeout_sec url = f"{base_url}/v1/models" req = urllib.request.Request(url=url) while time.time() < deadline: try: with urllib.request.urlopen(req, timeout=RUNTIME["healthcheck_timeout_sec"]) as resp: if resp.status == 200: return True except urllib.error.URLError: pass time.sleep(RUNTIME["healthcheck_interval_sec"]) return False def chat_once(base_url, model_name, messages): payload = {"model": model_name, "messages": messages, "skip_special_tokens": False} req = urllib.request.Request( url=f"{base_url}/v1/chat/completions", data=json.dumps(payload, ensure_ascii=False).encode("utf-8"), headers={"Content-Type": "application/json"}, method="POST", ) with urllib.request.urlopen(req, timeout=RUNTIME["chat_timeout_sec"]) as resp: data = json.loads(resp.read().decode("utf-8")) return data["choices"][0]["message"] def chat_loop(base_url, model_name): print("\n===== 对话已开始 =====\n") messages = [] while True: user_text = multiline_input() if user_text is None: break messages.append({"role": "user", "content": user_text}) try: assistant_msg = chat_once(base_url, model_name, messages) except Exception as e: print(f"\n请求失败: {e}\n") messages.pop() continue content = assistant_msg.get("content") tool_calls = assistant_msg.get("tool_calls") if content: print(f"\nAssistant:\n{content}\n") elif tool_calls: print("\nAssistant(tool_calls):") print(json.dumps(tool_calls, ensure_ascii=False, indent=2)) print() else: print("\nAssistant:\n(空回复)\n") normalized_msg = {"role": "assistant", "content": content or ""} if tool_calls: normalized_msg["tool_calls"] = tool_calls messages.append(normalized_msg) def build_client_command(args): cmd = [sys.executable, os.path.abspath(__file__), "--_client"] append_value_options(cmd, args, CLIENT_VALUE_OPTIONS) append_boolean_optional_options(cmd, args, CLIENT_BOOL_OPTIONS) return cmd def spawn_chat_terminal(args): client_cmd = build_client_command(args) terminal_cmd = None if os.name == "nt": # 在 Windows 上开启新的 cmd 窗口并保持打开 terminal_cmd = [ "cmd", "/c", "start", "", "cmd", "/k", subprocess.list2cmdline(client_cmd), ] elif shutil.which("gnome-terminal"): terminal_cmd = ["gnome-terminal", "--", *client_cmd] elif shutil.which("x-terminal-emulator"): terminal_cmd = ["x-terminal-emulator", "-e", *client_cmd] if not terminal_cmd: return False try: subprocess.Popen(terminal_cmd) return True except Exception as e: print(f"自动打开新终端失败: {e}") return False def parse_args(): parser = argparse.ArgumentParser(description="最简 vLLM 本地对话脚本") parser.add_argument("--_client", action="store_true", help=argparse.SUPPRESS) parser.add_argument("--model", default=DEFAULTS["model"]) parser.add_argument( "--served-model-name", default=DEFAULTS["served_model_name"], ) parser.add_argument("--host", default=DEFAULTS["host"]) parser.add_argument("--port", type=int, default=DEFAULTS["port"]) parser.add_argument("--max-model-len", type=int, default=DEFAULTS["max_model_len"]) parser.add_argument( "--max-num-seqs", "--max_num_seqs", dest="max_num_seqs", type=int, default=DEFAULTS["max_num_seqs"], ) parser.add_argument( "--enable-auto-tool-choice", action=argparse.BooleanOptionalAction, default=DEFAULTS["enable_auto_tool_choice"], ) parser.add_argument( "--async-scheduling", action=argparse.BooleanOptionalAction, default=DEFAULTS["async_scheduling"], ) parser.add_argument( "--allowed-local-media-path", default=DEFAULTS["allowed_local_media_path"], help="可选本地媒体目录路径。不填即不启用。", ) parser.add_argument("--tool-call-parser", default=DEFAULTS["tool_call_parser"]) parser.add_argument("--reasoning-parser", default=DEFAULTS["reasoning_parser"]) parser.add_argument( "--default-chat-template-kwargs", default=DEFAULTS["default_chat_template_kwargs"], ) return parser.parse_args() def main(): args = parse_args() base_url = f"http://{resolve_client_host(args.host)}:{args.port}" if args._client: chat_loop(base_url, args.served_model_name) return proc = launch_vllm(args) atexit.register(stop_vllm, proc) print(f"等待服务就绪: {base_url}") if not wait_vllm_ready(base_url): print("vLLM 启动超时，请检查上面的日志。") stop_vllm(proc) sys.exit(1) if spawn_chat_terminal(args): print("模型已就绪，已自动打开新终端用于对话。当前终端保留 vLLM 日志。") print("在当前终端按 Ctrl+C 可停止服务。") try: proc.wait() except KeyboardInterrupt: print("\n收到中断，正在停止 vLLM...") else: print("未找到可用终端，回退到当前终端对话模式。") chat_loop(base_url, args.served_model_name) if __name__ == "__main__": main() ``` 直接使用OpenAPI ### 非思考模式 ```shell vllm serve YCWTG/gemma-4-31B-it-NVFP4A16-GPTQ --served-model-name YCWTG/gemma-4-31B-it-NVFP4A16-GPTQ --host localhost --port 8000 --async-scheduling --max-model-len 66464 --enable-auto-tool-choice --tool-call-parser gemma4 --gpu-memory-utilization 0.97 --max_num_seqs 1 --allowed-local-media-path /home/ycwtg/image ``` ### 思考模式 ```shell vllm serve YCWTG/gemma-4-31B-it-NVFP4A16-GPTQ --served-model-name YCWTG/gemma-4-31B-it-NVFP4A16-GPTQ --host localhost --port 8000 --async-scheduling --max-model-len 66464 --enable-auto-tool-choice --tool-call-parser gemma4 --gpu-memory-utilization 0.97 --max_num_seqs 1 --allowed-local-media-path /home/ycwtg/image --reasoning-parser gemma4 --default-chat-template-kwargs '{"enable_thinking": true}' ``` 执行上述命令后，将在 `http://localhost:8000/v1` 创建 API endpoints。 更多详细信息请参阅其 [documentation](https://docs.vllm.ai/en/stable/getting_started/installation/index.html) ## 生成模型 [生成代码](https://huggingface.co/YCWTG/gemma-4-31B-it-NVFP4A16-GPTQ/blob/main/generate_model.py) ## 伦理考量与局限性 该模型可能会生成事实不准确的输出，因此不应被依赖用于提供绝对准确的信息。由于 pretrained model 以及 finetuning datasets 的局限性，模型可能生成带有低俗（lewd）、偏见（biased）或其他具有冒犯性的内容。 因此，在部署任何基于该模型的应用之前，开发者应进行安全测试。 ## 免责声明 本模型的 license 不构成法律建议。对于第三方使用本模型所产生的行为，我们不承担责任。在将该模型用于商业用途之前，请咨询律师。