import json
from collections.abc import Sequence
from random import choices
from string import ascii_letters, digits
from typing import Optional, Union, Any
import partial_json_parser
import regex as re
from partial_json_parser.core.options import Allow
from pydantic import Field
from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
DeltaFunctionCall, DeltaMessage,
DeltaToolCall,
ExtractedToolCallInformation,
FunctionCall, ToolCall)
from vllm.logger import init_logger
try:
from vllm.tool_parsers import ToolParser, ToolParserManager
except ImportError:
from vllm.entrypoints.openai.tool_parsers import ToolParser, ToolParserManager
try:
from vllm.tokenizers import TokenizerLike
except ImportError:
from vllm.transformers_utils.tokenizer import AnyTokenizer as TokenizerLike
logger = init_logger(__name__)
ALPHANUMERIC = ascii_letters + digits
# Common exception types for JSON parsing attempts
JSON_PARSE_EXCEPTIONS = (
partial_json_parser.MalformedJSON,
json.JSONDecodeError,
ValueError,
AssertionError,
)
class NemotronToolCall(ToolCall):
id: str = Field(
default_factory=lambda: NemotronToolCall.generate_random_id())
@staticmethod
def generate_random_id():
return "".join(choices(ALPHANUMERIC, k=9))
@staticmethod
def is_valid_id(id: str) -> bool:
return id.isalnum() and len(id) == 9
@ToolParserManager.register_module("nemotron_json")
class NemotronToolParser(ToolParser):
"""
Tool call parser for Nemotron-Nano-V2
Used when --enable-auto-tool-choice --tool-call-parser nemotron_json are all set
"""
def __init__(self, tokenizer: TokenizerLike):
super().__init__(tokenizer)
# initialize properties used for state when parsing tool calls in streaming mode
self.prev_tool_call_arr: list[dict] = []
self.current_tool_id: int = -1
self.current_tool_name_sent: bool = False
# map what has been streamed for each tool so far to a list
self.streamed_args_for_tool: list[str] = []
self.tool_args_emitted: list[bool] = []
self.bot_token = ""
self.eot_token = ""
# regex patterns for non-streaming parsing
self.tool_calls_regex = re.compile(r"(.*?)", re.DOTALL)
self.tool_call_regex = re.compile(r'(^\[|, ?)({"name": ?"[^"]*", ?"arguments":)', re.DOTALL)
# Buffer for partial tag sequences to disambiguate between normal content and
# a forthcoming or tag in streaming.
self._pending_tag_buffer: str = ""
# Track tool calls already parsed in fallback mode (by name and position)
# to avoid re-parsing the same tool calls multiple times during streaming
self._fallback_parsed_tools: list[tuple[str, int]] = [] # List of (name, start_pos) tuples
# Track the number of tool calls that were successfully parsed by standard parser
# to skip re-parsing them in fallback mode
self._last_standard_parse_count: int = 0
@staticmethod
def _strip_trailing_auto_closers(chunk: str) -> str:
"""
Remove parser auto-completed closing braces/brackets plus trailing whitespace.
These should be flushed only when a tool call completes to avoid duplicate
argument fragments.
"""
idx = len(chunk)
while idx > 0 and chunk[idx - 1] in " \t\r\n}]":
idx -= 1
# Remove trailing non-escaped double quotes (partial JSON auto-closes strings)
while idx > 0 and chunk[idx - 1] == '"':
# keep escaped quotes (\"), only strip bare ones
if idx - 2 >= 0 and chunk[idx - 2] == '\\':
break
idx -= 1
return chunk[:idx]
@staticmethod
def _common_prefix_len(left: str, right: str) -> int:
"""
Return the length of the shared prefix between left and right strings.
"""
max_len = min(len(left), len(right))
idx = 0
while idx < max_len and left[idx] == right[idx]:
idx += 1
return idx
@staticmethod
def _skip_whitespace_and_comma(content: str, pos: int) -> int:
"""Skip comma and whitespace characters, return new position."""
while pos < len(content) and content[pos] in ', \t\r\n':
pos += 1
return pos
@staticmethod
def _skip_json_object(content: str, start_pos: int) -> int:
"""
Skip past a JSON object by counting braces while respecting strings.
Assumes start_pos is positioned just after the opening '{'.
Returns position after the closing '}' (or end of content if unmatched).
"""
pos = start_pos
depth = 1
while pos < len(content) and depth > 0:
ch = content[pos]
if ch == '{':
depth += 1
elif ch == '}':
depth -= 1
elif ch == '"':
# Skip string content (handles escaped characters)
pos += 1
while pos < len(content):
if content[pos] == '\\' and pos + 1 < len(content):
pos += 2
continue
if content[pos] == '"':
break
pos += 1
pos += 1
return pos
@staticmethod
def _get_param_config(request: ChatCompletionRequest) -> dict[str, dict[str, str]]:
tool_param_config = {}
if not request.tools:
return tool_param_config
for tool in request.tools:
if not (hasattr(tool, "function") or hasattr(tool.function, "parameters")):
continue
param_config = {}
for param_name, param_type in tool.function.parameters.items():
if isinstance(param_type, dict) and "type" in param_type:
param_config[param_name] = param_type["type"]
tool_param_config[tool.function.name] = param_config
return tool_param_config
@staticmethod
def _convert_param_value(value: Union[str, Any], param_type: str) -> Any:
"""Convert parameter value to the correct type."""
param_type = param_type.lower()
if param_type in ["string", "str", "text"]:
try:
if isinstance(value, str):
return value
elif isinstance(value, dict) or isinstance(value, list):
return json.dumps(value, ensure_ascii=False)
elif value is not None:
return str(value)
else:
return value
except:
return value
if isinstance(value, str) and value.lower() == "null":
return None
if param_type in ["integer", "int"]:
try:
return int(value)
except (ValueError, TypeError):
return value
elif param_type in ["number", "float"]:
try:
val = float(value)
return val if val != int(val) else int(val)
except (ValueError, TypeError):
return value
elif isinstance(value, str) and param_type in ["boolean", "bool"]:
return value.lower() in ["true", "1"]
elif isinstance(value, str) and param_type in ["object", "array"]:
try:
return json.loads(value)
except json.JSONDecodeError:
return value
else:
# Try JSON parse first, fallback to string
try:
return json.loads(value)
except (json.JSONDecodeError, TypeError):
return value
def _compute_arguments_delta(self, cur_arguments_json: str,
end_of_call: bool) -> str:
"""
Determine the incremental suffix to stream for the current tool call.
Ensures we only emit monotonic chunks by trimming our tracked prefix to
the longest common prefix with the latest JSON snapshot.
"""
tool_idx = self.current_tool_id
if tool_idx < 0 or tool_idx >= len(self.streamed_args_for_tool):
return ""
streamed_prefix = self.streamed_args_for_tool[tool_idx]
had_any = (self.tool_args_emitted[tool_idx]
if tool_idx < len(self.tool_args_emitted) else False)
lcp_len = self._common_prefix_len(cur_arguments_json,
streamed_prefix)
if lcp_len != len(streamed_prefix):
streamed_prefix = streamed_prefix[:lcp_len]
self.streamed_args_for_tool[tool_idx] = streamed_prefix
if (not had_any and not end_of_call and lcp_len == 0
and cur_arguments_json.endswith('": ""}')
and '": ""' in cur_arguments_json):
closing_pos = cur_arguments_json.rfind('": ""}')
if closing_pos != -1:
arguments_delta = cur_arguments_json[:closing_pos + 4]
else:
arguments_delta = cur_arguments_json
else:
arguments_delta = cur_arguments_json[lcp_len:]
if not arguments_delta:
return ""
if not end_of_call:
arguments_delta = self._strip_trailing_auto_closers(
arguments_delta)
if (not had_any and not end_of_call and arguments_delta
and arguments_delta.endswith('}')):
arguments_delta = arguments_delta[:-1]
if arguments_delta.endswith('"'):
arguments_delta = arguments_delta[:-1]
return arguments_delta
def _visible_delta_outside_tool(self, delta_text: str,
start_token: Optional[str],
end_token: Optional[str]) -> str:
"""
Consume characters that could begin a tool tag. Only suppress the exact
/ sequences, and let everything else (e.g. )
pass through untouched.
"""
if not delta_text:
return delta_text
visible: list[str] = []
for ch in delta_text:
if self._pending_tag_buffer or ch == '<':
self._pending_tag_buffer += ch
if start_token and start_token.startswith(self._pending_tag_buffer):
if self._pending_tag_buffer == start_token:
self._pending_tag_buffer = ""
continue
if end_token and end_token.startswith(self._pending_tag_buffer):
if self._pending_tag_buffer == end_token:
self._pending_tag_buffer = ""
continue
# Not a tool tag; flush buffered characters as normal content.
visible.append(self._pending_tag_buffer)
self._pending_tag_buffer = ""
else:
visible.append(ch)
return "".join(visible)
def adjust_request(self, request: ChatCompletionRequest) -> ChatCompletionRequest:
if request.tools and request.tool_choice != 'none':
# Do not skip special tokens when using chat template
# with Mistral parser as TOOL_CALL token is needed
# for tool detection.
request.skip_special_tokens = False
return request
def _parse_tool_calls_fallback(self, parsable_arr: str) -> list[dict]:
"""
Fall back to incremental parsing with state tracking.
Tool calls must follow: {"name": "...", "arguments": {...}}
- Parse incrementally from each '{'
- Confirm "name" key appears first
- Then confirm "arguments" key appears second
- If state transitions are violated, skip to next '{'
- When parse fails (malformed JSON), use last successful parse as one tool call
"""
tool_call_arr: list[dict] = []
content = parsable_arr.strip()
if content.startswith('['):
content = content[1:]
if content.endswith(']'):
content = content[:-1]
content = content.strip()
if not content:
return tool_call_arr
current_pos = 0
# Optimization: If standard parser previously succeeded with N tool calls,
# use prev_tool_call_arr for those and skip to the (N+1)-th tool call position
if self._last_standard_parse_count > 0 and len(self.prev_tool_call_arr) >= self._last_standard_parse_count:
# Copy tool calls from previous successful parse
tool_call_arr = list(self.prev_tool_call_arr[:self._last_standard_parse_count])
# Skip past the first N '{' positions to find where to start incremental parsing
skipped = 0
while current_pos < len(content) and skipped < self._last_standard_parse_count:
brace_pos = content.find('{', current_pos)
if brace_pos == -1:
break
# Track this position as already parsed
if not any(pos == brace_pos for _, pos in self._fallback_parsed_tools):
tool_name = tool_call_arr[skipped].get('name', '') if skipped < len(tool_call_arr) else ''
self._fallback_parsed_tools.append((tool_name, brace_pos))
skipped += 1
current_pos = self._skip_json_object(content, brace_pos + 1)
current_pos = self._skip_whitespace_and_comma(content, current_pos)
while current_pos < len(content):
# Find next '{' from current position
brace_pos = content.find('{', current_pos)
if brace_pos == -1:
break
# Check if this position was already parsed in a previous streaming call
if any(pos == brace_pos for _, pos in self._fallback_parsed_tools):
current_pos = brace_pos + 1
continue
remaining = content[brace_pos:]
parsed_result = self._try_incremental_tool_parse(remaining)
if parsed_result is None:
# Not a valid tool call start, skip to next '{'
current_pos = brace_pos + 1
continue
tool_call, end_pos = parsed_result
if tool_call:
tool_call_arr.append(tool_call)
tool_name = tool_call.get('name', '')
self._fallback_parsed_tools.append((tool_name, brace_pos))
current_pos = brace_pos + end_pos
current_pos = self._skip_whitespace_and_comma(content, current_pos)
else:
# No valid tool call found from this '{', try next
current_pos = brace_pos + 1
return tool_call_arr
def _try_incremental_tool_parse(self, json_str: str) -> Optional[tuple[Optional[dict], int]]:
"""
Attempt to incrementally parse a tool call from JSON string.
Returns:
- (parsed_dict, end_pos) if a valid tool call was found
- (None, 0) if no valid tool call found but should continue
- None if the JSON structure doesn't look like a tool call (skip to next '{')
"""
last_valid_parse = None
last_valid_end = 0
for end_pos in range(1, len(json_str) + 1):
test_json = json_str[:end_pos]
try:
parsed = partial_json_parser.loads(test_json, Allow.ALL)
if parsed and isinstance(parsed, dict):
keys = list(parsed.keys())
if len(keys) == 0:
# Empty object, continue
continue
# First key must be 'name'
if keys[0] != 'name':
# Not a valid tool call start
return None
# If there are more keys, second must be 'arguments'
if len(keys) > 1 and keys[1] != 'arguments':
# Not a valid tool call
return None
# Valid state - track if we have both name and arguments
if 'name' in parsed and 'arguments' in parsed:
last_valid_parse = parsed
last_valid_end = end_pos
except JSON_PARSE_EXCEPTIONS:
# Parse failed - if we have a valid parse with both keys, use it
if last_valid_parse:
break
# Otherwise continue trying (might be incomplete JSON that becomes valid)
continue
return (last_valid_parse, last_valid_end)
def _try_single_tool_call_parse(self, str_tool_call: str) -> dict:
fix_brace = lambda s: s + '}'
fix_quote = lambda s: s.replace(r'\""', '"').replace(r'"\"', '"').replace(r"\'", "'")
for fixed_str_tool_call in [
str_tool_call,
fix_brace(str_tool_call),
fix_quote(str_tool_call),
fix_brace(fix_quote(str_tool_call)),
]:
try:
return json.loads(fixed_str_tool_call)
except json.JSONDecodeError:
continue
return None
def extract_tool_calls(
self,
model_output: str,
request: ChatCompletionRequest,
) -> ExtractedToolCallInformation:
"""
Extract the tool calls from a complete model response. Requires
find-and-replacing single quotes with double quotes for JSON parsing,
make sure your tool call arguments don't ever include quotes!
"""
# case -- if a tool call token is not present, return a text response
if self.bot_token not in model_output:
return ExtractedToolCallInformation(tools_called=False,
tool_calls=[],
content=model_output)
try:
str_tool_calls = self.tool_calls_regex.findall(model_output)[0].strip()
if not str_tool_calls.startswith("["):
str_tool_calls = "[" + str_tool_calls
if not str_tool_calls.endswith("]"):
str_tool_calls = "]" + str_tool_calls
try:
function_call_arr = json.loads(str_tool_calls)
except json.JSONDecodeError:
logger.warning(f"Error in parsing tool call: {str_tool_calls}")
tool_call_prefix_matches = list(self.tool_call_regex.finditer(str_tool_calls))
for i, match in enumerate(tool_call_prefix_matches):
if i == len(tool_call_prefix_matches) - 1:
str_tool_call = str_tool_calls[match.start():]
else:
str_tool_call = str_tool_calls[match.start():tool_call_prefix_matches[i+1].start()]
str_tool_call = str_tool_call[str_tool_call.find("{"):]
tool_call = self._try_single_tool_call_parse(str_tool_call)
if tool_call is not None:
function_call_arr.append(tool_call)
continue
# Get parameter configuration for type conversion
tool_param_config = self._get_param_config(request)
for function_call in function_call_arr:
fn_name = function_call["name"]
args = function_call["arguments"]
if isinstance(args, str):
args = json.loads(args)
for param_name in list(args.keys()):
# Get parameter type
param_type = tool_param_config.get(fn_name, {}).get(param_name, None)
# Convert param value to appropriate type
if param_type is not None:
args[param_name] = self._convert_param_value(args[param_name], param_type)
# Tool Call
tool_calls: list[NemotronToolCall] = [
NemotronToolCall(
type="function",
function=FunctionCall(
name=raw_function_call["name"],
# function call args are JSON but as a string
arguments=json.dumps(raw_function_call["arguments"], ensure_ascii=False)
)
) for raw_function_call in function_call_arr
]
# get any content before the tool call
content = model_output.split(self.bot_token)[0]
return ExtractedToolCallInformation(
tools_called=True,
tool_calls=tool_calls,
content=content if len(content) > 0 else None)
except Exception:
logger.warning(f"Error in extracting tool call from response: {model_output}")
# return information to just treat the tool call as regular JSON
return ExtractedToolCallInformation(tools_called=False,
tool_calls=[],
content=model_output)
def extract_tool_calls_streaming(
self,
previous_text: str,
current_text: str,
delta_text: str,
previous_token_ids: Sequence[int],
current_token_ids: Sequence[int],
delta_token_ids: Sequence[int],
request: ChatCompletionRequest,
) -> Union[DeltaMessage, None]:
# if candidates tool call tokens are in the tokens generated so far, that
# means we're parsing as tool calls now. Suppress streaming if we are
# currently generating any prefix of the start or end tag.
visible_delta_text = delta_text
try:
visible_delta_text = self._visible_delta_outside_tool(delta_text, self.bot_token, self.eot_token)
except Exception:
# Fallback to conservative checks in case of any issues
if current_text.endswith('<') or current_text.endswith('' in parsable_arr:
end_of_call = True
parsable_arr = parsable_arr.split('')[0]
# Reset fallback state for next tool call array
self._fallback_parsed_tools = []
self._last_standard_parse_count = 0
# tool calls are generated in an array, so do partial JSON
# parsing on the entire array
tool_call_arr: list[dict] = []
try:
tool_call_arr = partial_json_parser.loads(parsable_arr, flags)
# Track the number of successfully parsed tool calls
if isinstance(tool_call_arr, list):
self._last_standard_parse_count = len(tool_call_arr)
except JSON_PARSE_EXCEPTIONS:
tool_call_arr = self._parse_tool_calls_fallback(parsable_arr)
if len(tool_call_arr) == 0:
return None
current_tool_call: dict = tool_call_arr[self.current_tool_id] \
if len(tool_call_arr) > 0 and self.current_tool_id < len(tool_call_arr) else {}
# case: we are starting a new tool in the array
# -> array has > 0 length AND length has moved past cursor
if (len(tool_call_arr) > 0 and len(tool_call_arr) > self.current_tool_id + 1):
# if we're moving on to a new call, first make sure we
# haven't missed anything in the previous one that was
# auto-generated due to JSON completions, but wasn't
# streamed to the client yet.
if self.current_tool_id >= 0:
diff: Union[str, None] = current_tool_call.get("arguments")
if diff:
diff = json.dumps(diff, ensure_ascii=False).replace(
self.streamed_args_for_tool[self.current_tool_id], "")
if not diff:
return None
delta = DeltaMessage(tool_calls=[
DeltaToolCall(
index=self.current_tool_id,
function=DeltaFunctionCall(arguments=diff).model_dump(exclude_none=True)
)
])
self.streamed_args_for_tool[self.current_tool_id] += diff
else:
delta = None
else:
delta = None
# re-set stuff pertaining to progress in the current tool
self.current_tool_id = len(tool_call_arr) - 1
self.current_tool_name_sent = False
self.streamed_args_for_tool.append("")
self.tool_args_emitted.append(False)
return delta
# case: update an existing tool - this is handled below
# if the current tool name hasn't been sent, send if available
# - otherwise send nothing
if not self.current_tool_name_sent:
function_name = current_tool_call.get("name")
if function_name:
delta = DeltaMessage(tool_calls=[
DeltaToolCall(
index=self.current_tool_id,
type="function",
id=NemotronToolCall.generate_random_id(),
function=DeltaFunctionCall(name=function_name).model_dump(exclude_none=True)
)
])
self.current_tool_name_sent = True
else:
delta = None
# now we know we're on the same tool call and we're streaming
# arguments
else:
prev_arguments = self.prev_tool_call_arr[self.current_tool_id].get("arguments")
cur_arguments = current_tool_call.get("arguments")
if not cur_arguments and not prev_arguments:
delta = None
elif not cur_arguments and prev_arguments:
logger.error("INVARIANT - impossible to have arguments reset mid-arguments")
delta = None
elif cur_arguments:
cur_arguments_json = json.dumps(cur_arguments, ensure_ascii=False)
arguments_delta = self._compute_arguments_delta(cur_arguments_json, end_of_call)
if arguments_delta:
delta = DeltaMessage(tool_calls=[
DeltaToolCall(
index=self.current_tool_id,
function=DeltaFunctionCall(arguments=arguments_delta).model_dump(exclude_none=True)
)
])
self.streamed_args_for_tool[self.current_tool_id] += arguments_delta
self.tool_args_emitted[self.current_tool_id] = True
else:
# Do not flush final JSON here; let the serving layer
# compute a minimal remaining suffix on finish.
delta = None
else:
# End-of-call or equal state; do not force a final flush here.
delta = None
# check to see if the name is defined and has been sent. if so,
# stream the name - otherwise keep waiting
# finish by setting old and returning None as base case
self.prev_tool_call_arr = tool_call_arr
# If we've reached the end of a tool call, flush any remaining
# suffix (including a final '}') that hasn't been streamed yet.
if end_of_call and self.current_tool_id >= 0:
try:
cur_arguments = current_tool_call.get("arguments")
if cur_arguments is not None:
cur_args_json = json.dumps(cur_arguments, ensure_ascii=False)
remaining_suffix = self._compute_arguments_delta(cur_args_json, end_of_call=True)
# Only send remaining suffix if it's non-empty and contains meaningful content
# (not just whitespace or single characters like closing braces)
if remaining_suffix and remaining_suffix.strip():
extra = DeltaToolCall(
index=self.current_tool_id,
function=DeltaFunctionCall(arguments=remaining_suffix).model_dump(exclude_none=True)
)
if delta is None:
delta = DeltaMessage(tool_calls=[extra])
else:
if getattr(delta, "tool_calls", None):
delta.tool_calls.append(extra)
else:
delta.tool_calls = [extra]
self.streamed_args_for_tool[self.current_tool_id] += remaining_suffix
self.tool_args_emitted[self.current_tool_id] = True
else:
pass
except Exception:
pass
return delta
except Exception:
logger.exception("Error trying to handle streaming tool call.")
return None