| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | from __future__ import annotations |
| 5 | |
| 6 | import ast |
| 7 | import json |
| 8 | from collections.abc import Sequence |
| 9 | from typing import TYPE_CHECKING, Any, TypedDict |
| 10 | |
| 11 | import regex as re |
| 12 | |
| 13 | import vllm.envs as envs |
| 14 | from vllm.entrypoints.chat_utils import make_tool_call_id |
| 15 | from vllm.entrypoints.generate.base.protocol import ( |
| 16 | DeltaFunctionCall, |
| 17 | DeltaMessage, |
| 18 | DeltaToolCall, |
| 19 | ExtractedToolCallInformation, |
| 20 | FunctionCall, |
| 21 | ToolCall, |
| 22 | ) |
| 23 | from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest |
| 24 | from vllm.entrypoints.openai.responses.protocol import ResponsesRequest |
| 25 | from vllm.logger import init_logger |
| 26 | from vllm.tokenizers import TokenizerLike |
| 27 | from vllm.tool_parsers.abstract_tool_parser import ToolParser |
| 28 | from vllm.tool_parsers.tool_strict_level import ToolStrictLevel |
| 29 | from vllm.tool_parsers.utils import partial_tag_overlap |
| 30 | |
| 31 | if TYPE_CHECKING: |
| 32 | from xgrammar import StructuralTag |
| 33 | |
| 34 | logger = init_logger(__name__) |
| 35 | |
| 36 | # ============================================================================= |
| 37 | # Plain-dict contract with the vLLM-side wrapper |
| 38 | # ============================================================================= |
| 39 | |
| 40 | |
| 41 | class ToolSchema(TypedDict): |
| 42 | """A tool as seen by this parser (built by the wrapper from ``request.tools``).""" |
| 43 | |
| 44 | name: str |
| 45 | parameters: dict[str, Any] | None # JSON Schema, or None |
| 46 | |
| 47 | |
| 48 | class ToolCallDict(TypedDict): |
| 49 | """One extracted tool call. ``arguments`` is a JSON-encoded string.""" |
| 50 | |
| 51 | name: str |
| 52 | arguments: str |
| 53 | |
| 54 | |
| 55 | class ExtractResult(TypedDict): |
| 56 | """Return of ``extract_tool_calls`` (non-streaming).""" |
| 57 | |
| 58 | tools_called: bool |
| 59 | content: str | None |
| 60 | tool_calls: list[ToolCallDict] |
| 61 | |
| 62 | |
| 63 | class StreamToolCall(TypedDict): |
| 64 | """One streaming tool-call delta. |
| 65 | |
| 66 | ``new`` is True only on the chunk that first emits this tool's name -- the |
| 67 | wrapper mints an id + sets type on that chunk only. ``arguments`` is the |
| 68 | incremental JSON diff (or None). |
| 69 | """ |
| 70 | |
| 71 | index: int |
| 72 | new: bool |
| 73 | name: str | None |
| 74 | arguments: str | None |
| 75 | |
| 76 | |
| 77 | class StreamDelta(TypedDict): |
| 78 | """Return of ``extract_tool_calls_streaming``. |
| 79 | |
| 80 | A single engine delta can carry content plus every tool call that was |
| 81 | drained from the buffer (batched decode), so ``tool_calls`` is not limited |
| 82 | to one entry. |
| 83 | """ |
| 84 | |
| 85 | content: str | None |
| 86 | tool_calls: list[StreamToolCall] |
| 87 | |
| 88 | |
| 89 | # NOTE: mirrored in ``vllm.reasoning.hy_v4_reasoning_parser`` (same pattern as |
| 90 | # ``gemma4_utils``) so neither package depends on the other. |
| 91 | def detect_token_suffix(tokenizer: TokenizerLike) -> str: |
| 92 | """Detect the per-checkpoint suffix used by Hunyuan structural tokens. |
| 93 | |
| 94 | Args: |
| 95 | tokenizer: Tokenizer of the served checkpoint. |
| 96 | |
| 97 | Returns: |
| 98 | The suffix including its leading colon (e.g. ``":6124c78e"``), or ``""`` |
| 99 | when the checkpoint uses unsuffixed tokens. |
| 100 | |
| 101 | Raises: |
| 102 | RuntimeError: The tokenizer declares the structural tokens through |
| 103 | ``model_specific_special_tokens``, which transformers 5 no longer |
| 104 | round-trips. |
| 105 | |
| 106 | """ |
| 107 | init_kwargs = getattr(tokenizer, "init_kwargs", None) or {} |
| 108 | think_begin_as_special = init_kwargs.get("model_specific_special_tokens", {}).get( |
| 109 | "think_begin_token", "" |
| 110 | ) |
| 111 | if think_begin_as_special: |
| 112 | raise RuntimeError( |
| 113 | "This checkpoint declares HYV4 structural tokens (think_begin_token" |
| 114 | "/toolcalls_begin_token/argkey_begin_token) in " |
| 115 | "tokenizer_config.json, which transformers 5 no longer supports. " |
| 116 | "Remove those fields and keep the tokens in the tokenizer's own " |
| 117 | "token definitions so the suffix can be read from the vocab." |
| 118 | ) |
| 119 | |
| 120 | structural_token_re = re.compile( |
| 121 | r"<(?:think|tool_calls|tool_call|arg_key|arg_value)(:[^\s>]+)?>" |
| 122 | ) |
| 123 | for token in tokenizer.get_vocab(): |
| 124 | match = structural_token_re.fullmatch(token) |
| 125 | if match: |
| 126 | return match.group(1) or "" |
| 127 | |
| 128 | return "" |
| 129 | |
| 130 | |
| 131 | # ============================================================================= |
| 132 | # Argument-value parsing utilities (stateless, pure functions) |
| 133 | # |
| 134 | # These map the model's raw ``<arg_value>`` text to typed Python values using |
| 135 | # the tool's JSON Schema. They do not touch any parser state, so they live at |
| 136 | # module level instead of tangling up HYV4ToolExtractor. |
| 137 | # ============================================================================= |
| 138 | |
| 139 | _TYPE_ALIASES: dict[str, str] = { |
| 140 | "str": "string", |
| 141 | "text": "string", |
| 142 | "varchar": "string", |
| 143 | "char": "string", |
| 144 | "enum": "string", |
| 145 | "bool": "boolean", |
| 146 | "binary": "boolean", |
| 147 | "int": "integer", |
| 148 | "float": "number", |
| 149 | "double": "number", |
| 150 | "list": "array", |
| 151 | "dict": "object", |
| 152 | "map": "object", |
| 153 | } |
| 154 | |
| 155 | # Prefix-based wildcard matching for non-standard type names. |
| 156 | # Following the same approach as qwen3coder_tool_parser._convert_param_value |
| 157 | # which uses param_type.startswith("int"), startswith("uint"), etc. |
| 158 | _INTEGER_PREFIXES: tuple[str, ...] = ("int", "uint", "long", "short", "unsigned") |
| 159 | _NUMBER_PREFIXES: tuple[str, ...] = ("num", "float") |
| 160 | |
| 161 | |
| 162 | def _normalize_type(raw_type: str) -> str: |
| 163 | """Map non-standard type aliases to JSON Schema standard names. |
| 164 | |
| 165 | First performs exact lookup in _TYPE_ALIASES. On miss, falls back to |
| 166 | prefix-based matching using startswith() |
| 167 | - int*/uint*/long*/short*/unsigned* → "integer" |
| 168 | - num*/float* → "number" |
| 169 | """ |
| 170 | exact = _TYPE_ALIASES.get(raw_type) |
| 171 | if exact is not None: |
| 172 | return exact |
| 173 | lower = raw_type.lower() |
| 174 | if any(lower.startswith(p) for p in _INTEGER_PREFIXES): |
| 175 | return "integer" |
| 176 | if any(lower.startswith(p) for p in _NUMBER_PREFIXES): |
| 177 | return "number" |
| 178 | return raw_type |
| 179 | |
| 180 | |
| 181 | def _get_arg_schema( |
| 182 | function_name: str, |
| 183 | arg_key: str, |
| 184 | tools: list[ToolSchema] | None, |
| 185 | ) -> dict[str, Any]: |
| 186 | """Look up a specific argument's property schema from the tools list.""" |
| 187 | if tools is None: |
| 188 | return {} |
| 189 | for tool in tools: |
| 190 | if tool.get("name") == function_name: |
| 191 | parameters = tool.get("parameters") |
| 192 | if parameters is None: |
| 193 | return {} |
| 194 | return parameters.get("properties", {}).get(arg_key, {}) |
| 195 | logger.warning_once("No tool named '%s'.", function_name) |
| 196 | return {} |
| 197 | |
| 198 | |
| 199 | def _get_schema_options(arg_schema: dict[str, Any]) -> list[dict[str, Any]]: |
| 200 | """Normalize any property schema into a list of sub-schemas. |
| 201 | - has type (single type) → return [arg_schema] |
| 202 | - anyOf → return the anyOf list |
| 203 | - oneOf → return the oneOf list |
| 204 | - fallback → [{"type": "string"}] |
| 205 | |
| 206 | Note: single ``type`` has the highest priority. |
| 207 | """ |
| 208 | if "type" in arg_schema: |
| 209 | type_val = arg_schema["type"] |
| 210 | # JSON Schema allows "type" to be an array to represent union types, |
| 211 | # e.g. "type": ["string", "object"]. |
| 212 | # Expand it into an anyOf-equivalent format: |
| 213 | # [{"type": "string"}, {"type": "object"}] |
| 214 | # so that _get_types / _parse_value can handle it uniformly later. |
| 215 | if isinstance(type_val, list): |
| 216 | return [{"type": t} for t in type_val] |
| 217 | return [arg_schema] |
| 218 | if "anyOf" in arg_schema: |
| 219 | return arg_schema["anyOf"] |
| 220 | if "oneOf" in arg_schema: |
| 221 | return arg_schema["oneOf"] |
| 222 | |
| 223 | return [{"type": "string"}] |
| 224 | |
| 225 | |
| 226 | def _get_types(arg_schema: dict[str, Any]) -> set[str]: |
| 227 | """Extract normalized, non-null type set from a property schema.""" |
| 228 | schemas = _get_schema_options(arg_schema) |
| 229 | return {_normalize_type(s.get("type", "string")) for s in schemas} - {"null"} |
| 230 | |
| 231 | |
| 232 | def _is_only_string_type( |
| 233 | function_name: str, |
| 234 | arg_key: str, |
| 235 | tools: list[ToolSchema] | None, |
| 236 | ) -> bool: |
| 237 | """Return True if the parameter's type set is exactly {"string"}. |
| 238 | |
| 239 | Only pure string types get partial value streaming; compound types like |
| 240 | anyOf(string | array) do not, since the partial value might end up being a |
| 241 | JSON array or object. |
| 242 | """ |
| 243 | types = _get_types(_get_arg_schema(function_name, arg_key, tools)) |
| 244 | return types == {"string"} |
| 245 | |
| 246 | |
| 247 | def _try_parse_bool(value: str) -> bool | None: |
| 248 | """Try to parse a string as bool; return None on failure.""" |
| 249 | lower = value.lower() |
| 250 | if lower == "true": |
| 251 | return True |
| 252 | elif lower == "false": |
| 253 | return False |
| 254 | return None |
| 255 | |
| 256 | |
| 257 | def _try_parse_int(value: str) -> int | None: |
| 258 | """Try to parse a string as int; return None on failure.""" |
| 259 | try: |
| 260 | return int(value) |
| 261 | except (ValueError, TypeError): |
| 262 | return None |
| 263 | |
| 264 | |
| 265 | def _try_parse_wildcard_number(value: str) -> int | float | None: |
| 266 | """Try to parse a string as a number (int or float). |
| 267 | |
| 268 | Decision rule: if the string contains '.' or 'e'/'E' (scientific notation), |
| 269 | parse as float; otherwise parse as int. |
| 270 | |
| 271 | Examples: "5" → 5, "5.0" → 5.0, "1e3" → 1000.0, "-3" → -3. |
| 272 | Return None on failure. |
| 273 | """ |
| 274 | try: |
| 275 | if "." in value or "e" in value or "E" in value: |
| 276 | return float(value) |
| 277 | return int(value) |
| 278 | except (ValueError, TypeError): |
| 279 | return None |
| 280 | |
| 281 | |
| 282 | def _deserialize(value: str) -> Any: |
| 283 | """Deserialize a string value using json.loads then ast.literal_eval.""" |
| 284 | try: |
| 285 | return json.loads(value) |
| 286 | except Exception: |
| 287 | pass |
| 288 | try: |
| 289 | return ast.literal_eval(value) |
| 290 | except Exception: |
| 291 | pass |
| 292 | return value |
| 293 | |
| 294 | |
| 295 | def _parse_value( |
| 296 | value: str, |
| 297 | function_name: str, |
| 298 | arg_key: str, |
| 299 | tools: list[ToolSchema] | None, |
| 300 | ) -> Any: |
| 301 | """Unified argument value parser with anyOf/oneOf support. |
| 302 | |
| 303 | Fallthrough chain: |
| 304 | bool → int → number(wildcard_number) |
| 305 | → json.loads for array/object → string → _deserialize |
| 306 | """ |
| 307 | types = _get_types(_get_arg_schema(function_name, arg_key, tools)) |
| 308 | |
| 309 | # 1. Try bool |
| 310 | if "boolean" in types: |
| 311 | result_bool = _try_parse_bool(value) |
| 312 | if result_bool is not None: |
| 313 | return result_bool |
| 314 | |
| 315 | # 2. Try int |
| 316 | if "integer" in types: |
| 317 | result_int = _try_parse_int(value) |
| 318 | if result_int is not None: |
| 319 | return result_int |
| 320 | |
| 321 | # 3. Try number (wildcard_number: int if no '.'/e/E, float otherwise) |
| 322 | if "number" in types: |
| 323 | result_number = _try_parse_wildcard_number(value) |
| 324 | if result_number is not None: |
| 325 | return result_number |
| 326 | |
| 327 | # 4. Try json.loads (covers array/object and other unlisted types) |
| 328 | if types - {"string", "boolean", "integer", "number"}: |
| 329 | try: |
| 330 | return json.loads(value) |
| 331 | except (json.JSONDecodeError, ValueError): |
| 332 | pass |
| 333 | |
| 334 | # 5. String fallback |
| 335 | if "string" in types: |
| 336 | return value |
| 337 | |
| 338 | # 6. Final fallback |
| 339 | return _deserialize(value) |
| 340 | |
| 341 | |
| 342 | # ============================================================================= |
| 343 | # Stateful tool-call parser |
| 344 | # ============================================================================= |
| 345 | |
| 346 | |
| 347 | class HYV4ToolExtractor: |
| 348 | """Pure tool-call parsing logic for HYV4. Returns plain dicts; no vLLM types. |
| 349 | |
| 350 | Holds only parser state: the structural token strings/ids, the compiled |
| 351 | regexes, and the streaming incremental state. Stateless argument parsing is |
| 352 | delegated to the module-level utilities above. |
| 353 | |
| 354 | Streaming has two entry paths sharing one incremental argument parser, |
| 355 | selected by the ``guided`` flag of :meth:`extract_tool_calls_streaming`: |
| 356 | normal / auto tool choice detects markers on atomic token ids so ordinary |
| 357 | ``<`` in content is never withheld, while structural-tag guided decoding |
| 358 | detects them on the string level because the grammar may split a marker |
| 359 | across sub-word tokens. |
| 360 | """ |
| 361 | |
| 362 | def __init__(self, vocab: dict[str, int], token_suffix: str, strict: bool): |
| 363 | self._strict = strict |
| 364 | |
| 365 | # Kept so the wrapper can build a matching structural tag for |
| 366 | # constrained decoding without re-scanning the tokenizer vocab. |
| 367 | self.token_suffix: str = token_suffix |
| 368 | |
| 369 | # Running state read by the vLLM serving layer (mirrored by the wrapper). |
| 370 | self.prev_tool_call_arr: list[dict] = [] |
| 371 | self.current_tool_id: int = -1 |
| 372 | self.streamed_args_for_tool: list[str] = [] |
| 373 | |
| 374 | # Streaming state: send tool name first, then return arguments at once |
| 375 | self._streaming_tool_name: str | None = None # tool name being streamed |
| 376 | |
| 377 | # State fields for incremental argument streaming |
| 378 | self._completed_args: dict = {} # closed {key: parsed_value} |
| 379 | self._current_arg_key: str | None = None # key being collected |
| 380 | self._current_arg_is_string: bool = False # is current arg pure string? |
| 381 | self._streamed_json_len: int = 0 # bytes of JSON already sent |
| 382 | |
| 383 | self.tool_calls_start_token: str = f"<tool_calls{token_suffix}>" |
| 384 | self.tool_calls_end_token: str = f"</tool_calls{token_suffix}>" |
| 385 | |
| 386 | self.tool_call_start_token: str = f"<tool_call{token_suffix}>" |
| 387 | self.tool_call_end_token: str = f"</tool_call{token_suffix}>" |
| 388 | |
| 389 | self.arg_key_start_token: str = f"<arg_key{token_suffix}>" |
| 390 | self.arg_key_end_token: str = f"</arg_key{token_suffix}>" |
| 391 | |
| 392 | self.arg_value_start_token: str = f"<arg_value{token_suffix}>" |
| 393 | self.arg_value_end_token: str = f"</arg_value{token_suffix}>" |
| 394 | |
| 395 | logger.debug( |
| 396 | "HYV4ToolExtractor structural tokens: %s ... %s, suffix=%r, strict=%s", |
| 397 | self.tool_calls_start_token, |
| 398 | self.tool_calls_end_token, |
| 399 | token_suffix, |
| 400 | strict, |
| 401 | ) |
| 402 | |
| 403 | # The suffix comes from the tokenizer vocab, so it must not be treated |
| 404 | # as a pattern. |
| 405 | self.tool_call_regex = re.compile( |
| 406 | rf"{re.escape(self.tool_call_start_token)}(.*?)" |
| 407 | rf"{re.escape(self.tool_call_end_token)}", |
| 408 | re.DOTALL, |
| 409 | ) |
| 410 | |
| 411 | self.func_args_regex = re.compile( |
| 412 | rf"{re.escape(self.arg_key_start_token)}(.*?)" |
| 413 | rf"{re.escape(self.arg_key_end_token)}" |
| 414 | rf"{re.escape(self.arg_value_start_token)}(.*?)" |
| 415 | rf"{re.escape(self.arg_value_end_token)}", |
| 416 | re.DOTALL, |
| 417 | ) |
| 418 | |
| 419 | self.tool_calls_start_token_id = vocab.get(self.tool_calls_start_token) |
| 420 | self.tool_calls_end_token_id = vocab.get(self.tool_calls_end_token) |
| 421 | |
| 422 | self.tool_call_start_token_id = vocab.get(self.tool_call_start_token) |
| 423 | self.tool_call_end_token_id = vocab.get(self.tool_call_end_token) |
| 424 | self._buffer = "" |
| 425 | |
| 426 | if ( |
| 427 | self.tool_calls_start_token_id is None |
| 428 | or self.tool_calls_end_token_id is None |
| 429 | ): |
| 430 | raise RuntimeError( |
| 431 | "HYV4 tool extractor could not locate tool call " |
| 432 | "start/end tokens in the tokenizer!" |
| 433 | ) |
| 434 | |
| 435 | def _extract_tool_calls( |
| 436 | self, |
| 437 | model_output: str, |
| 438 | tools: list[ToolSchema] | None, |
| 439 | ) -> list[ToolCallDict]: |
| 440 | try: |
| 441 | tool_calls: list[ToolCallDict] = [] |
| 442 | function_calls = self.tool_call_regex.findall(model_output) |
| 443 | for tool_call_body in function_calls: |
| 444 | arg_start = tool_call_body.find(self.arg_key_start_token) |
| 445 | if arg_start == -1: |
| 446 | function_name = tool_call_body.strip() |
| 447 | function_args = "" |
| 448 | else: |
| 449 | function_name = tool_call_body[:arg_start].strip() |
| 450 | function_args = tool_call_body[arg_start:].strip() |
| 451 | |
| 452 | arg_pairs = self.func_args_regex.findall(function_args) |
| 453 | arg_dict = {} |
| 454 | for key, value in arg_pairs: |
| 455 | arg_dict[key] = _parse_value(value, function_name, key, tools) |
| 456 | tool_calls.append( |
| 457 | ToolCallDict( |
| 458 | name=function_name, |
| 459 | arguments=json.dumps(arg_dict, ensure_ascii=False), |
| 460 | ) |
| 461 | ) |
| 462 | return tool_calls |
| 463 | except Exception: |
| 464 | logger.exception("Error in extracting tool call from response.") |
| 465 | return [] |
| 466 | |
| 467 | def _extract_tool_calls_strict( |
| 468 | self, |
| 469 | model_output: str, |
| 470 | tools: list[ToolSchema] | None, |
| 471 | ) -> list[ToolCallDict]: |
| 472 | start_idx = model_output.find(self.tool_calls_start_token) |
| 473 | end_idx = model_output.find(self.tool_calls_end_token) |
| 474 | if start_idx == -1 or end_idx == -1 or end_idx < start_idx: |
| 475 | raise ValueError("Malformed tool block wrapper.") |
| 476 | |
| 477 | tool_block = model_output[ |
| 478 | start_idx + len(self.tool_calls_start_token) : end_idx |
| 479 | ] |
| 480 | if tool_block.count(self.tool_call_start_token) != tool_block.count( |
| 481 | self.tool_call_end_token |
| 482 | ): |
| 483 | raise ValueError("Malformed tool_call wrapper count.") |
| 484 | |
| 485 | tool_calls: list[ToolCallDict] = [] |
| 486 | function_calls = self.tool_call_regex.findall(tool_block) |
| 487 | if tool_block.count(self.tool_call_start_token) != len(function_calls): |
| 488 | raise ValueError("Malformed tool_call body.") |
| 489 | |
| 490 | for tool_call_body in function_calls: |
| 491 | arg_start = tool_call_body.find(self.arg_key_start_token) |
| 492 | if arg_start == -1: |
| 493 | function_name = tool_call_body.strip() |
| 494 | function_args = "" |
| 495 | else: |
| 496 | function_name = tool_call_body[:arg_start].strip() |
| 497 | function_args = tool_call_body[arg_start:].strip() |
| 498 | |
| 499 | if not function_name: |
| 500 | raise ValueError("Empty function name in tool call.") |
| 501 | |
| 502 | if function_args.count(self.arg_key_start_token) != function_args.count( |
| 503 | self.arg_key_end_token |
| 504 | ) or function_args.count(self.arg_value_start_token) != function_args.count( |
| 505 | self.arg_value_end_token |
| 506 | ): |
| 507 | raise ValueError("Malformed argument tag count.") |
| 508 | |
| 509 | arg_pairs = self.func_args_regex.findall(function_args) |
| 510 | remainder = self.func_args_regex.sub("", function_args) |
| 511 | if remainder.strip(): |
| 512 | raise ValueError("Unparsed argument payload remains.") |
| 513 | |
| 514 | arg_dict = {} |
| 515 | for key, value in arg_pairs: |
| 516 | arg_dict[key] = _parse_value(value, function_name, key, tools) |
| 517 | |
| 518 | tool_calls.append( |
| 519 | ToolCallDict( |
| 520 | name=function_name, |
| 521 | arguments=json.dumps(arg_dict, ensure_ascii=False), |
| 522 | ) |
| 523 | ) |
| 524 | |
| 525 | if not tool_calls: |
| 526 | raise ValueError("No valid tool calls extracted.") |
| 527 | |
| 528 | return tool_calls |
| 529 | |
| 530 | def extract_tool_calls( |
| 531 | self, |
| 532 | model_output: str, |
| 533 | tools: list[ToolSchema] | None, |
| 534 | ) -> ExtractResult: |
| 535 | # sanity check; avoid unnecessary processing |
| 536 | if self.tool_calls_start_token not in model_output: |
| 537 | return ExtractResult( |
| 538 | tools_called=False, content=model_output, tool_calls=[] |
| 539 | ) |
| 540 | else: |
| 541 | try: |
| 542 | if self._strict: |
| 543 | tool_calls = self._extract_tool_calls_strict(model_output, tools) |
| 544 | else: |
| 545 | tool_calls = self._extract_tool_calls(model_output, tools) |
| 546 | if not tool_calls: |
| 547 | raise ValueError("No valid tool calls extracted.") |
| 548 | |
| 549 | s_index = model_output.find(self.tool_calls_start_token) |
| 550 | content = model_output[:s_index] if s_index != -1 else model_output |
| 551 | return ExtractResult( |
| 552 | tools_called=True, |
| 553 | content=content if content else None, |
| 554 | tool_calls=tool_calls, |
| 555 | ) |
| 556 | |
| 557 | except Exception: |
| 558 | logger.warning( |
| 559 | "HYV4ToolExtractor detected malformed tool output; " |
| 560 | "falling back to raw content." |
| 561 | ) |
| 562 | return ExtractResult( |
| 563 | tools_called=False, |
| 564 | content=model_output, |
| 565 | tool_calls=[], |
| 566 | ) |
| 567 | |
| 568 | def _reset_streaming_tool_state(self): |
| 569 | """Reset the streaming state for a single tool call.""" |
| 570 | self._streaming_tool_name = None |
| 571 | self._completed_args = {} |
| 572 | self._current_arg_key = None |
| 573 | self._current_arg_is_string = False |
| 574 | self._streamed_json_len = 0 |
| 575 | |
| 576 | def extract_tool_calls_streaming( |
| 577 | self, |
| 578 | previous_text: str, |
| 579 | current_text: str, |
| 580 | delta_text: str, |
| 581 | previous_token_ids: Sequence[int], |
| 582 | current_token_ids: Sequence[int], |
| 583 | delta_token_ids: Sequence[int], |
| 584 | tools: list[ToolSchema] | None, |
| 585 | *, |
| 586 | guided: bool = False, |
| 587 | ) -> StreamDelta | None: |
| 588 | """Consume one engine delta and emit content / tool-call deltas. |
| 589 | |
| 590 | Two entry paths share the same incremental argument parser: |
| 591 | - ``guided=False`` (normal / auto tool choice): the structural markers |
| 592 | arrive as atomic tokenizer tokens, so detection is done on token ids |
| 593 | and ordinary ``<`` in model content is never withheld. |
| 594 | - ``guided=True`` (structural-tag constrained decoding): the grammar |
| 595 | only constrains strings, so ``<tool_calls...>`` may be split across |
| 596 | sub-word tokens and must be detected on the string level. |
| 597 | |
| 598 | Args: |
| 599 | previous_text: Text decoded before this delta. |
| 600 | current_text: Full text decoded so far. |
| 601 | delta_text: Text decoded in this delta. |
| 602 | previous_token_ids: Token ids before this delta. |
| 603 | current_token_ids: Token ids decoded so far. |
| 604 | delta_token_ids: Token ids in this delta. |
| 605 | tools: Plain tool schemas used for argument typing. |
| 606 | guided: Whether guided decoding may split markers into sub-word |
| 607 | tokens. |
| 608 | |
| 609 | Returns: |
| 610 | A streaming delta carrying content and/or the tool calls drained |
| 611 | from the buffer, or None when nothing can be emitted yet. |
| 612 | |
| 613 | """ |
| 614 | content_delta: str | None = None |
| 615 | tool_calls: list[StreamToolCall] = [] |
| 616 | |
| 617 | if guided: |
| 618 | # Guided path: a structural-tag grammar constrains strings, not |
| 619 | # token ids, so the atomic ``tool_calls_start_token_id`` never |
| 620 | # appears in the token stream and an id-based check would |
| 621 | # misclassify every tool delta as plain content. |
| 622 | marker = self.tool_calls_start_token |
| 623 | marker_pos = current_text.find(marker) |
| 624 | |
| 625 | if marker_pos == -1: |
| 626 | # Marker not seen yet. The tail of ``current_text`` might still |
| 627 | # be a partial prefix of the marker split across deltas; hold |
| 628 | # that tail back and only stream the safe portion as content. |
| 629 | safe_end = len(current_text) - partial_tag_overlap(current_text, marker) |
| 630 | # Emit only the newly-arrived, safe content (delta relative to |
| 631 | # what was previously streamed). |
| 632 | content = current_text[len(previous_text) : safe_end] |
| 633 | return StreamDelta( |
| 634 | content=content if content else None, |
| 635 | tool_calls=[], |
| 636 | ) |
| 637 | |
| 638 | # Marker present. Stream any content that precedes it (only the |
| 639 | # part we have not already emitted), then buffer the tool payload. |
| 640 | # |
| 641 | # The payload is everything after the marker in the *full* text. We |
| 642 | # keep the buffer in sync with it by only appending the portion that |
| 643 | # arrived in this delta, mirroring the incremental ``+=`` semantics |
| 644 | # while remaining correct when the marker itself was split across |
| 645 | # sub-word tokens. |
| 646 | content_end = marker_pos |
| 647 | content = current_text[len(previous_text) : content_end] |
| 648 | |
| 649 | payload_start = marker_pos + len(marker) |
| 650 | # How much of the payload had already been buffered before this |
| 651 | # delta. |
| 652 | prev_payload_len = max(len(previous_text) - payload_start, 0) |
| 653 | new_payload = current_text[payload_start + prev_payload_len :] |
| 654 | self._buffer += new_payload |
| 655 | |
| 656 | if content: |
| 657 | content_delta = content |
| 658 | else: |
| 659 | # Atomic-token path: normal / auto generation emits the HYV4 |
| 660 | # structural markers as single tokenizer tokens. Pass ordinary |
| 661 | # content through exactly as decoded, including comparison |
| 662 | # operators such as ``<``. |
| 663 | if self.tool_calls_start_token_id not in current_token_ids: |
| 664 | return StreamDelta(content=delta_text, tool_calls=[]) |
| 665 | |
| 666 | if self.tool_calls_start_token in delta_text: |
| 667 | text_parts = delta_text.split(self.tool_calls_start_token) |
| 668 | self._buffer += text_parts[-1] |
| 669 | if text_parts[0]: |
| 670 | content_delta = text_parts[0] |
| 671 | else: |
| 672 | self._buffer += delta_text |
| 673 | |
| 674 | # Encountered finish, extract valid arguments |
| 675 | if ( |
| 676 | current_text.find(self.tool_call_end_token + self.tool_calls_end_token) |
| 677 | != -1 |
| 678 | and self._buffer.find(self.tool_call_end_token) == -1 |
| 679 | ): |
| 680 | self._buffer += self.tool_call_end_token + self.tool_calls_end_token |
| 681 | |
| 682 | # Drain every complete tool call that is already buffered so a single |
| 683 | # engine delta can carry content + N tool calls (batched decode). |
| 684 | while True: |
| 685 | cur_text = self._buffer |
| 686 | |
| 687 | # Haven't encountered tool_call start tag yet; drop the inter-tag |
| 688 | # padding but keep any tail that may be a partial tool_call start |
| 689 | # tag (guided decoding can split it across deltas). |
| 690 | start_idx = cur_text.find(self.tool_call_start_token) |
| 691 | if start_idx == -1 and self._streaming_tool_name is None: |
| 692 | keep = partial_tag_overlap(cur_text, self.tool_call_start_token) |
| 693 | self._buffer = cur_text[len(cur_text) - keep :] if keep else "" |
| 694 | break |
| 695 | |
| 696 | # === Phase 1: Detect tool name (send when first arg_key is seen) = |
| 697 | # ``name_new`` mirrors the old ``name_delta is not None``: it marks |
| 698 | # the chunk that first emits this tool's name, which is the only |
| 699 | # chunk the wrapper mints an id + type for. |
| 700 | name_new = False |
| 701 | pending_name: str | None = None |
| 702 | if self._streaming_tool_name is None: |
| 703 | arg_idx = cur_text.find(self.arg_key_start_token, start_idx) |
| 704 | end_idx = cur_text.find(self.tool_call_end_token, start_idx) |
| 705 | if arg_idx == -1 and end_idx == -1: |
| 706 | # tool name not yet complete; keep buffering from |
| 707 | # tool_call_start |
| 708 | self._buffer = cur_text[start_idx:] |
| 709 | break |
| 710 | |
| 711 | name_start = start_idx + len(self.tool_call_start_token) |
| 712 | name_end = arg_idx if arg_idx != -1 else end_idx |
| 713 | |
| 714 | tool_name = cur_text[name_start:name_end].strip() |
| 715 | self._streaming_tool_name = tool_name |
| 716 | |
| 717 | if arg_idx != -1: |
| 718 | self._buffer = cur_text[arg_idx:] |
| 719 | else: |
| 720 | self._buffer = cur_text[end_idx:] |
| 721 | |
| 722 | # Increment tool_id and mark that a name chunk should be emitted |
| 723 | self.current_tool_id += 1 |
| 724 | name_new = True |
| 725 | pending_name = tool_name |
| 726 | |
| 727 | # Check if buffer already has complete arguments |
| 728 | # (all-in-one-delta); otherwise only the name is ready now. |
| 729 | if self.tool_call_end_token not in self._buffer: |
| 730 | tool_calls.append( |
| 731 | StreamToolCall( |
| 732 | index=self.current_tool_id, |
| 733 | new=True, |
| 734 | name=tool_name, |
| 735 | arguments=None, |
| 736 | ) |
| 737 | ) |
| 738 | break |
| 739 | # Buffer already has a complete tool call; fall through to |
| 740 | # phase 2 below. |
| 741 | |
| 742 | # === Phase 2: Incremental argument streaming === |
| 743 | result = self._extract_streaming_incremental(name_new, pending_name, tools) |
| 744 | if result: |
| 745 | tool_calls.extend(result["tool_calls"]) |
| 746 | |
| 747 | # Continue only after the current tool call has completed and the |
| 748 | # remaining structural payload starts with another tool call. |
| 749 | # A literal ``<tool_call>`` may legitimately appear inside an open |
| 750 | # string argument; in that case the current tool name is still set |
| 751 | # and the buffer has not been consumed, so continuing would spin on |
| 752 | # the same buffer forever. |
| 753 | if self._streaming_tool_name is None and self._buffer.startswith( |
| 754 | self.tool_call_start_token |
| 755 | ): |
| 756 | continue |
| 757 | break |
| 758 | |
| 759 | if content_delta is not None or tool_calls: |
| 760 | return StreamDelta(content=content_delta, tool_calls=tool_calls) |
| 761 | return None |
| 762 | |
| 763 | def _make_args_delta(self, argument_diff: str) -> StreamDelta: |
| 764 | """Build an args-only streaming delta (no id/type -> new=False).""" |
| 765 | return StreamDelta( |
| 766 | content=None, |
| 767 | tool_calls=[ |
| 768 | StreamToolCall( |
| 769 | index=self.current_tool_id, |
| 770 | new=False, |
| 771 | name=None, |
| 772 | arguments=argument_diff, |
| 773 | ) |
| 774 | ], |
| 775 | ) |
| 776 | |
| 777 | def _extract_streaming_incremental( |
| 778 | self, |
| 779 | name_new: bool, |
| 780 | pending_name: str | None, |
| 781 | tools: list[ToolSchema] | None, |
| 782 | ) -> StreamDelta | None: |
| 783 | """Incremental phase-2: scan tags in buffer, emit JSON diffs. |
| 784 | |
| 785 | Strategy: |
| 786 | - Track completed args and emit each one as a JSON fragment. |
| 787 | - For string-typed args, stream the value character-by-character. |
| 788 | - Withhold the closing ``}`` until ``</tool_call>`` is seen. |
| 789 | |
| 790 | We build JSON manually via fragments rather than using json.dumps |
| 791 | with a cursor, because json.dumps of partial-vs-full string values |
| 792 | produces incompatible prefixes (e.g. ``""}`` vs ``"Hello"}``). |
| 793 | """ |
| 794 | buf = self._buffer |
| 795 | is_complete = self.tool_call_end_token in buf |
| 796 | |
| 797 | if is_complete: |
| 798 | end_idx = buf.find(self.tool_call_end_token) |
| 799 | args_text = buf[:end_idx] |
| 800 | remaining = buf[end_idx + len(self.tool_call_end_token) :] |
| 801 | else: |
| 802 | args_text = buf |
| 803 | remaining = "" |
| 804 | |
| 805 | # --- scan all fully closed kv pairs --- |
| 806 | arg_pairs = self.func_args_regex.findall(args_text) |
| 807 | for key, value in arg_pairs: |
| 808 | key = key.strip() |
| 809 | if key not in self._completed_args: |
| 810 | self._completed_args[key] = _parse_value( |
| 811 | value, self._streaming_tool_name or "", key, tools |
| 812 | ) |
| 813 | |
| 814 | # --- detect partial (unclosed) kv at the tail --- |
| 815 | last_closed_end = 0 |
| 816 | for m in self.func_args_regex.finditer(args_text): |
| 817 | last_closed_end = m.end() |
| 818 | tail = args_text[last_closed_end:] |
| 819 | |
| 820 | partial_key: str | None = None |
| 821 | partial_value: str | None = None |
| 822 | |
| 823 | ak_start = tail.find(self.arg_key_start_token) |
| 824 | if ak_start != -1: |
| 825 | ak_end = tail.find( |
| 826 | self.arg_key_end_token, |
| 827 | ak_start + len(self.arg_key_start_token), |
| 828 | ) |
| 829 | if ak_end != -1: |
| 830 | partial_key = tail[ |
| 831 | ak_start + len(self.arg_key_start_token) : ak_end |
| 832 | ].strip() |
| 833 | self._current_arg_key = partial_key |
| 834 | self._current_arg_is_string = _is_only_string_type( |
| 835 | self._streaming_tool_name or "", |
| 836 | partial_key, |
| 837 | tools, |
| 838 | ) |
| 839 | |
| 840 | av_start = tail.find(self.arg_value_start_token, ak_end) |
| 841 | if av_start != -1: |
| 842 | val_content_start = av_start + len(self.arg_value_start_token) |
| 843 | if self._current_arg_is_string: |
| 844 | raw_value = tail[val_content_start:] |
| 845 | # Guided decoding can split ``</arg_value>`` across |
| 846 | # deltas; hold back a partial closing tag so it is not |
| 847 | # streamed as part of the value. |
| 848 | keep = len(raw_value) - partial_tag_overlap( |
| 849 | raw_value, self.arg_value_end_token |
| 850 | ) |
| 851 | partial_value = raw_value[:keep] |
| 852 | else: |
| 853 | # key not yet closed |
| 854 | self._current_arg_key = None |
| 855 | self._current_arg_is_string = False |
| 856 | |
| 857 | # --- build the current JSON snapshot as a string --- |
| 858 | # We construct JSON manually so we can precisely control |
| 859 | # what gets sent incrementally. |
| 860 | snapshot_parts: list[str] = [] |
| 861 | for k, v in self._completed_args.items(): |
| 862 | k_json = json.dumps(k, ensure_ascii=False) |
| 863 | v_json = json.dumps(v, ensure_ascii=False) |
| 864 | snapshot_parts.append(f"{k_json}: {v_json}") |
| 865 | |
| 866 | if partial_key is not None and partial_value is not None: |
| 867 | k_json = json.dumps(partial_key, ensure_ascii=False) |
| 868 | # For string partial value, we build the JSON string |
| 869 | # WITHOUT the closing quote, so the prefix stays stable |
| 870 | # as the value grows. The closing `"` and `}` will be |
| 871 | # sent when the value or tool_call closes. |
| 872 | escaped_val = json.dumps(partial_value, ensure_ascii=False)[1:-1] |
| 873 | # Note: no closing " here - it's appended only on close |
| 874 | snapshot_parts.append(f'{k_json}: "{escaped_val}') |
| 875 | |
| 876 | snapshot = "{" + ", ".join(snapshot_parts) + "}" |
| 877 | |
| 878 | # --- compute diff --- |
| 879 | argument_diff: str | None = None |
| 880 | |
| 881 | if is_complete: |
| 882 | # Tool call finished - send everything remaining. |
| 883 | # Build final snapshot with proper JSON (all values closed). |
| 884 | final_args = dict(self._completed_args) |
| 885 | final_json = json.dumps(final_args, ensure_ascii=False) |
| 886 | if self._streamed_json_len < len(final_json): |
| 887 | argument_diff = final_json[self._streamed_json_len :] |
| 888 | self._streamed_json_len = len(final_json) |
| 889 | |
| 890 | # Record into prev_tool_call_arr |
| 891 | self.prev_tool_call_arr.append( |
| 892 | { |
| 893 | "name": self._streaming_tool_name, |
| 894 | "arguments": final_args, |
| 895 | } |
| 896 | ) |
| 897 | self.streamed_args_for_tool.append(final_json) |
| 898 | |
| 899 | self._reset_streaming_tool_state() |
| 900 | self._buffer = remaining |
| 901 | else: |
| 902 | # Still in progress - withhold the tail. |
| 903 | # For open strings: snapshot ends with ...partial_val} |
| 904 | # we withhold "}" (1 char) - the missing closing " will |
| 905 | # be sent when the value closes. |
| 906 | # For no open string: snapshot ends with ...value"} |
| 907 | # we withhold "}" (1 char). |
| 908 | end = len(snapshot) - 1 # exclude trailing "}" |
| 909 | if end > self._streamed_json_len: |
| 910 | argument_diff = snapshot[self._streamed_json_len : end] |
| 911 | self._streamed_json_len = end |
| 912 | |
| 913 | # --- construct return dict --- |
| 914 | if name_new and argument_diff: |
| 915 | return StreamDelta( |
| 916 | content=None, |
| 917 | tool_calls=[ |
| 918 | StreamToolCall( |
| 919 | index=self.current_tool_id, |
| 920 | new=True, |
| 921 | name=pending_name, |
| 922 | arguments=argument_diff, |
| 923 | ) |
| 924 | ], |
| 925 | ) |
| 926 | elif name_new: |
| 927 | return StreamDelta( |
| 928 | content=None, |
| 929 | tool_calls=[ |
| 930 | StreamToolCall( |
| 931 | index=self.current_tool_id, |
| 932 | new=True, |
| 933 | name=pending_name, |
| 934 | arguments=None, |
| 935 | ) |
| 936 | ], |
| 937 | ) |
| 938 | elif argument_diff: |
| 939 | return self._make_args_delta(argument_diff) |
| 940 | else: |
| 941 | return None |
| 942 | |
| 943 | |
| 944 | def build_tool_extractor( |
| 945 | tokenizer: TokenizerLike, *, strict: bool |
| 946 | ) -> HYV4ToolExtractor: |
| 947 | return HYV4ToolExtractor( |
| 948 | tokenizer.get_vocab(), detect_token_suffix(tokenizer), strict |
| 949 | ) |
| 950 | |
| 951 | |
| 952 | def _tools_to_plain(tools) -> list[ToolSchema] | None: |
| 953 | """Convert tool-schema objects into plain dicts for the extractor.""" |
| 954 | if not tools: |
| 955 | return None |
| 956 | return [ |
| 957 | ToolSchema(name=t.function.name, parameters=t.function.parameters) |
| 958 | for t in tools |
| 959 | ] |
| 960 | |
| 961 | |
| 962 | class HYV4ToolParser(ToolParser): |
| 963 | """vLLM adapter around :class:`HYV4ToolExtractor`. |
| 964 | |
| 965 | Converts the extractor's plain dicts to vLLM protocol objects, mints |
| 966 | tool-call ids, and mirrors ``prev_tool_call_arr`` / |
| 967 | ``streamed_args_for_tool`` so the serving layer sees live state. |
| 968 | """ |
| 969 | |
| 970 | # HYV4 emits its own <tool_calls>/<tool_call>/<arg_key>/<arg_value> |
| 971 | # structural tokens, not a plain JSON array. The generic required/named |
| 972 | # JSON streaming path (streaming.extract_required_tool_call_streaming) |
| 973 | # cannot parse this format and produces misaligned multi-tool deltas in |
| 974 | # streaming mode. Fall back to our own (auto-path) parser for |
| 975 | # required/named too, so streaming and non-streaming stay consistent. |
| 976 | supports_required_and_named: bool = False |
| 977 | structural_tag_model: str = "hy_v4" |
| 978 | |
| 979 | def __init__(self, tokenizer: TokenizerLike, tools=None): |
| 980 | super().__init__(tokenizer, tools) |
| 981 | |
| 982 | # Tools are supplied at construction (``tool_parser(tokenizer, |
| 983 | # request.tools)``); keep a plain-dict copy for the extractor. |
| 984 | self._plain_tools = _tools_to_plain(tools) |
| 985 | # Malformed tag structure is rejected and returned as raw content |
| 986 | # instead of a half-parsed tool call. |
| 987 | self._extractor = build_tool_extractor(tokenizer, strict=True) |
| 988 | self._mirror_state() |
| 989 | |
| 990 | def get_structural_tag( |
| 991 | self, |
| 992 | request: ChatCompletionRequest | ResponsesRequest, |
| 993 | *, |
| 994 | reasoning: bool = False, |
| 995 | strict_level: ToolStrictLevel = ToolStrictLevel.AUTO, |
| 996 | ) -> StructuralTag | None: |
| 997 | """Build a structural tag matching HYV4's tool tokens. |
| 998 | |
| 999 | Overridden only to pass the per-checkpoint ``token_suffix`` through to |
| 1000 | the builder; everything else follows the base implementation. |
| 1001 | |
| 1002 | Named (forced) tool choice is constrained with the same structural tag: |
| 1003 | because ``supports_required_and_named`` is False, required *and* named |
| 1004 | requests are parsed by the HYV4 extractor, which only recognizes the HYV4 |
| 1005 | structural tokens -- the default JSON guided-decoding path would make |
| 1006 | the model emit JSON the extractor silently drops. |
| 1007 | |
| 1008 | Args: |
| 1009 | request: The request being adjusted. |
| 1010 | reasoning: Whether the grammar also covers the reasoning phase. |
| 1011 | strict_level: Server-side floor from ``--tool-strict-level``. |
| 1012 | |
| 1013 | Returns: |
| 1014 | The structural tag, or None when structural tagging does not apply. |
| 1015 | |
| 1016 | """ |
| 1017 | if not envs.VLLM_ENFORCE_STRICT_TOOL_CALLING: |
| 1018 | return None |
| 1019 | |
| 1020 | # Only constrain required / forced tool choice with the structural tag. |
| 1021 | # For "auto", tool calling is optional and most requests never emit a |
| 1022 | # tool call, so applying the grammar to every auto request would make |
| 1023 | # the whole stream pay the output inflation and the per-token |
| 1024 | # bitmask/accept overhead for a minority benefit. Skip it here so auto |
| 1025 | # falls through to free generation; the HYV4 parser still reads back |
| 1026 | # any tool calls the model emits. |
| 1027 | tool_choice = request.tool_choice |
| 1028 | if ( |
| 1029 | tool_choice is None |
| 1030 | or tool_choice == "auto" |
| 1031 | or getattr(tool_choice, "mode", None) == "auto" |
| 1032 | ): |
| 1033 | return None |
| 1034 | |
| 1035 | from vllm.tool_parsers.structural_tag_registry import get_model_structural_tag |
| 1036 | |
| 1037 | try: |
| 1038 | return get_model_structural_tag( |
| 1039 | model=self.structural_tag_model, |
| 1040 | tools=request.tools, |
| 1041 | tool_choice=request.tool_choice, |
| 1042 | reasoning=reasoning, |
| 1043 | token_suffix=self._extractor.token_suffix, |
| 1044 | strict_level=strict_level, |
| 1045 | ) |
| 1046 | except Exception: |
| 1047 | logger.warning( |
| 1048 | "HYV4ToolParser failed to build a structural tag; falling back " |
| 1049 | "to the default guided-decoding path.", |
| 1050 | exc_info=True, |
| 1051 | ) |
| 1052 | return None |
| 1053 | |
| 1054 | def _mirror_state(self) -> None: |
| 1055 | # Alias the SAME list objects the extractor appends to, so the serving |
| 1056 | # layer reads live state. ``current_tool_id`` is an int, so re-read it. |
| 1057 | self.prev_tool_call_arr = self._extractor.prev_tool_call_arr |
| 1058 | self.streamed_args_for_tool = self._extractor.streamed_args_for_tool |
| 1059 | self.current_tool_id = self._extractor.current_tool_id |
| 1060 | |
| 1061 | @staticmethod |
| 1062 | def _is_guided(request: ChatCompletionRequest) -> bool: |
| 1063 | """Whether a structural tag constrains this request's decoding. |
| 1064 | |
| 1065 | ``get_structural_tag`` only applies to required / named tool choice and |
| 1066 | the serving layer records the result on the request. A structural tag |
| 1067 | constrains strings rather than token ids, so the ``<tool_calls>`` marker |
| 1068 | can arrive split across sub-word tokens and must be detected on the |
| 1069 | string level. |
| 1070 | |
| 1071 | Args: |
| 1072 | request: The request being streamed. |
| 1073 | |
| 1074 | Returns: |
| 1075 | True when the streaming parser must use the string-marker path. |
| 1076 | |
| 1077 | """ |
| 1078 | structured_outputs = getattr(request, "structured_outputs", None) |
| 1079 | return ( |
| 1080 | structured_outputs is not None |
| 1081 | and structured_outputs.structural_tag is not None |
| 1082 | ) |
| 1083 | |
| 1084 | def _tools(self, request: ChatCompletionRequest) -> list[ToolSchema] | None: |
| 1085 | # Prefer the tools passed at construction; fall back to the request. |
| 1086 | if self._plain_tools is not None: |
| 1087 | return self._plain_tools |
| 1088 | return _tools_to_plain(getattr(request, "tools", None)) |
| 1089 | |
| 1090 | def extract_tool_calls( |
| 1091 | self, |
| 1092 | model_output: str, |
| 1093 | request: ChatCompletionRequest, |
| 1094 | ) -> ExtractedToolCallInformation: |
| 1095 | result = self._extractor.extract_tool_calls(model_output, self._tools(request)) |
| 1096 | self._mirror_state() |
| 1097 | return ExtractedToolCallInformation( |
| 1098 | tools_called=result["tools_called"], |
| 1099 | content=result["content"], |
| 1100 | tool_calls=[ |
| 1101 | ToolCall( |
| 1102 | type="function", |
| 1103 | function=FunctionCall( |
| 1104 | name=tc["name"], |
| 1105 | arguments=tc["arguments"], |
| 1106 | ), |
| 1107 | ) |
| 1108 | for tc in result["tool_calls"] |
| 1109 | ], |
| 1110 | ) |
| 1111 | |
| 1112 | def extract_tool_calls_streaming( |
| 1113 | self, |
| 1114 | previous_text: str, |
| 1115 | current_text: str, |
| 1116 | delta_text: str, |
| 1117 | previous_token_ids: Sequence[int], |
| 1118 | current_token_ids: Sequence[int], |
| 1119 | delta_token_ids: Sequence[int], |
| 1120 | request: ChatCompletionRequest, |
| 1121 | ) -> DeltaMessage | None: |
| 1122 | delta = self._extractor.extract_tool_calls_streaming( |
| 1123 | previous_text, |
| 1124 | current_text, |
| 1125 | delta_text, |
| 1126 | previous_token_ids, |
| 1127 | current_token_ids, |
| 1128 | delta_token_ids, |
| 1129 | self._tools(request), |
| 1130 | guided=self._is_guided(request), |
| 1131 | ) |
| 1132 | self._mirror_state() |
| 1133 | if delta is None: |
| 1134 | return None |
| 1135 | |
| 1136 | tool_calls: list[DeltaToolCall] = [] |
| 1137 | for tc in delta.get("tool_calls", []): |
| 1138 | # Only set the fields that are present, leaving the rest UNSET (not |
| 1139 | # None). Streaming serialization uses exclude_unset, so an explicit |
| 1140 | # ``arguments=None`` would emit ``"arguments": null`` and break |
| 1141 | # clients that concatenate argument deltas. Matches hy_v3. |
| 1142 | fn_kwargs: dict[str, str] = {} |
| 1143 | name = tc.get("name") |
| 1144 | if name is not None: |
| 1145 | fn_kwargs["name"] = name |
| 1146 | arguments = tc.get("arguments") |
| 1147 | if arguments is not None: |
| 1148 | fn_kwargs["arguments"] = arguments |
| 1149 | function = DeltaFunctionCall(**fn_kwargs) |
| 1150 | if tc.get("new"): |
| 1151 | # First chunk for this tool: mint an id + set type here only. |
| 1152 | tool_calls.append( |
| 1153 | DeltaToolCall( |
| 1154 | index=tc["index"], |
| 1155 | id=make_tool_call_id(), |
| 1156 | type="function", |
| 1157 | function=function, |
| 1158 | ) |
| 1159 | ) |
| 1160 | else: |
| 1161 | tool_calls.append(DeltaToolCall(index=tc["index"], function=function)) |
| 1162 | |
| 1163 | # A single engine delta can carry content plus N drained tool calls. |
| 1164 | # Only set the fields that are present so streaming serialization |
| 1165 | # (exclude_unset) never emits ``content: null`` on a tool-call chunk. |
| 1166 | content = delta.get("content") |
| 1167 | kwargs: dict[str, Any] = {} |
| 1168 | if content is not None: |
| 1169 | kwargs["content"] = content |
| 1170 | if tool_calls: |
| 1171 | kwargs["tool_calls"] = tool_calls |
| 1172 | if not kwargs: |
| 1173 | return None |
| 1174 | return DeltaMessage(**kwargs) |