| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | import json |
| 5 | from collections.abc import Sequence |
| 6 | from typing import Any |
| 7 | |
| 8 | import regex as re |
| 9 | |
| 10 | from vllm.entrypoints.chat_utils import make_tool_call_id |
| 11 | from vllm.entrypoints.generate.base.protocol import ( |
| 12 | DeltaFunctionCall, |
| 13 | DeltaMessage, |
| 14 | DeltaToolCall, |
| 15 | ExtractedToolCallInformation, |
| 16 | FunctionCall, |
| 17 | ToolCall, |
| 18 | ) |
| 19 | from vllm.entrypoints.openai.chat_completion.protocol import ( |
| 20 | ChatCompletionRequest, |
| 21 | ChatCompletionToolsParam, |
| 22 | ) |
| 23 | from vllm.logger import init_logger |
| 24 | from vllm.tokenizers import TokenizerLike |
| 25 | from vllm.tool_parsers.abstract_tool_parser import ( |
| 26 | Tool, |
| 27 | ToolParser, |
| 28 | ) |
| 29 | from vllm.tool_parsers.utils import safe_literal_eval |
| 30 | |
| 31 | logger = init_logger(__name__) |
| 32 | |
| 33 | |
| 34 | class HYV3ToolParser(ToolParser): |
| 35 | _TYPE_ALIASES: dict[str, str] = { |
| 36 | "str": "string", |
| 37 | "text": "string", |
| 38 | "varchar": "string", |
| 39 | "char": "string", |
| 40 | "enum": "string", |
| 41 | "bool": "boolean", |
| 42 | "binary": "boolean", |
| 43 | "int": "integer", |
| 44 | "float": "number", |
| 45 | "double": "number", |
| 46 | "list": "array", |
| 47 | "dict": "object", |
| 48 | "map": "object", |
| 49 | } |
| 50 | |
| 51 | # Prefix-based wildcard matching for non-standard type names. |
| 52 | # Following the same approach as |
| 53 | # qwen3coder_tool_parser._convert_param_value which uses |
| 54 | # param_type.startswith("int"), startswith("uint"), etc. |
| 55 | _INTEGER_PREFIXES: tuple[str, ...] = ( |
| 56 | "int", |
| 57 | "uint", |
| 58 | "long", |
| 59 | "short", |
| 60 | "unsigned", |
| 61 | ) |
| 62 | _NUMBER_PREFIXES: tuple[str, ...] = ("num", "float") |
| 63 | |
| 64 | @staticmethod |
| 65 | def _normalize_type(raw_type: str) -> str: |
| 66 | """Map non-standard type aliases to JSON Schema standard names. |
| 67 | |
| 68 | First performs exact lookup in _TYPE_ALIASES. On miss, falls back |
| 69 | to prefix-based matching using startswith() |
| 70 | - int*/uint*/long*/short*/unsigned* → "integer" |
| 71 | - num*/float* → "number" |
| 72 | """ |
| 73 | exact = HYV3ToolParser._TYPE_ALIASES.get(raw_type) |
| 74 | if exact is not None: |
| 75 | return exact |
| 76 | lower = raw_type.lower() |
| 77 | if any(lower.startswith(p) for p in HYV3ToolParser._INTEGER_PREFIXES): |
| 78 | return "integer" |
| 79 | if any(lower.startswith(p) for p in HYV3ToolParser._NUMBER_PREFIXES): |
| 80 | return "number" |
| 81 | return raw_type |
| 82 | |
| 83 | @staticmethod |
| 84 | def _get_arg_schema( |
| 85 | function_name: str, |
| 86 | arg_key: str, |
| 87 | tools: list[ChatCompletionToolsParam] | None, |
| 88 | ) -> dict: |
| 89 | """Look up a specific argument's property schema from the tools list.""" |
| 90 | if tools is None: |
| 91 | return {} |
| 92 | for tool in tools: |
| 93 | if tool.function.name == function_name: |
| 94 | if tool.function.parameters is None: |
| 95 | return {} |
| 96 | return tool.function.parameters.get("properties", {}).get(arg_key, {}) |
| 97 | logger.warning("No tool named '%s'.", function_name) |
| 98 | return {} |
| 99 | |
| 100 | @staticmethod |
| 101 | def _get_schema_options(arg_schema: dict) -> list[dict]: |
| 102 | """Normalize any property schema into a list of sub-schemas. |
| 103 | - has type (single type) → return [arg_schema] |
| 104 | - anyOf → return the anyOf list |
| 105 | - oneOf → return the oneOf list |
| 106 | - fallback → [{"type": "string"}] |
| 107 | |
| 108 | Note: single ``type`` has the highest priority. |
| 109 | """ |
| 110 | if "type" in arg_schema: |
| 111 | type_val = arg_schema["type"] |
| 112 | # JSON Schema allows "type" to be an array to represent union types, |
| 113 | # e.g. "type": ["string", "object"]. |
| 114 | # Expand it into an anyOf-equivalent format: |
| 115 | # [{"type": "string"}, {"type": "object"}] |
| 116 | # so that _get_types / _parse_value can handle it uniformly later. |
| 117 | if isinstance(type_val, list): |
| 118 | return [{"type": t} for t in type_val] |
| 119 | return [arg_schema] |
| 120 | if "anyOf" in arg_schema: |
| 121 | return arg_schema["anyOf"] |
| 122 | if "oneOf" in arg_schema: |
| 123 | return arg_schema["oneOf"] |
| 124 | |
| 125 | return [{"type": "string"}] |
| 126 | |
| 127 | @staticmethod |
| 128 | def _get_types(arg_schema: dict) -> set[str]: |
| 129 | """Extract normalized, non-null type set from a property schema.""" |
| 130 | schemas = HYV3ToolParser._get_schema_options(arg_schema) |
| 131 | return { |
| 132 | HYV3ToolParser._normalize_type(s.get("type", "string")) for s in schemas |
| 133 | } - {"null"} |
| 134 | |
| 135 | @staticmethod |
| 136 | def _is_only_string_type( |
| 137 | function_name: str, |
| 138 | arg_key: str, |
| 139 | tools: list[ChatCompletionToolsParam] | None, |
| 140 | ) -> bool: |
| 141 | """Return True if the parameter's type set is exactly {"string"}. |
| 142 | |
| 143 | Only pure string types get partial value streaming; compound types |
| 144 | like anyOf(string | array) do not, since the partial value might |
| 145 | end up being a JSON array or object. |
| 146 | """ |
| 147 | arg_schema = HYV3ToolParser._get_arg_schema(function_name, arg_key, tools) |
| 148 | types = HYV3ToolParser._get_types(arg_schema) |
| 149 | return types == {"string"} |
| 150 | |
| 151 | @staticmethod |
| 152 | def _try_parse_bool(value: str) -> bool | None: |
| 153 | """Try to parse a string as bool; return None on failure.""" |
| 154 | lower = value.lower() |
| 155 | if lower == "true": |
| 156 | return True |
| 157 | elif lower == "false": |
| 158 | return False |
| 159 | return None |
| 160 | |
| 161 | @staticmethod |
| 162 | def _try_parse_int(value: str) -> int | None: |
| 163 | """Try to parse a string as int; return None on failure.""" |
| 164 | try: |
| 165 | return int(value) |
| 166 | except (ValueError, TypeError): |
| 167 | return None |
| 168 | |
| 169 | @staticmethod |
| 170 | def _try_parse_wildcard_number(value: str) -> int | float | None: |
| 171 | """Try to parse a string as a number (int or float). |
| 172 | |
| 173 | Decision rule: if the string contains '.' or 'e'/'E' (scientific |
| 174 | notation), parse as float; otherwise parse as int. |
| 175 | |
| 176 | Examples: |
| 177 | "5" → int(5) |
| 178 | "5.0" → float(5.0) |
| 179 | "5.3" → float(5.3) |
| 180 | "1e3" → float(1000.0) |
| 181 | "-3" → int(-3) |
| 182 | |
| 183 | Return None on failure. |
| 184 | |
| 185 | """ |
| 186 | try: |
| 187 | if "." in value or "e" in value or "E" in value: |
| 188 | return float(value) |
| 189 | return int(value) |
| 190 | except (ValueError, TypeError): |
| 191 | return None |
| 192 | |
| 193 | @staticmethod |
| 194 | def _deserialize(value: str) -> Any: |
| 195 | """Deserialize a string value using json.loads then safe_literal_eval.""" |
| 196 | try: |
| 197 | return json.loads(value) |
| 198 | except Exception: |
| 199 | pass |
| 200 | try: |
| 201 | return safe_literal_eval(value) |
| 202 | except Exception: |
| 203 | pass |
| 204 | return value |
| 205 | |
| 206 | @staticmethod |
| 207 | def _parse_value( |
| 208 | value: str, |
| 209 | function_name: str, |
| 210 | arg_key: str, |
| 211 | tools: list[ChatCompletionToolsParam] | None, |
| 212 | ) -> Any: |
| 213 | """Unified argument value parser with anyOf/oneOf support. |
| 214 | |
| 215 | Fallthrough chain: |
| 216 | bool → int → number(wildcard_number) |
| 217 | → json.loads for array/object |
| 218 | → string → _deserialize |
| 219 | """ |
| 220 | arg_schema = HYV3ToolParser._get_arg_schema(function_name, arg_key, tools) |
| 221 | types = HYV3ToolParser._get_types(arg_schema) |
| 222 | |
| 223 | # 1. Try bool |
| 224 | if "boolean" in types: |
| 225 | result_bool = HYV3ToolParser._try_parse_bool(value) |
| 226 | if result_bool is not None: |
| 227 | return result_bool |
| 228 | |
| 229 | # 2. Try int |
| 230 | if "integer" in types: |
| 231 | result_int = HYV3ToolParser._try_parse_int(value) |
| 232 | if result_int is not None: |
| 233 | return result_int |
| 234 | |
| 235 | # 3. Try number (wildcard_number: int if no '.'/e/E, float otherwise) |
| 236 | if "number" in types: |
| 237 | result_number = HYV3ToolParser._try_parse_wildcard_number(value) |
| 238 | if result_number is not None: |
| 239 | return result_number |
| 240 | |
| 241 | # 4. Try json.loads (covers array/object and other unlisted types) |
| 242 | if types - {"string", "boolean", "integer", "number"}: |
| 243 | try: |
| 244 | return json.loads(value) |
| 245 | except (json.JSONDecodeError, ValueError): |
| 246 | pass |
| 247 | |
| 248 | # 5. String fallback |
| 249 | if "string" in types: |
| 250 | return value |
| 251 | |
| 252 | # 6. Final fallback |
| 253 | return HYV3ToolParser._deserialize(value) |
| 254 | |
| 255 | def __init__(self, tokenizer: TokenizerLike, tools: list[Tool] | None = None): |
| 256 | super().__init__(tokenizer, tools) |
| 257 | |
| 258 | self.prev_tool_call_arr: list[dict] = [] |
| 259 | self.current_tool_id: int = -1 |
| 260 | self.streamed_args_for_tool: list[ |
| 261 | str |
| 262 | ] = [] # map what has been streamed for each tool so far to a list |
| 263 | |
| 264 | # Streaming state: send tool name first, then return arguments at once |
| 265 | self._streaming_tool_name: str | None = None # tool name being streamed |
| 266 | |
| 267 | # State fields for incremental argument streaming |
| 268 | self._completed_args: dict = {} # closed {key: parsed_value} |
| 269 | self._current_arg_key: str | None = None # key being collected |
| 270 | self._current_arg_is_string: bool = False # is current arg pure string? |
| 271 | self._streamed_json_len: int = 0 # bytes of JSON already sent |
| 272 | |
| 273 | init_kwargs = getattr(tokenizer, "init_kwargs", None) or {} |
| 274 | self.suffix: str = init_kwargs.get("token_suffix") or "" |
| 275 | |
| 276 | self.tool_calls_start_token: str = f"<tool_calls{self.suffix}>" |
| 277 | self.tool_calls_end_token: str = f"</tool_calls{self.suffix}>" |
| 278 | |
| 279 | self.tool_call_start_token: str = f"<tool_call{self.suffix}>" |
| 280 | self.tool_call_end_token: str = f"</tool_call{self.suffix}>" |
| 281 | |
| 282 | self.tool_sep_token: str = f"<tool_sep{self.suffix}>" |
| 283 | |
| 284 | self.arg_key_start_token: str = f"<arg_key{self.suffix}>" |
| 285 | self.arg_key_end_token: str = f"</arg_key{self.suffix}>" |
| 286 | |
| 287 | self.arg_value_start_token: str = f"<arg_value{self.suffix}>" |
| 288 | self.arg_value_end_token: str = f"</arg_value{self.suffix}>" |
| 289 | |
| 290 | self.tool_call_regex = re.compile( |
| 291 | rf"{self.tool_call_start_token}(.*?){self.tool_sep_token}" |
| 292 | rf"(.*?){self.tool_call_end_token}", |
| 293 | re.DOTALL, |
| 294 | ) |
| 295 | |
| 296 | self.tool_call_portion_regex = re.compile( |
| 297 | rf"{self.tool_call_start_token}(.*?){self.tool_sep_token}(.*)", re.DOTALL |
| 298 | ) |
| 299 | |
| 300 | self.func_args_regex = re.compile( |
| 301 | rf"{self.arg_key_start_token}(.*?){self.arg_key_end_token}\s*" |
| 302 | rf"{self.arg_value_start_token}(.*?){self.arg_value_end_token}", |
| 303 | re.DOTALL, |
| 304 | ) |
| 305 | |
| 306 | if not self.model_tokenizer: |
| 307 | raise ValueError( |
| 308 | "The model tokenizer must be passed to the ToolParser " |
| 309 | "constructor during construction." |
| 310 | ) |
| 311 | self.tool_calls_start_token_id = self.vocab.get(self.tool_calls_start_token) |
| 312 | self.tool_calls_end_token_id = self.vocab.get(self.tool_calls_end_token) |
| 313 | |
| 314 | self.tool_call_start_token_id = self.vocab.get(self.tool_call_start_token) |
| 315 | self.tool_call_end_token_id = self.vocab.get(self.tool_call_end_token) |
| 316 | self._buffer = "" |
| 317 | |
| 318 | if ( |
| 319 | self.tool_calls_start_token_id is None |
| 320 | or self.tool_calls_end_token_id is None |
| 321 | ): |
| 322 | raise RuntimeError( |
| 323 | "HYV3 Tool parser could not locate tool call " |
| 324 | "start/end tokens in the tokenizer!" |
| 325 | ) |
| 326 | |
| 327 | def _extract_tool_calls( |
| 328 | self, |
| 329 | model_output: str, |
| 330 | request: ChatCompletionRequest, |
| 331 | ) -> list[ToolCall]: |
| 332 | try: |
| 333 | function_call_tuples = [] |
| 334 | # start_token{name}sep_token{args}end_token... |
| 335 | function_calls = self.tool_call_regex.findall(model_output) |
| 336 | if function_calls: |
| 337 | function_call_tuples.extend(function_calls) |
| 338 | remaining = model_output.split(self.tool_call_end_token)[-1] |
| 339 | function_calls = self.tool_call_portion_regex.findall(remaining) |
| 340 | function_call_tuples += function_calls |
| 341 | else: |
| 342 | function_calls = self.tool_call_portion_regex.findall(model_output) |
| 343 | if function_calls: |
| 344 | function_call_tuples.extend(function_calls) |
| 345 | tool_calls = [] |
| 346 | for match in function_call_tuples: |
| 347 | function_name, function_args = match |
| 348 | function_name = function_name.strip() |
| 349 | function_args = function_args.strip() |
| 350 | |
| 351 | arg_pairs = self.func_args_regex.findall(function_args) |
| 352 | arg_dict = {} |
| 353 | for key, value in arg_pairs: |
| 354 | parsed_value = HYV3ToolParser._parse_value( |
| 355 | value, function_name, key, request.tools |
| 356 | ) |
| 357 | arg_dict[key] = parsed_value |
| 358 | tool_calls.append( |
| 359 | ToolCall( |
| 360 | type="function", |
| 361 | function=FunctionCall( |
| 362 | name=function_name, |
| 363 | arguments=json.dumps(arg_dict, ensure_ascii=False), |
| 364 | ), |
| 365 | ) |
| 366 | ) |
| 367 | return tool_calls |
| 368 | except Exception: |
| 369 | logger.exception("Error in extracting tool call from response.") |
| 370 | return [] |
| 371 | |
| 372 | def extract_tool_calls( |
| 373 | self, |
| 374 | model_output: str, |
| 375 | request: ChatCompletionRequest, |
| 376 | ) -> ExtractedToolCallInformation: |
| 377 | # sanity check; avoid unnecessary processing |
| 378 | if self.tool_calls_start_token not in model_output: |
| 379 | return ExtractedToolCallInformation( |
| 380 | tools_called=False, tool_calls=[], content=model_output |
| 381 | ) |
| 382 | else: |
| 383 | try: |
| 384 | tool_calls = self._extract_tool_calls(model_output, request) |
| 385 | |
| 386 | s_index = model_output.find(self.tool_calls_start_token) |
| 387 | content = model_output[:s_index] if s_index != -1 else model_output |
| 388 | return ExtractedToolCallInformation( |
| 389 | tools_called=True, |
| 390 | tool_calls=tool_calls, |
| 391 | content=content if content else None, |
| 392 | ) |
| 393 | |
| 394 | except Exception: |
| 395 | logger.exception("Error in extracting tool call from response.") |
| 396 | return ExtractedToolCallInformation( |
| 397 | tools_called=False, tool_calls=[], content=model_output |
| 398 | ) |
| 399 | |
| 400 | def _reset_streaming_tool_state(self): |
| 401 | """Reset the streaming state for a single tool call.""" |
| 402 | self._streaming_tool_name = None |
| 403 | self._completed_args = {} |
| 404 | self._current_arg_key = None |
| 405 | self._current_arg_is_string = False |
| 406 | self._streamed_json_len = 0 |
| 407 | |
| 408 | def extract_tool_calls_streaming( |
| 409 | self, |
| 410 | previous_text: str, |
| 411 | current_text: str, |
| 412 | delta_text: str, |
| 413 | previous_token_ids: Sequence[int], |
| 414 | current_token_ids: Sequence[int], |
| 415 | delta_token_ids: Sequence[int], |
| 416 | request: ChatCompletionRequest, |
| 417 | ) -> DeltaMessage | None: |
| 418 | # Check whether current tokens contain the tool_calls start token |
| 419 | if self.tool_calls_start_token_id not in current_token_ids: |
| 420 | return DeltaMessage(content=delta_text) |
| 421 | |
| 422 | # Encountered tool_calls start tag; extract preceding content and buffer |
| 423 | content_delta: str | None = None |
| 424 | if self.tool_calls_start_token in delta_text: |
| 425 | text_parts = delta_text.split(self.tool_calls_start_token) |
| 426 | self._buffer += text_parts[-1] |
| 427 | content_delta = text_parts[0] or None |
| 428 | else: |
| 429 | self._buffer += delta_text |
| 430 | |
| 431 | # Encountered finish, extract valid arguments |
| 432 | if ( |
| 433 | current_text.find(self.tool_call_end_token + self.tool_calls_end_token) |
| 434 | != -1 |
| 435 | and self._buffer.find(self.tool_call_end_token) == -1 |
| 436 | ): |
| 437 | self._buffer += self.tool_call_end_token + self.tool_calls_end_token |
| 438 | |
| 439 | cur_text = self._buffer |
| 440 | |
| 441 | # Haven't encountered tool_call start tag yet; keep buffering |
| 442 | start_idx = cur_text.find(self.tool_call_start_token) |
| 443 | if start_idx == -1 and self._streaming_tool_name is None: |
| 444 | if content_delta is not None: |
| 445 | return DeltaMessage(content=content_delta) |
| 446 | self._buffer = "" |
| 447 | return None |
| 448 | |
| 449 | # === Phase 1: Detect tool name (send when tool_sep_token is seen) === |
| 450 | name_delta: DeltaMessage | None = None |
| 451 | if self._streaming_tool_name is None: |
| 452 | sep_idx = cur_text.find(self.tool_sep_token) |
| 453 | if sep_idx == -1: |
| 454 | # tool_sep not yet seen; keep buffering from tool_call_start |
| 455 | self._buffer = cur_text[start_idx:] |
| 456 | if content_delta is None: |
| 457 | return None |
| 458 | return DeltaMessage(content=content_delta) |
| 459 | |
| 460 | # Extract tool name: between tool_call_start_token and tool_sep_token |
| 461 | name_start = start_idx + len(self.tool_call_start_token) |
| 462 | tool_name = cur_text[name_start:sep_idx].strip() |
| 463 | self._streaming_tool_name = tool_name |
| 464 | |
| 465 | # Update buffer: keep only content after tool_sep (i.e. the args portion) |
| 466 | self._buffer = cur_text[sep_idx + len(self.tool_sep_token) :] |
| 467 | |
| 468 | # Increment tool_id and send a chunk containing only the name |
| 469 | self.current_tool_id += 1 |
| 470 | self._current_tool_call_id = make_tool_call_id() |
| 471 | name_delta = DeltaMessage( |
| 472 | tool_calls=[ |
| 473 | DeltaToolCall( |
| 474 | index=self.current_tool_id, |
| 475 | id=self._current_tool_call_id, |
| 476 | type="function", |
| 477 | function=DeltaFunctionCall( |
| 478 | name=tool_name, |
| 479 | ), |
| 480 | ) |
| 481 | ] |
| 482 | ) |
| 483 | name_delta.content = content_delta |
| 484 | |
| 485 | # Check if buffer already has complete arguments (all-in-one-delta) |
| 486 | if self.tool_call_end_token not in self._buffer: |
| 487 | return name_delta |
| 488 | # Buffer already has a complete tool call; continue to phase 2 below |
| 489 | |
| 490 | # === Phase 2: Incremental argument streaming === |
| 491 | return self._extract_streaming_incremental(name_delta, request) |
| 492 | |
| 493 | def _make_args_delta(self, argument_diff: str) -> DeltaMessage: |
| 494 | """Build a DeltaMessage containing only an arguments diff.""" |
| 495 | return DeltaMessage( |
| 496 | tool_calls=[ |
| 497 | DeltaToolCall( |
| 498 | index=self.current_tool_id, |
| 499 | function=DeltaFunctionCall(arguments=argument_diff), |
| 500 | ) |
| 501 | ] |
| 502 | ) |
| 503 | |
| 504 | def _drain_calls(self, request: ChatCompletionRequest) -> list[DeltaToolCall]: |
| 505 | deltas: list[DeltaToolCall] = [] |
| 506 | while (start_idx := self._buffer.find(self.tool_call_start_token)) != -1: |
| 507 | end_idx = self._buffer.find(self.tool_call_end_token, start_idx) |
| 508 | if end_idx == -1: |
| 509 | break |
| 510 | end_idx += len(self.tool_call_end_token) |
| 511 | complete_call = self._buffer[start_idx:end_idx] |
| 512 | self._buffer = self._buffer[end_idx:] |
| 513 | for call in self._extract_tool_calls(complete_call, request): |
| 514 | self.current_tool_id += 1 |
| 515 | arguments = call.function.arguments |
| 516 | self.prev_tool_call_arr.append( |
| 517 | {"name": call.function.name, "arguments": json.loads(arguments)} |
| 518 | ) |
| 519 | self.streamed_args_for_tool.append(arguments) |
| 520 | deltas.append( |
| 521 | DeltaToolCall( |
| 522 | index=self.current_tool_id, |
| 523 | id=make_tool_call_id(), |
| 524 | type="function", |
| 525 | function=DeltaFunctionCall( |
| 526 | name=call.function.name, arguments=arguments |
| 527 | ), |
| 528 | ) |
| 529 | ) |
| 530 | return deltas |
| 531 | |
| 532 | def _extract_streaming_incremental( |
| 533 | self, |
| 534 | name_delta: DeltaMessage | None, |
| 535 | request: ChatCompletionRequest, |
| 536 | ) -> DeltaMessage | None: |
| 537 | """Incremental phase-2: scan tags in buffer, emit JSON diffs. |
| 538 | |
| 539 | Strategy: |
| 540 | - Track completed args and emit each one as a JSON fragment. |
| 541 | - For string-typed args, stream the value character-by-character. |
| 542 | - Withhold the closing ``}`` until ``</tool_call>`` is seen. |
| 543 | |
| 544 | We build JSON manually via fragments rather than using json.dumps |
| 545 | with a cursor, because json.dumps of partial-vs-full string values |
| 546 | produces incompatible prefixes (e.g. ``""}`` vs ``"Hello"}``). |
| 547 | """ |
| 548 | buf = self._buffer |
| 549 | is_complete = self.tool_call_end_token in buf |
| 550 | |
| 551 | if is_complete: |
| 552 | end_idx = buf.find(self.tool_call_end_token) |
| 553 | args_text = buf[:end_idx] |
| 554 | remaining = buf[end_idx + len(self.tool_call_end_token) :] |
| 555 | else: |
| 556 | args_text = buf |
| 557 | remaining = "" |
| 558 | |
| 559 | # --- scan all fully closed kv pairs --- |
| 560 | arg_pairs = self.func_args_regex.findall(args_text) |
| 561 | for key, value in arg_pairs: |
| 562 | key = key.strip() |
| 563 | if key not in self._completed_args: |
| 564 | parsed_value = HYV3ToolParser._parse_value( |
| 565 | value, self._streaming_tool_name or "", key, request.tools |
| 566 | ) |
| 567 | self._completed_args[key] = parsed_value |
| 568 | |
| 569 | # --- detect partial (unclosed) kv at the tail --- |
| 570 | last_closed_end = 0 |
| 571 | for m in self.func_args_regex.finditer(args_text): |
| 572 | last_closed_end = m.end() |
| 573 | tail = args_text[last_closed_end:] |
| 574 | |
| 575 | partial_key: str | None = None |
| 576 | partial_value: str | None = None |
| 577 | |
| 578 | ak_start = tail.find(self.arg_key_start_token) |
| 579 | if ak_start != -1: |
| 580 | ak_end = tail.find( |
| 581 | self.arg_key_end_token, |
| 582 | ak_start + len(self.arg_key_start_token), |
| 583 | ) |
| 584 | if ak_end != -1: |
| 585 | partial_key = tail[ |
| 586 | ak_start + len(self.arg_key_start_token) : ak_end |
| 587 | ].strip() |
| 588 | self._current_arg_key = partial_key |
| 589 | self._current_arg_is_string = HYV3ToolParser._is_only_string_type( |
| 590 | self._streaming_tool_name or "", |
| 591 | partial_key, |
| 592 | request.tools, |
| 593 | ) |
| 594 | |
| 595 | av_start = tail.find(self.arg_value_start_token, ak_end) |
| 596 | if av_start != -1: |
| 597 | val_content_start = av_start + len(self.arg_value_start_token) |
| 598 | if self._current_arg_is_string: |
| 599 | partial_value = tail[val_content_start:] |
| 600 | else: |
| 601 | # key not yet closed |
| 602 | self._current_arg_key = None |
| 603 | self._current_arg_is_string = False |
| 604 | |
| 605 | # --- build the current JSON snapshot as a string --- |
| 606 | # We construct JSON manually so we can precisely control |
| 607 | # what gets sent incrementally. |
| 608 | snapshot_parts: list[str] = [] |
| 609 | for k, v in self._completed_args.items(): |
| 610 | k_json = json.dumps(k, ensure_ascii=False) |
| 611 | v_json = json.dumps(v, ensure_ascii=False) |
| 612 | snapshot_parts.append(f"{k_json}: {v_json}") |
| 613 | |
| 614 | if partial_key is not None and partial_value is not None: |
| 615 | k_json = json.dumps(partial_key, ensure_ascii=False) |
| 616 | # For string partial value, we build the JSON string |
| 617 | # WITHOUT the closing quote, so the prefix stays stable |
| 618 | # as the value grows. The closing `"` and `}` will be |
| 619 | # sent when the value or tool_call closes. |
| 620 | escaped_val = ( |
| 621 | partial_value.replace("\\", "\\\\") |
| 622 | .replace('"', '\\"') |
| 623 | .replace("\n", "\\n") |
| 624 | .replace("\r", "\\r") |
| 625 | .replace("\t", "\\t") |
| 626 | ) |
| 627 | # Note: no closing " here – it's appended only on close |
| 628 | snapshot_parts.append(f'{k_json}: "{escaped_val}') |
| 629 | |
| 630 | snapshot = "{" + ", ".join(snapshot_parts) + "}" |
| 631 | |
| 632 | # --- compute diff --- |
| 633 | argument_diff: str | None = None |
| 634 | |
| 635 | if is_complete: |
| 636 | # Tool call finished – send everything remaining. |
| 637 | # Build final snapshot with proper JSON (all values closed). |
| 638 | final_args = dict(self._completed_args) |
| 639 | final_json = json.dumps(final_args, ensure_ascii=False) |
| 640 | if self._streamed_json_len < len(final_json): |
| 641 | argument_diff = final_json[self._streamed_json_len :] |
| 642 | self._streamed_json_len = len(final_json) |
| 643 | |
| 644 | # Record into prev_tool_call_arr |
| 645 | self.prev_tool_call_arr.append( |
| 646 | { |
| 647 | "name": self._streaming_tool_name, |
| 648 | "arguments": final_args, |
| 649 | } |
| 650 | ) |
| 651 | self.streamed_args_for_tool.append(final_json) |
| 652 | |
| 653 | self._reset_streaming_tool_state() |
| 654 | self._buffer = remaining |
| 655 | else: |
| 656 | # Still in progress – withhold the tail. |
| 657 | # For open strings: snapshot ends with ...partial_val} |
| 658 | # we withhold "}" (1 char) – the missing closing " will |
| 659 | # be sent when the value closes. |
| 660 | # For no open string: snapshot ends with ...value"} |
| 661 | # we withhold "}" (1 char). |
| 662 | end = len(snapshot) - 1 # exclude trailing "}" |
| 663 | if end > self._streamed_json_len: |
| 664 | argument_diff = snapshot[self._streamed_json_len : end] |
| 665 | self._streamed_json_len = end |
| 666 | |
| 667 | # --- construct return DeltaMessage --- |
| 668 | delta: DeltaMessage | None |
| 669 | if name_delta is not None and argument_diff: |
| 670 | nd_func = name_delta.tool_calls[0].function |
| 671 | delta = DeltaMessage( |
| 672 | tool_calls=[ |
| 673 | DeltaToolCall( |
| 674 | index=self.current_tool_id, |
| 675 | id=self._current_tool_call_id, |
| 676 | type="function", |
| 677 | function=DeltaFunctionCall( |
| 678 | name=nd_func.name if nd_func else None, |
| 679 | arguments=argument_diff, |
| 680 | ), |
| 681 | ) |
| 682 | ] |
| 683 | ) |
| 684 | delta.content = name_delta.content |
| 685 | elif name_delta is not None: |
| 686 | delta = name_delta |
| 687 | elif argument_diff: |
| 688 | delta = self._make_args_delta(argument_diff) |
| 689 | else: |
| 690 | delta = None |
| 691 | |
| 692 | extra_tool_calls = self._drain_calls(request) if is_complete else [] |
| 693 | if extra_tool_calls: |
| 694 | if delta is None: |
| 695 | return DeltaMessage(tool_calls=extra_tool_calls) |
| 696 | delta.tool_calls.extend(extra_tool_calls) |
| 697 | return delta |