| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | import json |
| 5 | from collections.abc import Sequence |
| 6 | from typing import Any |
| 7 | |
| 8 | import regex as re |
| 9 | |
| 10 | from vllm.entrypoints.chat_utils import make_tool_call_id |
| 11 | from vllm.entrypoints.generate.base.protocol import ( |
| 12 | DeltaFunctionCall, |
| 13 | DeltaMessage, |
| 14 | DeltaToolCall, |
| 15 | ExtractedToolCallInformation, |
| 16 | FunctionCall, |
| 17 | ToolCall, |
| 18 | ) |
| 19 | from vllm.entrypoints.openai.chat_completion.protocol import ( |
| 20 | ChatCompletionRequest, |
| 21 | ChatCompletionToolsParam, |
| 22 | ) |
| 23 | from vllm.entrypoints.openai.responses.protocol import ResponsesRequest |
| 24 | from vllm.logger import init_logger |
| 25 | from vllm.tokenizers import TokenizerLike |
| 26 | from vllm.tool_parsers.abstract_tool_parser import ( |
| 27 | Tool, |
| 28 | ToolParser, |
| 29 | ) |
| 30 | from vllm.tool_parsers.utils import partial_tag_overlap, safe_literal_eval |
| 31 | from vllm.utils import random_uuid |
| 32 | |
| 33 | logger = init_logger(__name__) |
| 34 | |
| 35 | try: |
| 36 | from lxml import etree as ET # type: ignore |
| 37 | |
| 38 | _HAS_LXML = True |
| 39 | except Exception: # pragma: no cover |
| 40 | import xml.etree.ElementTree as ET # type: ignore |
| 41 | |
| 42 | _HAS_LXML = False |
| 43 | |
| 44 | _FUNC_NAME_V1_REGEX = re.compile(r"<function\s+name=['\"]([^'\"]+)['\"][^>]*>") |
| 45 | _PARAM_WITH_NAME_REGEX = re.compile( |
| 46 | r"<param\s+name=['\"]([^'\"]+)['\"]>([\s\S]*?)</param>", re.DOTALL |
| 47 | ) |
| 48 | _PARAM_MISSING_NAME_REGEX = re.compile(r"<param(?![^>]*\bname=)[^>]*>", re.DOTALL) |
| 49 | _FUNC_BLOCK_REGEX = re.compile(r"<function.*?</function>", re.DOTALL) |
| 50 | |
| 51 | # SentencePiece/GPT-style decoders may emit U+0120 (Ġ) / U+010A (Ċ). |
| 52 | _TOKENIZER_SPACE = "\u0120" |
| 53 | _TOKENIZER_NEWLINE = "\u010a" |
| 54 | |
| 55 | |
| 56 | def _normalize_model_output(text: str) -> str: |
| 57 | if ( |
| 58 | _TOKENIZER_SPACE not in text |
| 59 | and _TOKENIZER_NEWLINE not in text |
| 60 | and "<functionname=" not in text |
| 61 | and "<paramname=" not in text |
| 62 | ): |
| 63 | return text |
| 64 | |
| 65 | normalized = text.replace(_TOKENIZER_SPACE, " ") |
| 66 | normalized = normalized.replace(_TOKENIZER_NEWLINE, "\n") |
| 67 | # Some model outputs collapse tag names and attributes, e.g. |
| 68 | # <functionname="foo"> or <paramname="bar">. |
| 69 | normalized = normalized.replace("<functionname=", "<function name=") |
| 70 | normalized = normalized.replace("<paramname=", "<param name=") |
| 71 | return normalized |
| 72 | |
| 73 | |
| 74 | def _strip_thinking_content(text: str) -> str: |
| 75 | """Return only user-visible content after MiniCPM5 thinking, when present.""" |
| 76 | if "</think>" not in text: |
| 77 | return text |
| 78 | visible = text.rsplit("</think>", 1)[-1].lstrip() |
| 79 | return visible if visible else text |
| 80 | |
| 81 | |
| 82 | def _streaming_args_snapshot(args_json: str, *, is_complete: bool) -> str: |
| 83 | """Return the streamed arguments prefix, omitting the closing brace until done.""" |
| 84 | if is_complete or not args_json.endswith("}"): |
| 85 | return args_json |
| 86 | return args_json[:-1] |
| 87 | |
| 88 | |
| 89 | def _streaming_args_diff( |
| 90 | prev_args: str, args_json: str, *, is_complete: bool |
| 91 | ) -> str | None: |
| 92 | """Compute the next arguments fragment for OpenAI-style streaming accumulation.""" |
| 93 | if prev_args == "{}": |
| 94 | prev_args = "" |
| 95 | |
| 96 | target = _streaming_args_snapshot(args_json, is_complete=is_complete) |
| 97 | if not prev_args: |
| 98 | return target or None |
| 99 | if target == prev_args: |
| 100 | return None |
| 101 | if target.startswith(prev_args): |
| 102 | return target[len(prev_args) :] or None |
| 103 | |
| 104 | # Recover from a previously closed partial JSON snapshot. |
| 105 | if prev_args.endswith("}"): |
| 106 | prev_open = prev_args[:-1] |
| 107 | if target.startswith(prev_open): |
| 108 | return target[len(prev_open) :] or None |
| 109 | |
| 110 | return None |
| 111 | |
| 112 | |
| 113 | def _parse_arguments(json_value: str) -> tuple[Any, bool]: |
| 114 | try: |
| 115 | try: |
| 116 | parsed_value = json.loads(json_value) |
| 117 | except json.JSONDecodeError: |
| 118 | parsed_value = safe_literal_eval(json_value) |
| 119 | return parsed_value, True |
| 120 | except Exception: |
| 121 | return json_value, False |
| 122 | |
| 123 | |
| 124 | def _get_argument_type( |
| 125 | func_name: str, |
| 126 | arg_key: str, |
| 127 | name_to_tool: dict[str, ChatCompletionToolsParam], |
| 128 | ) -> str | None: |
| 129 | tool = name_to_tool.get(func_name) |
| 130 | if tool is None or tool.function.parameters is None: |
| 131 | return None |
| 132 | props = tool.function.parameters.get("properties", {}) |
| 133 | if arg_key not in props: |
| 134 | return None |
| 135 | return props[arg_key].get("type") |
| 136 | |
| 137 | |
| 138 | def _coerce_argument_value( |
| 139 | func_name: str, |
| 140 | arg_key: str, |
| 141 | value: Any, |
| 142 | name_to_tool: dict[str, ChatCompletionToolsParam], |
| 143 | ) -> Any: |
| 144 | arg_type = _get_argument_type(func_name, arg_key, name_to_tool) |
| 145 | if arg_type == "string": |
| 146 | return ( |
| 147 | value if isinstance(value, str) else json.dumps(value, ensure_ascii=False) |
| 148 | ) |
| 149 | if isinstance(value, str): |
| 150 | parsed_val, _ = _parse_arguments(value) |
| 151 | return parsed_val |
| 152 | return value |
| 153 | |
| 154 | |
| 155 | def _add_argument( |
| 156 | func_name: str, |
| 157 | key: str, |
| 158 | val_text: str, |
| 159 | arguments: dict[str, Any], |
| 160 | seen_keys: set[str], |
| 161 | allowed_props: set[str], |
| 162 | name_to_tool: dict[str, ChatCompletionToolsParam], |
| 163 | ) -> bool: |
| 164 | """Add one argument; return False if it violates the tool schema.""" |
| 165 | # MiniCPM5 can emit OpenAI-style wrappers, e.g. |
| 166 | # <param name="properties">{'id': '...'}</param>. Keep known fields and |
| 167 | # ignore extra fields so valid tool calls do not fall back to long text. |
| 168 | if ( |
| 169 | key in {"properties", "arguments"} |
| 170 | and allowed_props |
| 171 | and key not in allowed_props |
| 172 | ): |
| 173 | parsed_val, parsed_ok = _parse_arguments(val_text) |
| 174 | if not parsed_ok or not isinstance(parsed_val, dict): |
| 175 | return False |
| 176 | added = False |
| 177 | for wrapped_key, wrapped_value in parsed_val.items(): |
| 178 | if wrapped_key not in allowed_props: |
| 179 | continue |
| 180 | if wrapped_key in seen_keys: |
| 181 | return False |
| 182 | seen_keys.add(wrapped_key) |
| 183 | arguments[wrapped_key] = _coerce_argument_value( |
| 184 | func_name, wrapped_key, wrapped_value, name_to_tool |
| 185 | ) |
| 186 | added = True |
| 187 | return added |
| 188 | |
| 189 | if allowed_props and key not in allowed_props: |
| 190 | return True |
| 191 | if key in seen_keys: |
| 192 | return False |
| 193 | seen_keys.add(key) |
| 194 | arguments[key] = _coerce_argument_value(func_name, key, val_text, name_to_tool) |
| 195 | return True |
| 196 | |
| 197 | |
| 198 | def _normalize_alias_tool_call( |
| 199 | func_name: str, |
| 200 | arguments: dict[str, Any], |
| 201 | tool_names: set[str], |
| 202 | ) -> tuple[str, dict[str, Any]]: |
| 203 | """Map common MiniCPM5 alias tool names to the exposed tool schema.""" |
| 204 | if func_name in tool_names: |
| 205 | return func_name, arguments |
| 206 | |
| 207 | if func_name == "get_details_by_phone" and "get_customer_by_phone" in tool_names: |
| 208 | phone_number = arguments.get("phone_number") or arguments.get("phone") |
| 209 | if phone_number is not None: |
| 210 | return "get_customer_by_phone", {"phone_number": phone_number} |
| 211 | |
| 212 | if func_name == "get_details_by_name" and "get_customer_by_name" in tool_names: |
| 213 | full_name = arguments.get("full_name") or arguments.get("name") |
| 214 | dob = arguments.get("date_of_birth") or arguments.get("dob") |
| 215 | mapped = {} |
| 216 | if full_name is not None: |
| 217 | mapped["full_name"] = full_name |
| 218 | if dob is not None: |
| 219 | mapped["date_of_birth"] = dob |
| 220 | mapped["dob"] = dob |
| 221 | if mapped: |
| 222 | return "get_customer_by_name", mapped |
| 223 | |
| 224 | if ( |
| 225 | func_name |
| 226 | in { |
| 227 | "get_line_details", |
| 228 | "get_line_status", |
| 229 | "get_roaming_status", |
| 230 | } |
| 231 | and "get_details_by_id" in tool_names |
| 232 | ): |
| 233 | detail_id = arguments.get("line_id") or arguments.get("id") |
| 234 | if detail_id is not None: |
| 235 | return "get_details_by_id", {"id": detail_id} |
| 236 | |
| 237 | if func_name == "get_plan_details" and "get_details_by_id" in tool_names: |
| 238 | plan_id = arguments.get("plan_id") or arguments.get("id") |
| 239 | if plan_id is not None: |
| 240 | return "get_details_by_id", {"id": plan_id} |
| 241 | |
| 242 | if func_name == "enable_roaming" and "toggle_roaming" in tool_names: |
| 243 | line_id = arguments.get("line_id") or arguments.get("id") |
| 244 | if line_id is not None: |
| 245 | return "toggle_roaming", {"line_id": line_id, "enabled": True} |
| 246 | |
| 247 | if func_name == "disable_roaming" and "toggle_roaming" in tool_names: |
| 248 | line_id = arguments.get("line_id") or arguments.get("id") |
| 249 | if line_id is not None: |
| 250 | return "toggle_roaming", {"line_id": line_id, "enabled": False} |
| 251 | |
| 252 | if ( |
| 253 | func_name in {"add_refueled_data", "add_data_to_line"} |
| 254 | and "refuel_data" in tool_names |
| 255 | ): |
| 256 | line_id = arguments.get("line_id") or arguments.get("id") |
| 257 | amount = ( |
| 258 | arguments.get("amount_gb") |
| 259 | or arguments.get("gb") |
| 260 | or arguments.get("gb_amount") |
| 261 | or arguments.get("amount") |
| 262 | ) |
| 263 | mapped = {} |
| 264 | if "customer_id" in arguments: |
| 265 | mapped["customer_id"] = arguments["customer_id"] |
| 266 | if line_id is not None: |
| 267 | mapped["line_id"] = line_id |
| 268 | if amount is not None: |
| 269 | mapped["amount_gb"] = amount |
| 270 | if mapped: |
| 271 | return "refuel_data", mapped |
| 272 | |
| 273 | return func_name, arguments |
| 274 | |
| 275 | |
| 276 | def _build_tool_maps( |
| 277 | tools: list[ChatCompletionToolsParam] | None, |
| 278 | ) -> tuple[ |
| 279 | set[str], |
| 280 | dict[str, set[str]], |
| 281 | dict[str, set[str]], |
| 282 | dict[str, ChatCompletionToolsParam], |
| 283 | ]: |
| 284 | name_to_tool: dict[str, ChatCompletionToolsParam] = {} |
| 285 | name_to_allowed_props: dict[str, set[str]] = {} |
| 286 | name_to_required: dict[str, set[str]] = {} |
| 287 | |
| 288 | for tool in tools or []: |
| 289 | name = tool.function.name |
| 290 | if not name: |
| 291 | continue |
| 292 | name_to_tool[name] = tool |
| 293 | params = tool.function.parameters or {} |
| 294 | props = params.get("properties", {}) if isinstance(params, dict) else {} |
| 295 | name_to_allowed_props[name] = set(props.keys()) |
| 296 | required = params.get("required", []) if isinstance(params, dict) else [] |
| 297 | try: |
| 298 | name_to_required[name] = set(required) |
| 299 | except TypeError: |
| 300 | logger.warning( |
| 301 | "Failed to parse 'required' field for tool %s. " |
| 302 | "It should be a list of strings. Got: %s", |
| 303 | name, |
| 304 | required, |
| 305 | ) |
| 306 | name_to_required[name] = set() |
| 307 | |
| 308 | return ( |
| 309 | set(name_to_tool.keys()), |
| 310 | name_to_allowed_props, |
| 311 | name_to_required, |
| 312 | name_to_tool, |
| 313 | ) |
| 314 | |
| 315 | |
| 316 | def _parse_function_block( |
| 317 | block: str, |
| 318 | tool_names: set[str], |
| 319 | name_to_allowed_props: dict[str, set[str]], |
| 320 | name_to_required: dict[str, set[str]], |
| 321 | name_to_tool: dict[str, ChatCompletionToolsParam], |
| 322 | ) -> dict[str, Any] | None: |
| 323 | """Return {name, parameters} if block is valid, else None.""" |
| 324 | func_name: str | None = None |
| 325 | arguments: dict[str, Any] = {} |
| 326 | parsed_ok = False |
| 327 | param_invalid = False |
| 328 | |
| 329 | try: |
| 330 | if _HAS_LXML: |
| 331 | try: |
| 332 | parser = ET.XMLParser(**{"strip_cdata": False}) # type: ignore[call-arg] |
| 333 | except TypeError: |
| 334 | parser = ET.XMLParser() |
| 335 | root = ET.fromstring(block, parser=parser) |
| 336 | else: |
| 337 | root = ET.fromstring(block) |
| 338 | |
| 339 | if root.tag == "function": |
| 340 | func_node = root |
| 341 | else: |
| 342 | func_node = root.find("function") if hasattr(root, "find") else None |
| 343 | |
| 344 | if func_node is not None: |
| 345 | func_name = (func_node.attrib.get("name") or "").strip() |
| 346 | |
| 347 | args_node = func_node.find("arguments") if func_node is not None else None |
| 348 | param_nodes: list[Any] = [] |
| 349 | if func_node is not None: |
| 350 | param_nodes = list(func_node.findall("param")) |
| 351 | if args_node is not None and not param_nodes: |
| 352 | param_nodes = list(args_node.findall("param")) |
| 353 | |
| 354 | if func_node is not None: |
| 355 | seen_keys: set[str] = set() |
| 356 | allowed_props = name_to_allowed_props.get(func_name or "", set()) |
| 357 | has_invalid_param = False |
| 358 | for param in param_nodes: |
| 359 | key = param.attrib.get("name") |
| 360 | if not key: |
| 361 | has_invalid_param = True |
| 362 | break |
| 363 | val_text = param.text or "" |
| 364 | if not _add_argument( |
| 365 | func_name or "", |
| 366 | key, |
| 367 | val_text, |
| 368 | arguments, |
| 369 | seen_keys, |
| 370 | allowed_props, |
| 371 | name_to_tool, |
| 372 | ): |
| 373 | has_invalid_param = True |
| 374 | break |
| 375 | if has_invalid_param: |
| 376 | arguments.clear() |
| 377 | param_invalid = True |
| 378 | parsed_ok = bool(func_name) |
| 379 | except Exception: |
| 380 | parsed_ok = False |
| 381 | |
| 382 | if not parsed_ok: |
| 383 | try: |
| 384 | m_fn = _FUNC_NAME_V1_REGEX.search(block) |
| 385 | if m_fn: |
| 386 | func_name = (m_fn.group(1) or "").strip() |
| 387 | has_invalid_param = bool(_PARAM_MISSING_NAME_REGEX.search(block)) |
| 388 | seen_keys = set() |
| 389 | allowed_props = name_to_allowed_props.get(func_name or "", set()) |
| 390 | for pm in _PARAM_WITH_NAME_REGEX.finditer(block): |
| 391 | key = pm.group(1).strip() |
| 392 | val_text = pm.group(2) or "" |
| 393 | if val_text.startswith("<![CDATA[") and val_text.endswith("]]>"): |
| 394 | val_text = val_text[len("<![CDATA[") : -len("]]>")] |
| 395 | else: |
| 396 | val_text = val_text.strip() |
| 397 | if not _add_argument( |
| 398 | func_name or "", |
| 399 | key, |
| 400 | val_text, |
| 401 | arguments, |
| 402 | seen_keys, |
| 403 | allowed_props, |
| 404 | name_to_tool, |
| 405 | ): |
| 406 | has_invalid_param = True |
| 407 | break |
| 408 | if has_invalid_param: |
| 409 | arguments.clear() |
| 410 | param_invalid = True |
| 411 | parsed_ok = bool(func_name) |
| 412 | except Exception: |
| 413 | parsed_ok = False |
| 414 | |
| 415 | if not func_name or param_invalid: |
| 416 | return None |
| 417 | |
| 418 | func_name, arguments = _normalize_alias_tool_call(func_name, arguments, tool_names) |
| 419 | |
| 420 | if func_name not in tool_names: |
| 421 | return None |
| 422 | |
| 423 | req_props = name_to_required.get(func_name, set()) |
| 424 | if req_props and not req_props.issubset(arguments.keys()): |
| 425 | return None |
| 426 | |
| 427 | if not parsed_ok: |
| 428 | return None |
| 429 | |
| 430 | return {"name": func_name, "parameters": arguments} |
| 431 | |
| 432 | |
| 433 | def _parse_partial_params( |
| 434 | block: str, |
| 435 | func_name: str, |
| 436 | name_to_allowed_props: dict[str, set[str]], |
| 437 | name_to_tool: dict[str, ChatCompletionToolsParam], |
| 438 | ) -> dict[str, Any]: |
| 439 | arguments: dict[str, Any] = {} |
| 440 | seen_keys: set[str] = set() |
| 441 | allowed_props = name_to_allowed_props.get(func_name, set()) |
| 442 | for pm in _PARAM_WITH_NAME_REGEX.finditer(block): |
| 443 | key = pm.group(1).strip() |
| 444 | if not key or key in seen_keys: |
| 445 | continue |
| 446 | val_text = pm.group(2) or "" |
| 447 | if val_text.startswith("<![CDATA[") and val_text.endswith("]]>"): |
| 448 | val_text = val_text[len("<![CDATA[") : -len("]]>")] |
| 449 | else: |
| 450 | val_text = val_text.strip() |
| 451 | _add_argument( |
| 452 | func_name, |
| 453 | key, |
| 454 | val_text, |
| 455 | arguments, |
| 456 | seen_keys, |
| 457 | allowed_props, |
| 458 | name_to_tool, |
| 459 | ) |
| 460 | return arguments |
| 461 | |
| 462 | |
| 463 | class MiniCPM5XMLToolParser(ToolParser): |
| 464 | """MiniCPM5 XML tool parser.""" |
| 465 | |
| 466 | def __init__( |
| 467 | self, |
| 468 | tokenizer: TokenizerLike, |
| 469 | tools: list[Tool] | None = None, |
| 470 | ): |
| 471 | super().__init__(tokenizer, tools) |
| 472 | self.tool_call_start_token = "<function" |
| 473 | self.tool_call_end_token = "</function>" |
| 474 | self._processed_len = 0 |
| 475 | |
| 476 | def _reset_stream_state(self) -> None: |
| 477 | self._processed_len = 0 |
| 478 | self.current_tool_id = -1 |
| 479 | self.current_tool_name_sent = False |
| 480 | self.prev_tool_call_arr = [] |
| 481 | self.streamed_args_for_tool = [] |
| 482 | |
| 483 | def adjust_request( |
| 484 | self, request: ChatCompletionRequest | ResponsesRequest |
| 485 | ) -> ChatCompletionRequest | ResponsesRequest: |
| 486 | request = super().adjust_request(request) |
| 487 | if request.tools and request.tool_choice != "none": |
| 488 | # Tool XML tags are special tokens in MiniCPM5; must not strip them |
| 489 | # before tool parsing (see internlm2/mistral vLLM tool parsers). |
| 490 | request.skip_special_tokens = False |
| 491 | return request |
| 492 | |
| 493 | def extract_tool_calls( |
| 494 | self, |
| 495 | model_output: str, |
| 496 | request: ChatCompletionRequest, |
| 497 | ) -> ExtractedToolCallInformation: |
| 498 | model_output = _normalize_model_output(model_output) |
| 499 | if self.tool_call_start_token not in model_output: |
| 500 | logger.debug("[MiniCPM5XMLToolParser] no <function token in output") |
| 501 | return ExtractedToolCallInformation( |
| 502 | tools_called=False, |
| 503 | tool_calls=[], |
| 504 | content=_strip_thinking_content(model_output), |
| 505 | ) |
| 506 | |
| 507 | tool_names, name_to_allowed_props, name_to_required, name_to_tool = ( |
| 508 | _build_tool_maps(request.tools) |
| 509 | ) |
| 510 | |
| 511 | tool_calls: list[ToolCall] = [] |
| 512 | normal_parts: list[str] = [] |
| 513 | last_end = 0 |
| 514 | |
| 515 | try: |
| 516 | for match in _FUNC_BLOCK_REGEX.finditer(model_output): |
| 517 | if match.start() > last_end: |
| 518 | normal_parts.append(model_output[last_end : match.start()]) |
| 519 | |
| 520 | block = match.group(0) |
| 521 | parsed = _parse_function_block( |
| 522 | block, |
| 523 | tool_names, |
| 524 | name_to_allowed_props, |
| 525 | name_to_required, |
| 526 | name_to_tool, |
| 527 | ) |
| 528 | if parsed is not None: |
| 529 | tool_calls.append( |
| 530 | ToolCall( |
| 531 | id=f"call_{random_uuid()}", |
| 532 | type="function", |
| 533 | function=FunctionCall( |
| 534 | name=parsed["name"], |
| 535 | arguments=json.dumps( |
| 536 | parsed["parameters"], |
| 537 | ensure_ascii=False, |
| 538 | ), |
| 539 | ), |
| 540 | ) |
| 541 | ) |
| 542 | else: |
| 543 | normal_parts.append(block) |
| 544 | |
| 545 | last_end = match.end() |
| 546 | |
| 547 | if last_end < len(model_output): |
| 548 | normal_parts.append(model_output[last_end:]) |
| 549 | |
| 550 | content = _strip_thinking_content("".join(normal_parts).strip()) |
| 551 | |
| 552 | logger.debug( |
| 553 | "[MiniCPM5XMLToolParser] extracted %d tool calls", |
| 554 | len(tool_calls), |
| 555 | ) |
| 556 | |
| 557 | tools_called = len(tool_calls) > 0 |
| 558 | return ExtractedToolCallInformation( |
| 559 | tools_called=tools_called, |
| 560 | tool_calls=tool_calls, |
| 561 | content=None if tools_called else content, |
| 562 | ) |
| 563 | except Exception as e: |
| 564 | logger.error("Error in MiniCPM5XMLToolParser.extract_tool_calls: %s", e) |
| 565 | return ExtractedToolCallInformation( |
| 566 | tools_called=False, |
| 567 | tool_calls=[], |
| 568 | content=model_output, |
| 569 | ) |
| 570 | |
| 571 | def _emit_tool_args_delta( |
| 572 | self, |
| 573 | tool_index: int, |
| 574 | args_json: str, |
| 575 | *, |
| 576 | is_complete: bool = False, |
| 577 | ) -> DeltaMessage | None: |
| 578 | prev_args = ( |
| 579 | self.streamed_args_for_tool[tool_index] |
| 580 | if tool_index < len(self.streamed_args_for_tool) |
| 581 | else "" |
| 582 | ) |
| 583 | arg_diff = _streaming_args_diff( |
| 584 | prev_args, |
| 585 | args_json, |
| 586 | is_complete=is_complete, |
| 587 | ) |
| 588 | if not arg_diff: |
| 589 | return None |
| 590 | while len(self.streamed_args_for_tool) <= tool_index: |
| 591 | self.streamed_args_for_tool.append("") |
| 592 | self.streamed_args_for_tool[tool_index] = prev_args + arg_diff |
| 593 | return DeltaMessage( |
| 594 | tool_calls=[ |
| 595 | DeltaToolCall( |
| 596 | index=tool_index, |
| 597 | function=DeltaFunctionCall( |
| 598 | arguments=arg_diff, |
| 599 | ).model_dump(exclude_none=True), |
| 600 | ) |
| 601 | ], |
| 602 | ) |
| 603 | |
| 604 | def _start_tool_call(self, func_name: str) -> DeltaMessage: |
| 605 | self.current_tool_id += 1 |
| 606 | self.current_tool_name_sent = True |
| 607 | while len(self.streamed_args_for_tool) <= self.current_tool_id: |
| 608 | self.streamed_args_for_tool.append("") |
| 609 | while len(self.prev_tool_call_arr) <= self.current_tool_id: |
| 610 | self.prev_tool_call_arr.append({}) |
| 611 | self.prev_tool_call_arr[self.current_tool_id] = {"name": func_name} |
| 612 | return DeltaMessage( |
| 613 | tool_calls=[ |
| 614 | DeltaToolCall( |
| 615 | index=self.current_tool_id, |
| 616 | id=make_tool_call_id(), |
| 617 | type="function", |
| 618 | function=DeltaFunctionCall( |
| 619 | name=func_name, |
| 620 | ).model_dump(exclude_none=True), |
| 621 | ) |
| 622 | ], |
| 623 | ) |
| 624 | |
| 625 | def _process_complete_block_streaming( |
| 626 | self, |
| 627 | block: str, |
| 628 | request: ChatCompletionRequest, |
| 629 | ) -> DeltaMessage | None: |
| 630 | tool_names, name_to_allowed_props, name_to_required, name_to_tool = ( |
| 631 | _build_tool_maps(request.tools) |
| 632 | ) |
| 633 | parsed = _parse_function_block( |
| 634 | block, |
| 635 | tool_names, |
| 636 | name_to_allowed_props, |
| 637 | name_to_required, |
| 638 | name_to_tool, |
| 639 | ) |
| 640 | if parsed is None: |
| 641 | return DeltaMessage(content=block) |
| 642 | |
| 643 | args_json = json.dumps(parsed["parameters"], ensure_ascii=False) |
| 644 | func_name = parsed["name"] |
| 645 | |
| 646 | if not self.current_tool_name_sent: |
| 647 | self.current_tool_id += 1 |
| 648 | tool_index = self.current_tool_id |
| 649 | while len(self.streamed_args_for_tool) <= tool_index: |
| 650 | self.streamed_args_for_tool.append("") |
| 651 | while len(self.prev_tool_call_arr) <= tool_index: |
| 652 | self.prev_tool_call_arr.append({}) |
| 653 | self.streamed_args_for_tool[tool_index] = args_json |
| 654 | self.prev_tool_call_arr[tool_index] = { |
| 655 | "name": func_name, |
| 656 | "arguments": parsed["parameters"], |
| 657 | } |
| 658 | if not self.prev_tool_call_arr: |
| 659 | self.prev_tool_call_arr = [{"arguments": {}}] |
| 660 | return DeltaMessage( |
| 661 | tool_calls=[ |
| 662 | DeltaToolCall( |
| 663 | index=tool_index, |
| 664 | id=make_tool_call_id(), |
| 665 | type="function", |
| 666 | function=DeltaFunctionCall( |
| 667 | name=func_name, |
| 668 | arguments=args_json, |
| 669 | ).model_dump(exclude_none=True), |
| 670 | ) |
| 671 | ], |
| 672 | ) |
| 673 | |
| 674 | tool_index = self.current_tool_id |
| 675 | self.prev_tool_call_arr[tool_index]["arguments"] = parsed["parameters"] |
| 676 | delta = self._emit_tool_args_delta( |
| 677 | tool_index, |
| 678 | args_json, |
| 679 | is_complete=True, |
| 680 | ) |
| 681 | self.current_tool_name_sent = False |
| 682 | if delta: |
| 683 | if not self.prev_tool_call_arr: |
| 684 | self.prev_tool_call_arr = [{"arguments": {}}] |
| 685 | return delta |
| 686 | return DeltaMessage(content="") |
| 687 | |
| 688 | def _process_partial_block_streaming( |
| 689 | self, |
| 690 | block: str, |
| 691 | request: ChatCompletionRequest, |
| 692 | ) -> DeltaMessage | None: |
| 693 | tool_names, name_to_allowed_props, _, name_to_tool = _build_tool_maps( |
| 694 | request.tools |
| 695 | ) |
| 696 | if _PARAM_MISSING_NAME_REGEX.search(block): |
| 697 | return None |
| 698 | |
| 699 | match = _FUNC_NAME_V1_REGEX.search(block) |
| 700 | if not match: |
| 701 | return None |
| 702 | func_name = (match.group(1) or "").strip() |
| 703 | if func_name not in tool_names: |
| 704 | return None |
| 705 | |
| 706 | if not self.current_tool_name_sent: |
| 707 | return self._start_tool_call(func_name) |
| 708 | |
| 709 | arguments = _parse_partial_params( |
| 710 | block, |
| 711 | func_name, |
| 712 | name_to_allowed_props, |
| 713 | name_to_tool, |
| 714 | ) |
| 715 | if not arguments: |
| 716 | return None |
| 717 | args_json = json.dumps(arguments, ensure_ascii=False) |
| 718 | self.prev_tool_call_arr[self.current_tool_id]["arguments"] = arguments |
| 719 | delta = self._emit_tool_args_delta( |
| 720 | self.current_tool_id, |
| 721 | args_json, |
| 722 | is_complete=False, |
| 723 | ) |
| 724 | if delta and not self.prev_tool_call_arr: |
| 725 | self.prev_tool_call_arr = [{"arguments": {}}] |
| 726 | return delta |
| 727 | |
| 728 | def extract_tool_calls_streaming( |
| 729 | self, |
| 730 | previous_text: str, |
| 731 | current_text: str, |
| 732 | delta_text: str, |
| 733 | previous_token_ids: Sequence[int], |
| 734 | current_token_ids: Sequence[int], |
| 735 | delta_token_ids: Sequence[int], |
| 736 | request: ChatCompletionRequest, |
| 737 | ) -> DeltaMessage | None: |
| 738 | del previous_token_ids, current_token_ids, delta_token_ids |
| 739 | try: |
| 740 | current_text = _normalize_model_output(current_text) |
| 741 | if not previous_text: |
| 742 | self._reset_stream_state() |
| 743 | |
| 744 | if self.tool_call_start_token not in current_text: |
| 745 | if self._processed_len < len(current_text): |
| 746 | content = current_text[self._processed_len :] |
| 747 | self._processed_len = len(current_text) |
| 748 | return DeltaMessage(content=content) if content else None |
| 749 | return None |
| 750 | |
| 751 | for match in _FUNC_BLOCK_REGEX.finditer(current_text): |
| 752 | if match.end() <= self._processed_len: |
| 753 | continue |
| 754 | |
| 755 | if match.start() > self._processed_len: |
| 756 | gap = current_text[self._processed_len : match.start()] |
| 757 | self._processed_len = match.start() |
| 758 | return DeltaMessage(content=gap) if gap else None |
| 759 | |
| 760 | block = match.group(0) |
| 761 | delta = self._process_complete_block_streaming(block, request) |
| 762 | self._processed_len = match.end() |
| 763 | if delta is not None: |
| 764 | return delta |
| 765 | |
| 766 | remainder = current_text[self._processed_len :] |
| 767 | if not remainder: |
| 768 | if ( |
| 769 | not delta_text |
| 770 | and self.current_tool_id >= 0 |
| 771 | and not self.current_tool_name_sent |
| 772 | ): |
| 773 | return DeltaMessage(content="") |
| 774 | return None |
| 775 | |
| 776 | func_idx = remainder.find(self.tool_call_start_token) |
| 777 | if func_idx > 0: |
| 778 | gap = remainder[:func_idx] |
| 779 | self._processed_len += func_idx |
| 780 | return DeltaMessage(content=gap) |
| 781 | |
| 782 | if func_idx == -1: |
| 783 | overlap = partial_tag_overlap(remainder, self.tool_call_start_token) |
| 784 | if overlap: |
| 785 | return None |
| 786 | self._processed_len = len(current_text) |
| 787 | return DeltaMessage(content=remainder) if remainder else None |
| 788 | |
| 789 | partial_block = remainder[func_idx:] |
| 790 | if self.tool_call_end_token in partial_block: |
| 791 | end_idx = partial_block.rfind(self.tool_call_end_token) |
| 792 | complete_block = partial_block[ |
| 793 | : end_idx + len(self.tool_call_end_token) |
| 794 | ] |
| 795 | delta = self._process_complete_block_streaming(complete_block, request) |
| 796 | self._processed_len += func_idx + len(complete_block) |
| 797 | if delta is not None: |
| 798 | return delta |
| 799 | partial_block = partial_block[end_idx + len(self.tool_call_end_token) :] |
| 800 | if not partial_block.strip(): |
| 801 | return None |
| 802 | func_idx = partial_block.find(self.tool_call_start_token) |
| 803 | if func_idx == -1: |
| 804 | self._processed_len = len(current_text) |
| 805 | if partial_block: |
| 806 | return DeltaMessage(content=partial_block) |
| 807 | return None |
| 808 | partial_block = partial_block[func_idx:] |
| 809 | |
| 810 | return self._process_partial_block_streaming(partial_block, request) |
| 811 | except Exception: |
| 812 | logger.exception( |
| 813 | "Error in MiniCPM5XMLToolParser.extract_tool_calls_streaming" |
| 814 | ) |
| 815 | return None |