@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 2d7053438a

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 9, 14:45 UTC

fix: preserve native tool call ID in multi-turn tool calling (#32768)

Signed-off-by: wanglinian <wanglinian@stu.pku.edu.cn> Signed-off-by: wangln19 <96399074+wangln19@users.noreply.github.com> Signed-off-by: Roger Wang <hey@rogerw.io> Co-authored-by: Roger Wang <hey@rogerw.io> Co-authored-by: Isotr0py <2037008807@qq.com>

wangln19 committed Jan 27, 2026 at 10:22 UTC 2d7053438a112e2be55cf6d2bde9deb8a169d0a4

7 files changed +183 -68

tests/entrypoints/openai/test_chat_error.py

@@ -42,6 +42,7 @@ class MockModelConfig:

42 tokenizer_revision = None

43 multimodal_config = MultiModalConfig()

44 hf_config = MockHFConfig()

45 + hf_text_config = MockHFConfig()

46 logits_processor_pattern = None

47 logits_processors: list[str] | None = None

48 diff_sampling_param: dict | None = None

tests/entrypoints/openai/test_serving_chat.py

@@ -518,6 +518,7 @@ class MockModelConfig:

518 tokenizer_revision = None

519 multimodal_config = MultiModalConfig()

520 hf_config = MockHFConfig()

521 + hf_text_config = MockHFConfig()

522 logits_processors: list[str] | None = None

523 logits_processor_pattern = None

524 diff_sampling_param: dict | None = None

vllm/entrypoints/openai/chat_completion/serving.py

+121 -39

@@ -44,6 +44,7 @@ from vllm.entrypoints.openai.engine.protocol import (

44 DeltaMessage,

45 DeltaToolCall,

46 ErrorResponse,

47 + FunctionCall,

48 PromptTokenUsageInfo,

49 RequestResponseMetadata,

50 ToolCall,

@@ -143,11 +144,6 @@ class OpenAIServingChat(OpenAIServing):

144 self.enable_prompt_tokens_details = enable_prompt_tokens_details

145 self.enable_force_include_usage = enable_force_include_usage

146 self.default_sampling_params = self.model_config.get_diff_sampling_param()

146 - if self.model_config.hf_config.model_type == "kimi_k2":

147 - self.tool_call_id_type = "kimi_k2"

148 - else:

149 - self.tool_call_id_type = "random"

150 -

147 self.use_harmony = self.model_config.hf_config.model_type == "gpt_oss"

148 if self.use_harmony:

149 if "stop_token_ids" not in self.default_sampling_params:

@@ -156,6 +152,16 @@ class OpenAIServingChat(OpenAIServing):

152 get_stop_tokens_for_assistant_actions()

153 )

154

155 + # Handle tool call ID type for Kimi K2 (supporting test mocking via overrides)

156 + hf_overrides = getattr(self.model_config, "hf_overrides", None)

157 + if self.model_config.hf_text_config.model_type == "kimi_k2" or (

158 + isinstance(hf_overrides, dict)

159 + and hf_overrides.get("model_type") == "kimi_k2"

160 + ):

161 + self.tool_call_id_type = "kimi_k2"

162 + else:

163 + self.tool_call_id_type = "random"

164 +

165 # NOTE(woosuk): While OpenAI's chat completion API supports browsing

166 # for some models, currently vLLM doesn't support it. Please use the

167 # Responses API instead.

@@ -247,8 +253,8 @@ class OpenAIServingChat(OpenAIServing):

253 # because of issues with pydantic we need to potentially

254 # re-serialize the tool_calls field of the request

255 # for more info: see comment in `maybe_serialize_tool_calls`

250 - maybe_serialize_tool_calls(request)

251 - truncate_tool_call_ids(request)

256 + maybe_serialize_tool_calls(request) # type: ignore[arg-type]

257 + truncate_tool_call_ids(request) # type: ignore[arg-type]

258 validate_request_params(request)

259

260 # Check if tool parsing is unavailable (common condition)

@@ -454,6 +460,7 @@ class OpenAIServingChat(OpenAIServing):

460

461 # Streaming response

462 tokenizer = self.renderer.tokenizer

463 + assert tokenizer is not None

464

465 if request.stream:

466 return self.chat_completion_stream_generator(

@@ -632,9 +639,11 @@ class OpenAIServingChat(OpenAIServing):

639 request_id: str,

640 model_name: str,

641 conversation: list[ConversationMessage],

635 - tokenizer: TokenizerLike | None,

642 + tokenizer: TokenizerLike,

643 request_metadata: RequestResponseMetadata,

644 ) -> AsyncGenerator[str, None]:

645 + from vllm.tokenizers.mistral import MistralTokenizer

646 +

647 created_time = int(time.time())

648 chunk_object_type: Final = "chat.completion.chunk"

649 first_iteration = True

@@ -698,7 +707,7 @@ class OpenAIServingChat(OpenAIServing):

707 )

708 reasoning_parser = self.reasoning_parser(

709 tokenizer,

701 - chat_template_kwargs=chat_template_kwargs, # type: ignore[call-arg]

710 + chat_template_kwargs=chat_template_kwargs or {}, # type: ignore[call-arg]

711 )

712 except RuntimeError as e:

713 logger.exception("Error in reasoning parser creation.")

@@ -955,8 +964,17 @@ class OpenAIServingChat(OpenAIServing):

964 index=i,

965 )

966 else:

967 + # Generate ID based on tokenizer type

968 + if isinstance(tokenizer, MistralTokenizer):

969 + tool_call_id = MistralToolCall.generate_random_id()

970 + else:

971 + tool_call_id = make_tool_call_id(

972 + id_type=self.tool_call_id_type,

973 + func_name=tool_choice_function_name,

974 + idx=history_tool_call_cnt,

975 + )

976 delta_tool_call = DeltaToolCall(

959 - id=make_tool_call_id(),

977 + id=tool_call_id,

978 type="function",

979 function=DeltaFunctionCall(

980 name=tool_choice_function_name,

@@ -1387,9 +1405,11 @@ class OpenAIServingChat(OpenAIServing):

1405 request_id: str,

1406 model_name: str,

1407 conversation: list[ConversationMessage],

1390 - tokenizer: TokenizerLike | None,

1408 + tokenizer: TokenizerLike,

1409 request_metadata: RequestResponseMetadata,

1410 ) -> ErrorResponse | ChatCompletionResponse:

1411 + from vllm.tokenizers.mistral import MistralTokenizer

1412 +

1413 created_time = int(time.time())

1414 final_res: RequestOutput | None = None

1415

@@ -1524,39 +1544,85 @@ class OpenAIServingChat(OpenAIServing):

1544 tool_call_class = (

1545 MistralToolCall if isinstance(tokenizer, MistralTokenizer) else ToolCall

1546 )

1527 - if (not self.enable_auto_tools or not self.tool_parser) and (

1547 + if self.use_harmony:

1548 + # Harmony models already have parsed content and tool_calls

1549 + # through parse_chat_output. Respect its output directly.

1550 + message = ChatMessage(

1551 + role=role,

1552 + reasoning=reasoning,

1553 + content=content,

1554 + tool_calls=tool_calls if tool_calls else [],

1555 + )

1556 +

1557 + elif (not self.enable_auto_tools or not self.tool_parser) and (

1558 not isinstance(request.tool_choice, ChatCompletionNamedToolChoiceParam)

1559 and request.tool_choice != "required"

1560 ):

1561 message = ChatMessage(role=role, reasoning=reasoning, content=content)

1562

1533 - # if the request uses tools and specified a tool choice

1563 elif (

1564 request.tool_choice

1565 and type(request.tool_choice) is ChatCompletionNamedToolChoiceParam

1566 ):

1567 assert tool_calls is not None and len(tool_calls) > 0

1568 + tool_call_class_items = []

1569 + for idx, tc in enumerate(tool_calls):

1570 + # Use native ID if available (e.g., Kimi K2),

1571 + # otherwise generate ID with correct id_type

1572 + if tc.id:

1573 + tool_call_class_items.append(

1574 + tool_call_class(id=tc.id, function=tc)

1575 + )

1576 + else:

1577 + # Generate ID using the correct format (kimi_k2 or random),

1578 + # but leave it to the class if it's Mistral to preserve

1579 + # 9-char IDs

1580 + if isinstance(tokenizer, MistralTokenizer):

1581 + tool_call_class_items.append(tool_call_class(function=tc))

1582 + else:

1583 + generated_id = make_tool_call_id(

1584 + id_type=self.tool_call_id_type,

1585 + func_name=tc.name,

1586 + idx=history_tool_call_cnt + idx,

1587 + )

1588 + tool_call_class_items.append(

1589 + tool_call_class(id=generated_id, function=tc)

1590 + )

1591 + history_tool_call_cnt += 1

1592 message = ChatMessage(

1593 role=role,

1594 reasoning=reasoning,

1595 content="",

1543 - tool_calls=[tool_call_class(function=tc) for tc in tool_calls],

1596 + tool_calls=tool_call_class_items,

1597 )

1598

1599 elif request.tool_choice and request.tool_choice == "required":

1600 tool_call_class_items = []

1601 assert tool_calls is not None and len(tool_calls) > 0

1549 - for tool_call in tool_calls:

1550 - tool_call_class_items.append(

1551 - tool_call_class(

1552 - id=make_tool_call_id(

1602 + for idx, tool_call in enumerate(tool_calls):

1603 + # Use native ID if available,

1604 + # otherwise generate ID with correct id_type

1605 + if tool_call.id:

1606 + tool_call_class_items.append(

1607 + tool_call_class(id=tool_call.id, function=tool_call)

1608 + )

1609 + else:

1610 + # Generate ID using the correct format (kimi_k2 or random),

1611 + # but leave it to the class if it's Mistral to preserve

1612 + # 9-char IDs

1613 + if isinstance(tokenizer, MistralTokenizer):

1614 + tool_call_class_items.append(

1615 + tool_call_class(function=tool_call)

1616 + )

1617 + else:

1618 + generated_id = make_tool_call_id(

1619 id_type=self.tool_call_id_type,

1620 func_name=tool_call.name,

1555 - idx=history_tool_call_cnt,

1556 - ),

1557 - function=tool_call,

1558 - )

1559 - )

1621 + idx=history_tool_call_cnt + idx,

1622 + )

1623 + tool_call_class_items.append(

1624 + tool_call_class(id=generated_id, function=tool_call)

1625 + )

1626 history_tool_call_cnt += 1

1627 message = ChatMessage(

1628 role=role,

@@ -1582,17 +1648,35 @@ class OpenAIServingChat(OpenAIServing):

1648 # call. The same is not true for named function calls

1649 auto_tools_called = tool_calls is not None and len(tool_calls) > 0

1650 if tool_calls:

1651 + tool_call_items = []

1652 + for idx, tc in enumerate(tool_calls):

1653 + # Use native ID if available (e.g., Kimi K2),

1654 + # otherwise generate ID with correct id_type

1655 + if tc.id:

1656 + tool_call_items.append(

1657 + tool_call_class(id=tc.id, function=tc)

1658 + )

1659 + else:

1660 + # Generate ID using the correct format (kimi_k2 or random),

1661 + # but leave it to the class if it's Mistral to preserve

1662 + # 9-char IDs

1663 + if isinstance(tokenizer, MistralTokenizer):

1664 + tool_call_items.append(tool_call_class(function=tc))

1665 + else:

1666 + generated_id = make_tool_call_id(

1667 + id_type=self.tool_call_id_type,

1668 + func_name=tc.name,

1669 + idx=history_tool_call_cnt + idx,

1670 + )

1671 + tool_call_items.append(

1672 + tool_call_class(id=generated_id, function=tc)

1673 + )

1674 + history_tool_call_cnt += 1

1675 message = ChatMessage(

1676 role=role,

1677 reasoning=reasoning,

1678 content=content,

1589 - tool_calls=[

1590 - ToolCall(

1591 - function=tc,

1592 - type="function",

1593 - )

1594 - for tc in tool_calls

1595 - ],

1679 + tool_calls=tool_call_items,

1680 )

1681

1682 else:

@@ -1701,13 +1785,11 @@ class OpenAIServingChat(OpenAIServing):

1785 elif choice.message.tool_calls:

1786 # For tool calls, log the function name and arguments

1787 tool_call_descriptions = []

1704 - for tc in choice.message.tool_calls:

1705 - if hasattr(tc.function, "name") and hasattr(

1706 - tc.function, "arguments"

1707 - ):

1708 - tool_call_descriptions.append(

1709 - f"{tc.function.name}({tc.function.arguments})"

1710 - )

1788 + for tc in choice.message.tool_calls: # type: ignore

1789 + function_call: FunctionCall = tc.function # type: ignore

1790 + tool_call_descriptions.append(

1791 + f"{function_call.name}({function_call.arguments})"

1792 + )

1793 tool_calls_str = ", ".join(tool_call_descriptions)

1794 output_text = f"[tool_calls: {tool_calls_str}]"

1795

@@ -1895,7 +1977,7 @@ class OpenAIServingChat(OpenAIServing):

1977 # because of issues with pydantic we need to potentially

1978 # re-serialize the tool_calls field of the request

1979 # for more info: see comment in `maybe_serialize_tool_calls`

1898 - maybe_serialize_tool_calls(request)

1980 + maybe_serialize_tool_calls(request) # type: ignore[arg-type]

1981

1982 # Add system message.

1983 # NOTE: In Chat Completion API, browsing is enabled by default

@@ -1913,7 +1995,7 @@ class OpenAIServingChat(OpenAIServing):

1995 # Add developer message.

1996 if request.tools:

1997 dev_msg = get_developer_message(

1916 - tools=request.tools if should_include_tools else None

1998 + tools=request.tools if should_include_tools else None # type: ignore[arg-type]

1999 )

2000 messages.append(dev_msg)

2001

vllm/entrypoints/openai/engine/protocol.py

@@ -218,6 +218,10 @@ def get_logits_processors(

218

219

220 class FunctionCall(OpenAIBaseModel):

221 + # Internal field to preserve native tool call ID from tool parser.

222 + # Excluded from serialization to maintain OpenAI API compatibility

223 + # (function object should only contain 'name' and 'arguments').

224 + id: str | None = Field(default=None, exclude=True)

225 name: str

226 arguments: str

227

vllm/entrypoints/openai/engine/serving.py

@@ -1525,6 +1525,7 @@ class OpenAIServing:

1525 # extract_tool_calls() returns a list of tool calls.

1526 function_calls.extend(

1527 FunctionCall(

1528 + id=tool_call.id,

1529 name=tool_call.function.name,

1530 arguments=tool_call.function.arguments,

1531 )

vllm/entrypoints/openai/responses/serving.py

+53 -27

@@ -63,6 +63,7 @@ from vllm.engine.protocol import EngineClient

63 from vllm.entrypoints.chat_utils import (

64 ChatCompletionMessageParam,

65 ChatTemplateContentFormatOption,

66 + make_tool_call_id,

67 )

68 from vllm.entrypoints.logger import RequestLogger

69 from vllm.entrypoints.mcp.tool_server import ToolServer

@@ -250,6 +251,17 @@ class OpenAIServingResponses(OpenAIServing):

251 self.default_sampling_params["stop_token_ids"].extend(

252 get_stop_tokens_for_assistant_actions()

253 )

254 +

255 + # Handle tool call ID type for Kimi K2 (supporting test mocking via overrides)

256 + hf_overrides = getattr(self.model_config, "hf_overrides", None)

257 + if self.model_config.hf_text_config.model_type == "kimi_k2" or (

258 + isinstance(hf_overrides, dict)

259 + and hf_overrides.get("model_type") == "kimi_k2"

260 + ):

261 + self.tool_call_id_type = "kimi_k2"

262 + else:

263 + self.tool_call_id_type = "random"

264 +

265 self.enable_auto_tools = enable_auto_tools

266 # set up tool use

267 self.tool_parser = self._get_tool_parser(

@@ -954,25 +966,28 @@ class OpenAIServingResponses(OpenAIServing):

966 enable_auto_tools=self.enable_auto_tools,

967 tool_parser_cls=self.tool_parser,

968 )

957 - if content:

958 - output_text = ResponseOutputText(

959 - text=content,

960 - annotations=[], # TODO

961 - type="output_text",

962 - logprobs=(

963 - self._create_response_logprobs(

964 - token_ids=final_output.token_ids,

965 - logprobs=final_output.logprobs,

966 - tokenizer=tokenizer,

967 - top_logprobs=request.top_logprobs,

968 - )

969 - if request.is_include_output_logprobs()

970 - else None

971 - ),

972 - )

969 +

970 + if content or (self.use_harmony and tool_calls):

971 + res_text_part = None

972 + if content:

973 + res_text_part = ResponseOutputText(

974 + text=content,

975 + annotations=[], # TODO

976 + type="output_text",

977 + logprobs=(

978 + self._create_response_logprobs(

979 + token_ids=final_output.token_ids,

980 + logprobs=final_output.logprobs,

981 + tokenizer=tokenizer,

982 + top_logprobs=request.top_logprobs,

983 + )

984 + if request.is_include_output_logprobs()

985 + else None

986 + ),

987 + )

988 message_item = ResponseOutputMessage(

989 id=f"msg_{random_uuid()}",

975 - content=[output_text],

990 + content=[res_text_part] if res_text_part else [],

991 role="assistant",

992 status="completed",

993 type="message",

@@ -984,17 +999,28 @@ class OpenAIServingResponses(OpenAIServing):

999 if message_item:

1000 outputs.append(message_item)

1001 if tool_calls:

987 - tool_call_items = [

988 - ResponseFunctionToolCall(

989 - id=f"fc_{random_uuid()}",

990 - call_id=f"call_{random_uuid()}",

991 - type="function_call",

992 - status="completed",

993 - name=tool_call.name,

994 - arguments=tool_call.arguments,

1002 + # We use a simple counter for history_tool_call_count because

1003 + # we don't track the history of tool calls in the Responses API yet.

1004 + # This means that the tool call index will start from 0 for each

1005 + # request.

1006 + tool_call_items = []

1007 + for history_tool_call_cnt, tool_call in enumerate(tool_calls):

1008 + tool_call_items.append(

1009 + ResponseFunctionToolCall(

1010 + id=f"fc_{random_uuid()}",

1011 + call_id=tool_call.id

1012 + if tool_call.id

1013 + else make_tool_call_id(

1014 + id_type=self.tool_call_id_type,

1015 + func_name=tool_call.name,

1016 + idx=history_tool_call_cnt,

1017 + ),

1018 + type="function_call",

1019 + status="completed",

1020 + name=tool_call.name,

1021 + arguments=tool_call.arguments,

1022 + )

1023 )

996 - for tool_call in tool_calls

997 - ]

1024 outputs.extend(tool_call_items)

1025 return outputs

1026

vllm/tool_parsers/kimi_k2_tool_parser.py

+2 -2

@@ -448,7 +448,7 @@ class KimiK2ToolParser(ToolParser):

448 if current_tool_call_matches:

449 tool_id, tool_args = current_tool_call_matches.groups()

450 tool_name = tool_id.split(":")[0].split(".")[-1]

451 - current_tool_call["id"] = tool_id

451 + current_tool_call["id"] = tool_id.strip()

452 current_tool_call["name"] = tool_name

453 current_tool_call["arguments"] = tool_args

454 else:

@@ -458,7 +458,7 @@ class KimiK2ToolParser(ToolParser):

458 if current_tool_call_name_matches:

459 (tool_id_str,) = current_tool_call_name_matches.groups()

460 tool_name = tool_id_str.split(":")[0].split(".")[-1]

461 - current_tool_call["id"] = tool_id_str

461 + current_tool_call["id"] = tool_id_str.strip()

462 current_tool_call["name"] = tool_name

463 current_tool_call["arguments"] = ""

464 else: