@tduncan / Nvidia-Optimization-Stack-VLLM / commits / 84c2f9f0fb

[Frontend] Fix Kimi K2 tool call IDs for required tool choice (#46344)

Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com>

Chauncey committed Jun 25, 2026 at 03:59 UTC 84c2f9f0fb3f12b79e68d20b62ff7ec154dc860e
11 files changed +406 -178
tests/entrypoints/openai/chat_completion/test_completion_with_function_calling.py
+1 -74
@@ -24,6 +24,7 @@ tools = [
24 "description": "Get the current weather in a given location",
25 "parameters": {
26 "type": "object",
27 + "strict": True,
28 "properties": {
29 "city": {
30 "type": "string",
@@ -215,80 +216,6 @@ async def test_function_tool_use(
216 assert len(reasoning) > 0
217
218
218 -@pytest.fixture(scope="module")
219 -def k2_server():
220 - args = [
221 - # use half precision for speed and memory savings in CI environment
222 - "--dtype",
223 - "half",
224 - "--enable-auto-tool-choice",
225 - "--structured-outputs-config.backend",
226 - "xgrammar",
227 - "--tool-call-parser",
228 - "hermes",
229 - "--reasoning-parser",
230 - "qwen3",
231 - "--gpu-memory-utilization",
232 - "0.4",
233 - ] + ROCM_EXTRA_ARGS
234 - # Test kimi_k2 tool use tool_id format by overriding model_type.
235 - # is_deepseek_mla safely returns False via getattr when kv_lora_rank
236 - # is absent from the underlying config.
237 - with RemoteOpenAIServer(
238 - MODEL_NAME,
239 - args,
240 - env_dict=ROCM_ENV_OVERRIDES,
241 - override_hf_configs={"model_type": "kimi_k2"},
242 - ) as remote_server:
243 - yield remote_server
244 -
245 -
246 -@pytest_asyncio.fixture
247 -async def k2_client(k2_server):
248 - async with k2_server.get_async_client() as async_client:
249 - yield async_client
250 -
251 -
252 -@pytest.mark.asyncio
253 -@pytest.mark.skip(reason="Skipping Kimi K2 tool ID test")
254 -@pytest.mark.parametrize("model_name", [MODEL_NAME])
255 -@pytest.mark.parametrize("stream", [True, False])
256 -@pytest.mark.parametrize("tool_choice", ["required"])
257 -async def test_tool_id_kimi_k2(
258 - k2_client: openai.AsyncOpenAI, model_name: str, stream: bool, tool_choice: str
259 -):
260 - if not stream:
261 - # Non-streaming test
262 - chat_completion = await k2_client.chat.completions.create(
263 - messages=messages, model=model_name, tools=tools, tool_choice=tool_choice
264 - )
265 - assert chat_completion.choices[0].message.tool_calls is not None
266 - assert len(chat_completion.choices[0].message.tool_calls) > 0
267 - assert chat_completion.choices[0].message.tool_calls[0].id in [
268 - "functions.get_current_weather:0",
269 - "functions.get_forecast:1",
270 - ]
271 - else:
272 - # Streaming test
273 - output_stream = await k2_client.chat.completions.create(
274 - messages=messages,
275 - model=model_name,
276 - tools=tools,
277 - tool_choice=tool_choice,
278 - stream=True,
279 - )
280 -
281 - output = []
282 - async for chunk in output_stream:
283 - if chunk.choices and chunk.choices[0].delta.tool_calls:
284 - output.extend(chunk.choices[0].delta.tool_calls)
285 - for o in output:
286 - assert o.id is None or o.id in [
287 - "functions.get_current_weather:0",
288 - "functions.get_forecast:1",
289 - ]
290 -
291 -
219 @pytest.mark.asyncio
220 @pytest.mark.parametrize("model_name", [MODEL_NAME])
221 @pytest.mark.parametrize("arguments", ["{}", ""])
tests/entrypoints/openai/responses/test_parsable_context_unit.py
+9 -1
@@ -183,11 +183,19 @@ def _make_request_output(
183
184
185 def _make_context(parser_cls, **overrides):
186 + # ParsableContext no longer lazily builds a parser from ``parser_cls``;
187 + # the caller (here, the serving layer in production) must supply one.
188 + request = overrides.get("request", _make_request())
189 + response_parser = overrides.pop("response_parser", None)
190 + if response_parser is None and parser_cls is not None:
191 + response_parser = parser_cls(MagicMock(), request.tools)
192 +
193 defaults = dict(
194 tokenizer=MagicMock(),
195 parser_cls=parser_cls,
196 + response_parser=response_parser,
197 response_messages=[],
190 - request=_make_request(),
198 + request=request,
199 available_tools=None,
200 chat_template=None,
201 chat_template_content_format="auto",
tests/parser/test_parse.py
+149 -2
@@ -3,6 +3,7 @@
3
4 import json
5 import os
6 +from types import SimpleNamespace
7
8 import pytest
9
@@ -13,7 +14,9 @@ os.environ[_STRICT_TOOL_CALLING_ENV] = "0"
14 from vllm.entrypoints.openai.chat_completion.protocol import ( # noqa: E402
15 ChatCompletionRequest,
16 )
17 +from vllm.entrypoints.openai.responses.protocol import ResponsesRequest # noqa: E402
18 from vllm.parser.abstract_parser import DelegatingParser # noqa: E402
19 +from vllm.parser.utils import count_history_tool_calls # noqa: E402
20 from vllm.reasoning.basic_parsers import ( # noqa: E402
21 BaseThinkingReasoningParser,
22 )
@@ -82,12 +85,55 @@ TOOLS = [
85 ]
86
87
85 -def make_parser(tokenizer, reasoning=False, tool=False):
88 +KIMI_K2_MODEL_CONFIG = SimpleNamespace(
89 + hf_text_config=SimpleNamespace(model_type="kimi_k2"),
90 + hf_overrides=None,
91 +)
92 +
93 +HISTORY_MESSAGES = [
94 + {"role": "user", "content": "first"},
95 + {
96 + "role": "assistant",
97 + "content": None,
98 + "tool_calls": [
99 + {
100 + "id": "functions.get_current_weather:0",
101 + "type": "function",
102 + "function": {
103 + "name": "get_current_weather",
104 + "arguments": "{}",
105 + },
106 + },
107 + {
108 + "id": "functions.get_forecast:1",
109 + "type": "function",
110 + "function": {
111 + "name": "get_forecast",
112 + "arguments": "{}",
113 + },
114 + },
115 + ],
116 + },
117 + {
118 + "role": "tool",
119 + "tool_call_id": "functions.get_current_weather:0",
120 + "content": "{}",
121 + },
122 + {
123 + "role": "tool",
124 + "tool_call_id": "functions.get_forecast:1",
125 + "content": "{}",
126 + },
127 + {"role": "user", "content": "again"},
128 +]
129 +
130 +
131 +def make_parser(tokenizer, reasoning=False, tool=False, **kwargs):
132 class TestParser(DelegatingParser):
133 reasoning_parser_cls = ThinkReasoningParser if reasoning else None
134 tool_parser_cls = Hermes2ProToolParser if tool else None
135
90 - return TestParser(tokenizer)
136 + return TestParser(tokenizer, **kwargs)
137
138
139 @pytest.mark.parametrize(
@@ -232,6 +278,107 @@ def test_parse_required_tool_choice(tokenizer):
278 assert json.loads(tool_calls[1].arguments) == {"timezone": "UTC"}
279
280
281 +def test_parse_required_tool_choice_kimi_k2_ids(tokenizer):
282 + parser = make_parser(
283 + tokenizer, reasoning=False, tool=True, model_config=KIMI_K2_MODEL_CONFIG
284 + )
285 + functions_json = json.dumps(
286 + [
287 + {"name": "get_current_weather", "parameters": {"city": "Dallas"}},
288 + {"name": "get_forecast", "parameters": {"city": "Dallas", "days": 2}},
289 + ]
290 + )
291 + request = make_request(tools=TOOLS, tool_choice="required")
292 + _, content, tool_calls = parser.parse(
293 + functions_json, request, enable_auto_tools=True
294 + )
295 +
296 + assert content is None
297 + assert tool_calls is not None
298 + assert [tc.id for tc in tool_calls] == [
299 + "functions.get_current_weather:0",
300 + "functions.get_forecast:1",
301 + ]
302 +
303 +
304 +def test_parse_required_tool_choice_kimi_k2_ids_after_history(tokenizer):
305 + parser = make_parser(
306 + tokenizer, reasoning=False, tool=True, model_config=KIMI_K2_MODEL_CONFIG
307 + )
308 + functions_json = json.dumps(
309 + [{"name": "get_current_weather", "parameters": {"city": "Dallas"}}]
310 + )
311 + request = make_request(
312 + messages=HISTORY_MESSAGES,
313 + tools=TOOLS,
314 + tool_choice="required",
315 + )
316 + _, _, tool_calls = parser.parse(functions_json, request, enable_auto_tools=True)
317 +
318 + assert tool_calls is not None
319 + assert tool_calls[0].id == "functions.get_current_weather:2"
320 +
321 +
322 +def test_count_history_tool_calls_responses_request():
323 + request = ResponsesRequest.model_validate(
324 + {
325 + "model": "test-model",
326 + "input": [
327 + {
328 + "type": "function_call",
329 + "call_id": "call_0",
330 + "name": "get_current_weather",
331 + "arguments": "{}",
332 + },
333 + {
334 + "type": "function_call",
335 + "call_id": "call_1",
336 + "name": "get_forecast",
337 + "arguments": "{}",
338 + },
339 + ],
340 + }
341 + )
342 +
343 + assert count_history_tool_calls(request) == 2
344 +
345 +
346 +def test_parse_required_tool_choice_random_ids_deferred(tokenizer):
347 + parser = make_parser(tokenizer, reasoning=False, tool=True)
348 + functions_json = json.dumps(
349 + [{"name": "get_current_weather", "parameters": {"city": "Dallas"}}]
350 + )
351 + request = make_request(
352 + messages=HISTORY_MESSAGES,
353 + tools=TOOLS,
354 + tool_choice="required",
355 + )
356 + _, _, tool_calls = parser.parse(functions_json, request, enable_auto_tools=True)
357 +
358 + assert tool_calls is not None
359 + assert tool_calls[0].id is None
360 +
361 +
362 +def test_parse_named_tool_choice_kimi_k2_id(tokenizer):
363 + parser = make_parser(
364 + tokenizer, reasoning=False, tool=True, model_config=KIMI_K2_MODEL_CONFIG
365 + )
366 + request = make_request(
367 + tools=TOOLS,
368 + tool_choice={
369 + "type": "function",
370 + "function": {"name": "get_weather"},
371 + },
372 + )
373 + _, content, tool_calls = parser.parse(
374 + TOOL_ARGUMENTS, request, enable_auto_tools=True
375 + )
376 +
377 + assert content is None
378 + assert tool_calls is not None
379 + assert tool_calls[0].id == "functions.get_weather:0"
380 +
381 +
382 def test_parse_named_tool_choice_content_none(tokenizer):
383 parser = make_parser(tokenizer, reasoning=False, tool=True)
384 request = make_request(
tests/parser/test_streaming.py
+106 -2
@@ -2,6 +2,7 @@
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4 import json
5 +from types import SimpleNamespace
6
7 import pytest
8
@@ -47,6 +48,36 @@ TOOLS = [
48 ]
49
50
51 +KIMI_K2_MODEL_CONFIG = SimpleNamespace(
52 + hf_text_config=SimpleNamespace(model_type="kimi_k2"),
53 + hf_overrides=None,
54 +)
55 +
56 +HISTORY_MESSAGES = [
57 + {"role": "user", "content": "first"},
58 + {
59 + "role": "assistant",
60 + "content": None,
61 + "tool_calls": [
62 + {
63 + "id": "functions.get_current_weather:0",
64 + "type": "function",
65 + "function": {
66 + "name": "get_current_weather",
67 + "arguments": "{}",
68 + },
69 + }
70 + ],
71 + },
72 + {
73 + "role": "tool",
74 + "tool_call_id": "functions.get_current_weather:0",
75 + "content": "{}",
76 + },
77 + {"role": "user", "content": "again"},
78 +]
79 +
80 +
81 @pytest.fixture
82 def request_obj():
83 return ChatCompletionRequest(
@@ -57,12 +88,12 @@ def request_obj():
88 )
89
90
60 -def make_parser(tokenizer, reasoning=False, tool=False):
91 +def make_parser(tokenizer, reasoning=False, tool=False, **kwargs):
92 class TestParser(DelegatingParser):
93 reasoning_parser_cls = ThinkReasoningParser if reasoning else None
94 tool_parser_cls = Hermes2ProToolParser if tool else None
95
65 - return TestParser(tokenizer)
96 + return TestParser(tokenizer, **kwargs)
97
98
99 def stream_text(parser, tokenizer, text, request, prompt_token_ids=None):
@@ -365,3 +396,76 @@ def test_parse_delta_tool_choice_none_with_reasoning(tokenizer, request_obj):
396 assert len(tool_calls) == 0
397 assert "<tool_call>" in content
398 assert "get_weather" in content
399 +
400 +
401 +def test_parse_delta_required_tool_choice_kimi_k2_ids(tokenizer, request_obj):
402 + parser = make_parser(
403 + tokenizer, reasoning=False, tool=True, model_config=KIMI_K2_MODEL_CONFIG
404 + )
405 + request = request_obj.model_copy(update={"tool_choice": "required"})
406 + output = json.dumps(
407 + [
408 + {
409 + "name": "get_current_weather",
410 + "parameters": {"city": "Dallas"},
411 + }
412 + ]
413 + )
414 +
415 + results: list[DeltaMessage | None] = []
416 + prompt_token_ids: list[int] | None = []
417 + for i in range(0, len(output), 3):
418 + chunk = output[i : i + 3]
419 + results.append(
420 + parser.parse_delta(
421 + chunk,
422 + [],
423 + request,
424 + prompt_token_ids=prompt_token_ids,
425 + finished=False,
426 + )
427 + )
428 + prompt_token_ids = None
429 +
430 + _, content, tool_calls = collect_fields(results)
431 + assert content == ""
432 + assert any(tc.id == "functions.get_current_weather:0" for tc in tool_calls)
433 + assert all(tc.id in (None, "functions.get_current_weather:0") for tc in tool_calls)
434 +
435 +
436 +def test_parse_delta_required_tool_choice_kimi_k2_ids_after_history(
437 + tokenizer, request_obj
438 +):
439 + parser = make_parser(
440 + tokenizer, reasoning=False, tool=True, model_config=KIMI_K2_MODEL_CONFIG
441 + )
442 + request = request_obj.model_copy(
443 + update={"messages": HISTORY_MESSAGES, "tool_choice": "required"}
444 + )
445 + output = json.dumps(
446 + [
447 + {
448 + "name": "get_current_weather",
449 + "parameters": {"city": "Dallas"},
450 + }
451 + ]
452 + )
453 +
454 + results: list[DeltaMessage | None] = []
455 + prompt_token_ids: list[int] | None = []
456 + for i in range(0, len(output), 3):
457 + chunk = output[i : i + 3]
458 + results.append(
459 + parser.parse_delta(
460 + chunk,
461 + [],
462 + request,
463 + prompt_token_ids=prompt_token_ids,
464 + finished=False,
465 + )
466 + )
467 + prompt_token_ids = None
468 +
469 + _, _, tool_calls = collect_fields(results)
470 + assert any(tc.id == "functions.get_current_weather:1" for tc in tool_calls)
471 + assert all(tc.id in (None, "functions.get_current_weather:1") for tc in tool_calls)
vllm/entrypoints/openai/chat_completion/serving.py
+17 -68
@@ -17,8 +17,6 @@ from vllm.engine.protocol import EngineClient
17 from vllm.entrypoints.chat_utils import (
18 ChatTemplateContentFormatOption,
19 ConversationMessage,
20 - get_history_tool_calls_cnt,
21 - get_tool_call_id_type,
20 make_tool_call_id,
21 )
22 from vllm.entrypoints.openai.chat_completion.protocol import (
@@ -170,8 +168,6 @@ class OpenAIServingChat(OpenAIServing):
168 if mc.generation_config not in ("auto", "vllm")
169 else getattr(mc, "override_generation_config", {}).get("max_new_tokens")
170 )
173 - self.tool_call_id_type = get_tool_call_id_type(self.model_config)
174 -
171 # NOTE(woosuk): While OpenAI's chat completion API supports browsing
172 # for some models, currently vLLM doesn't support it. Please use the
173 # Responses API instead.
@@ -261,6 +257,7 @@ class OpenAIServingChat(OpenAIServing):
257 tokenizer,
258 request.tools,
259 chat_template_kwargs=chat_template_kwargs,
260 + model_config=self.model_config,
261 )
262 result = await self.render_chat_request(request)
263 if isinstance(result, ErrorResponse):
@@ -433,11 +430,6 @@ class OpenAIServingChat(OpenAIServing):
430 else:
431 tool_choice_function_name = None
432
436 - if self.tool_call_id_type == "kimi_k2":
437 - history_tool_call_cnt = get_history_tool_calls_cnt(conversation)
438 - else:
439 - history_tool_call_cnt = 0
440 -
433 previous_texts = [""] * num_choices
434
435 try:
@@ -451,14 +443,10 @@ class OpenAIServingChat(OpenAIServing):
443 tokenizer,
444 request.tools,
445 chat_template_kwargs=chat_template_kwargs,
446 + model_config=self.model_config,
447 )
448 for _ in range(num_choices)
449 ]
457 - for p in parsers:
458 - if p is not None:
459 - # NOTE: HarmonyParser ignores _stream_state (uses its own FSM).
460 - p._stream_state.tool_call_id_type = self.tool_call_id_type
461 - p._stream_state.history_tool_call_cnt = history_tool_call_cnt
450 else:
451 parsers = [None] * num_choices
452 except Exception as e:
@@ -842,10 +830,6 @@ class OpenAIServingChat(OpenAIServing):
830 )
831
832 choices: list[ChatCompletionResponseChoice] = []
845 - if self.tool_call_id_type == "kimi_k2":
846 - history_tool_call_cnt = get_history_tool_calls_cnt(conversation)
847 - else:
848 - history_tool_call_cnt = 0
833
834 role = self.get_chat_request_role(request)
835 tool_parser_cls = (
@@ -885,54 +869,26 @@ class OpenAIServingChat(OpenAIServing):
869 tool_calls = []
870
871 auto_tools_called = False
872 + is_named_tool_choice = (
873 + request.tool_choice is not None
874 + and type(request.tool_choice) is ChatCompletionNamedToolChoiceParam
875 + )
876 + is_required_tool_choice = request.tool_choice == "required"
877
878 if (not self.enable_auto_tools or not tool_parser_cls) and (
890 - not isinstance(request.tool_choice, ChatCompletionNamedToolChoiceParam)
891 - and request.tool_choice != "required"
879 + not is_named_tool_choice and not is_required_tool_choice
880 ):
881 message = ChatMessage(role=role, reasoning=reasoning, content=content)
882
895 - elif (
896 - request.tool_choice
897 - and type(request.tool_choice) is ChatCompletionNamedToolChoiceParam
898 - ):
899 - tool_call_items = []
900 - tool_calls = tool_calls or []
901 - for tc in tool_calls:
902 - if not tc.id:
903 - tc.id = make_tool_call_id(
904 - id_type=self.tool_call_id_type,
905 - func_name=tc.name,
906 - idx=history_tool_call_cnt,
907 - )
908 - tool_call_items.append(ToolCall(id=tc.id, function=tc))
909 - history_tool_call_cnt += 1
883 + elif is_named_tool_choice or is_required_tool_choice:
884 message = ChatMessage(
885 role=role,
886 reasoning=reasoning,
887 content=content or "",
914 - tool_calls=tool_call_items,
915 - )
916 -
917 - elif request.tool_choice and request.tool_choice == "required":
918 - tool_call_items = []
919 - tool_calls = tool_calls or []
920 - for tool_call in tool_calls:
921 - if not tool_call.id:
922 - tool_call.id = make_tool_call_id(
923 - id_type=self.tool_call_id_type,
924 - func_name=tool_call.name,
925 - idx=history_tool_call_cnt,
926 - )
927 - tool_call_items.append(
928 - ToolCall(id=tool_call.id, function=tool_call)
929 - )
930 - history_tool_call_cnt += 1
931 - message = ChatMessage(
932 - role=role,
933 - content=content or "",
934 - tool_calls=tool_call_items,
935 - reasoning=reasoning,
888 + tool_calls=[
889 + ToolCall(id=tc.id or make_tool_call_id(), function=tc)
890 + for tc in (tool_calls or [])
891 + ],
892 )
893
894 # if the request doesn't use tool choice
@@ -949,21 +905,14 @@ class OpenAIServingChat(OpenAIServing):
905 ):
906 auto_tools_called = tool_calls is not None and len(tool_calls) > 0
907 if tool_calls:
952 - tool_call_items = []
953 - for tc in tool_calls:
954 - if not tc.id:
955 - tc.id = make_tool_call_id(
956 - id_type=self.tool_call_id_type,
957 - func_name=tc.name,
958 - idx=history_tool_call_cnt,
959 - )
960 - tool_call_items.append(ToolCall(id=tc.id, function=tc))
961 - history_tool_call_cnt += 1
908 message = ChatMessage(
909 role=role,
910 reasoning=reasoning,
911 content=content,
966 - tool_calls=tool_call_items,
912 + tool_calls=[
913 + ToolCall(id=tc.id or make_tool_call_id(), function=tc)
914 + for tc in tool_calls
915 + ],
916 )
917
918 else:
vllm/entrypoints/openai/responses/context.py
-14
@@ -301,7 +301,6 @@ class ParsableContext(ConversationContext):
301 chat_template_content_format: ChatTemplateContentFormatOption,
302 response_parser: Parser | None = None,
303 enable_auto_tools: bool = False,
304 - tool_call_id_type: str = "random",
304 ):
305 self.num_prompt_tokens = 0
306 self.num_output_tokens = 0
@@ -314,20 +313,8 @@ class ParsableContext(ConversationContext):
313 self.num_init_messages = len(response_messages)
314 self.finish_reason: str | None = None
315 self.enable_auto_tools = enable_auto_tools
317 - self.tool_call_id_type = tool_call_id_type
316
317 self.response_parser = response_parser
320 - if self.response_parser is None and parser_cls is not None:
321 - chat_template_kwargs = request.build_chat_params(
322 - default_template=chat_template,
323 - default_template_content_format=chat_template_content_format,
324 - ).chat_template_kwargs
325 - self.response_parser = parser_cls(
326 - tokenizer,
327 - tools=request.tools,
328 - chat_template_kwargs=chat_template_kwargs,
329 - )
330 -
318 self.parser_cls = parser_cls
319 self.request = request
320
@@ -365,7 +352,6 @@ class ParsableContext(ConversationContext):
352 reasoning=reasoning,
353 content=content,
354 tool_calls=tool_calls,
368 - tool_call_id_type=self.tool_call_id_type,
355 )
356 )
357 elif completion.text:
vllm/entrypoints/openai/responses/serving.py
+1 -6
@@ -30,7 +30,6 @@ from vllm.engine.protocol import EngineClient
30 from vllm.entrypoints.chat_utils import (
31 ChatCompletionMessageParam,
32 ChatTemplateContentFormatOption,
33 - get_tool_call_id_type,
33 )
34 from vllm.entrypoints.mcp.tool_server import ToolServer
35 from vllm.entrypoints.openai.engine.protocol import (
@@ -222,9 +221,6 @@ class OpenAIServingResponses(OpenAIServing):
221 "For gpt-oss, we ignore --enable-auto-tool-choice "
222 "and always enable tool use."
223 )
225 -
226 - self.tool_call_id_type = get_tool_call_id_type(self.model_config)
227 -
224 self.enable_auto_tools = enable_auto_tools
225 # HACK(woosuk): This is a hack. We should use a better store.
226 # FIXME: If enable_store=True, this may cause a memory leak since we
@@ -272,6 +268,7 @@ class OpenAIServingResponses(OpenAIServing):
268 tokenizer,
269 request.tools,
270 chat_template_kwargs=chat_template_kwargs,
271 + model_config=self.model_config,
272 )
273
274 def _validate_generator_input(
@@ -493,7 +490,6 @@ class OpenAIServingResponses(OpenAIServing):
490 chat_template=self.chat_template,
491 chat_template_content_format=self.chat_template_content_format,
492 enable_auto_tools=self.enable_auto_tools,
496 - tool_call_id_type=self.tool_call_id_type,
493 )
494 else:
495 context = SimpleContext(
@@ -1073,7 +1069,6 @@ class OpenAIServingResponses(OpenAIServing):
1069 content=content,
1070 tool_calls=tool_calls,
1071 logprobs=logprobs,
1076 - tool_call_id_type=self.tool_call_id_type,
1072 )
1073
1074 # Fallback when no parser is configured
vllm/entrypoints/openai/responses/utils.py
+1 -7
@@ -45,7 +45,6 @@ def build_response_output_items(
45 content: str | None,
46 tool_calls: list[FunctionCall] | None,
47 logprobs: list[Logprob] | None = None,
48 - tool_call_id_type: str = "random",
48 ) -> list[ResponseOutputItem]:
49 outputs: list[ResponseOutputItem] = []
50
@@ -86,12 +85,7 @@ def build_response_output_items(
85 ResponseFunctionToolCall(
86 id=f"fc_{random_uuid()}",
87 call_id=tool_call.id
89 - if tool_call.id
90 - else make_tool_call_id(
91 - id_type=tool_call_id_type,
92 - func_name=tool_call.name,
93 - idx=idx,
94 - ),
88 + or make_tool_call_id(func_name=tool_call.name, idx=idx),
89 type="function_call",
90 status="completed",
91 name=tool_call.name,
vllm/parser/abstract_parser.py
+46 -2
@@ -11,6 +11,10 @@ from functools import cached_property
11 from openai.types.responses import ToolChoiceFunction
12 from pydantic import TypeAdapter, ValidationError
13
14 +from vllm.entrypoints.chat_utils import (
15 + get_tool_call_id_type,
16 + make_tool_call_id,
17 +)
18 from vllm.entrypoints.openai.chat_completion.protocol import (
19 ChatCompletionNamedToolChoiceParam,
20 ChatCompletionRequest,
@@ -24,6 +28,7 @@ from vllm.entrypoints.openai.engine.protocol import (
28 from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
29 from vllm.logger import init_logger
30 from vllm.parser.metrics import record_tool_parser_invocation
31 +from vllm.parser.utils import count_history_tool_calls
32 from vllm.reasoning.abs_reasoning_parsers import ReasoningParser
33 from vllm.sampling_params import StructuredOutputsParams
34 from vllm.tokenizers import TokenizerLike
@@ -46,6 +51,7 @@ class StreamState:
51 previous_text: str = ""
52 previous_token_ids: list[int] = field(default_factory=list)
53 history_tool_call_cnt: int = 0
54 + history_tool_call_cnt_initialized: bool = False
55 tool_call_id_type: str = "random"
56 # only used for "required" and "named tool" choices,
57 # tracks whether function name has been fully returned in the stream yet
@@ -108,6 +114,7 @@ class Parser:
114 tokenizer: TokenizerLike,
115 tools: list[Tool] | None = None,
116 *args,
117 + model_config=None,
118 **kwargs,
119 ):
120 self.model_tokenizer = tokenizer
@@ -124,7 +131,14 @@ class Parser:
131 self._reasoning_parser is None
132 or self._reasoning_parser.engine_based_streaming
133 ) and (self._tool_parser is None or self._tool_parser.engine_based_streaming)
127 - self._stream_state = StreamState(engine_based=self._engine_based)
134 + self._stream_state = StreamState(
135 + tool_call_id_type=(
136 + get_tool_call_id_type(model_config)
137 + if model_config is not None
138 + else "random"
139 + ),
140 + engine_based=self._engine_based,
141 + )
142
143 @cached_property
144 def vocab(self) -> dict[str, int]:
@@ -149,6 +163,19 @@ class Parser:
163 def tool_parser(self, parser: ToolParser | None) -> None:
164 self._tool_parser = parser
165
166 + def _initialize_history_tool_call_cnt(
167 + self,
168 + request: ChatCompletionRequest | ResponsesRequest,
169 + ) -> None:
170 + state = self._stream_state
171 + if state.history_tool_call_cnt_initialized:
172 + return
173 + if state.tool_call_id_type != "kimi_k2":
174 + state.history_tool_call_cnt_initialized = True
175 + return
176 + state.history_tool_call_cnt = count_history_tool_calls(request)
177 + state.history_tool_call_cnt_initialized = True
178 +
179 # ========== Reasoning Parser Methods ==========
180
181 @abstractmethod
@@ -375,6 +402,18 @@ class DelegatingParser(Parser):
402 return request.tool_choice.function.name
403 raise ValueError("Invalid tool_choice for function name extraction.")
404
405 + def _make_tool_call_id(self, function_name: str) -> str | None:
406 + state = self._stream_state
407 + if state.tool_call_id_type != "kimi_k2":
408 + return None
409 + tool_call_id = make_tool_call_id(
410 + id_type=state.tool_call_id_type,
411 + func_name=function_name,
412 + idx=state.history_tool_call_cnt,
413 + )
414 + state.history_tool_call_cnt += 1
415 + return tool_call_id
416 +
417 def _extract_tool_calls(
418 self,
419 content: str | None,
@@ -404,9 +443,11 @@ class DelegatingParser(Parser):
443 if is_named_tool_choice and supports_required_and_named:
444 if content is None:
445 return [], None
446 + function_name = self._get_function_name(request)
447 tool_calls.append(
448 FunctionCall(
409 - name=self._get_function_name(request),
449 + id=self._make_tool_call_id(function_name),
450 + name=function_name,
451 arguments=content,
452 )
453 )
@@ -422,6 +463,7 @@ class DelegatingParser(Parser):
463 for tc in parsed_calls:
464 tool_calls.append(
465 FunctionCall(
466 + id=self._make_tool_call_id(tc.name),
467 name=tc.name,
468 arguments=json.dumps(tc.parameters, ensure_ascii=False),
469 )
@@ -733,6 +775,7 @@ class DelegatingParser(Parser):
775 enable_auto_tools: bool = False,
776 model_output_token_ids: Sequence[int] = (),
777 ) -> tuple[str | None, str | None, list[FunctionCall] | None]:
778 + self._initialize_history_tool_call_cnt(request)
779 reasoning, content = self.extract_reasoning(model_output, request)
780 tool_calls, content = self._extract_tool_calls(
781 content=content,
@@ -750,6 +793,7 @@ class DelegatingParser(Parser):
793 *,
794 finished: bool,
795 ) -> DeltaMessage | None:
796 + self._initialize_history_tool_call_cnt(request)
797 state = self._stream_state
798
799 if not state.prompt_reasoning_checked and prompt_token_ids is not None:
vllm/parser/engine/parser_engine.py
+11 -2
@@ -13,7 +13,7 @@ from typing import TYPE_CHECKING
13
14 import regex as re
15
16 -from vllm.entrypoints.chat_utils import make_tool_call_id
16 +from vllm.entrypoints.chat_utils import get_tool_call_id_type, make_tool_call_id
17 from vllm.entrypoints.openai.engine.protocol import (
18 DeltaFunctionCall,
19 DeltaMessage,
@@ -89,11 +89,18 @@ class ParserEngine(Parser):
89 tools: list[Tool] | None = None,
90 *,
91 parser_engine_config: ParserEngineConfig,
92 + model_config=None,
93 **kwargs,
94 ) -> None:
95 self.model_tokenizer = tokenizer
96 self._tools = tools
96 - self._stream_state = StreamState()
97 + self._stream_state = StreamState(
98 + tool_call_id_type=(
99 + get_tool_call_id_type(model_config)
100 + if model_config is not None
101 + else "random"
102 + ),
103 + )
104 self._reasoning_parser = None
105 self._tool_parser = None
106 self.parser_engine_config = parser_engine_config
@@ -419,6 +426,7 @@ class ParserEngine(Parser):
426 *,
427 finished: bool,
428 ) -> DeltaMessage | None:
429 + self._initialize_history_tool_call_cnt(request)
430 if not self._prompt_streaming_prepared and prompt_token_ids is not None:
431 # NOTE: call the hook BEFORE setting the flag, because the hook
432 # may invoke ``_reset`` (e.g. via ``initialize_streaming``) which
@@ -658,6 +666,7 @@ class ParserEngine(Parser):
666 enable_auto_tools: bool = False,
667 model_output_token_ids: Sequence[int] = (),
668 ) -> tuple[str | None, str | None, list[FunctionCall] | None]:
669 + self._initialize_history_tool_call_cnt(request)
670 self._check_skip_tool_parsing(request)
671 reasoning, content, tool_call_info = self._single_pass_parse(
672 model_output,
vllm/parser/utils.py new
+65
@@ -0,0 +1,65 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +from collections.abc import Iterable, Sequence
5 +
6 +from openai.types.responses import ResponseFunctionToolCall
7 +
8 +from vllm.entrypoints.chat_utils import ChatCompletionMessageParam
9 +from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
10 +from vllm.entrypoints.openai.responses.protocol import (
11 + ResponseInputOutputItem,
12 + ResponsesRequest,
13 +)
14 +
15 +
16 +def count_tool_calls(tool_calls: object) -> int:
17 + if tool_calls is None:
18 + return 0
19 + if isinstance(tool_calls, (str, bytes, dict)):
20 + return 1
21 + if isinstance(tool_calls, Iterable):
22 + return sum(1 for _ in tool_calls)
23 + return 1
24 +
25 +
26 +def count_chat_history_tool_calls(
27 + messages: Sequence[ChatCompletionMessageParam],
28 +) -> int:
29 + return sum(
30 + count_tool_calls(msg.get("tool_calls"))
31 + for msg in messages
32 + if isinstance(msg, dict) and msg.get("role") == "assistant"
33 + )
34 +
35 +
36 +def count_response_history_tool_calls(
37 + response_items: Sequence[ResponseInputOutputItem],
38 +) -> int:
39 + count = 0
40 + for item in response_items:
41 + if isinstance(item, ResponseFunctionToolCall):
42 + count += 1
43 + continue
44 +
45 + if isinstance(item, dict):
46 + item_type = item.get("type")
47 + if item_type == "function_call":
48 + count += 1
49 + elif item.get("role") == "assistant":
50 + count += count_tool_calls(item.get("tool_calls"))
51 +
52 + return count
53 +
54 +
55 +def count_history_tool_calls(
56 + request: ChatCompletionRequest | ResponsesRequest,
57 +) -> int:
58 + if isinstance(request, ChatCompletionRequest):
59 + return count_chat_history_tool_calls(request.messages)
60 +
61 + request_input = request.input
62 + if isinstance(request_input, str):
63 + return 0
64 +
65 + return count_response_history_tool_calls(request_input)