A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 4, 06:15 UTC
fix(openai): tolerate empty content in forced tool choice (#40148)
Signed-off-by: QwertyJack <7554089+QwertyJack@users.noreply.github.com> Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com> Co-authored-by: QwertyJack <7554089+QwertyJack@users.noreply.github.com> Co-authored-by: chaunceyjiang <chaunceyjiang@gmail.com>
jack committed
May 6, 2026 at 22:16 UTC
6467213a9f118fcf5ebf65d5e1ee8d64d2e101e4
5 files changed
+109
-7
tests/entrypoints/openai/chat_completion/test_completion_with_function_calling.py
+8
-3
@@ -518,7 +518,13 @@ async def test_inconsistent_tool_choice_and_tools(
518
519
520
@pytest.mark.asyncio
521
-async def test_max_tokens_with_tool_choice_required(client: openai.AsyncOpenAI):
521
+@pytest.mark.parametrize(
522
+ "tool_choice",
523
+ ["required", {"type": "function", "function": {"name": "get_current_weather"}}],
524
+)
525
+async def test_max_tokens_with_tool_choice_required(
526
+ client: openai.AsyncOpenAI, tool_choice
527
+):
528
""" """
529
models = await client.models.list()
530
model_name: str = models.data[0].id
@@ -530,7 +536,7 @@ async def test_max_tokens_with_tool_choice_required(client: openai.AsyncOpenAI):
536
max_completion_tokens=1,
537
model=model_name,
538
tools=tools,
533
- tool_choice="required",
539
+ tool_choice=tool_choice,
540
)
541
# When `tool_choice="required"` and the tokens of `tools` exceed `max_tokens`,
542
# both `tool_calls` and `content` should be empty.
@@ -538,4 +544,3 @@ async def test_max_tokens_with_tool_choice_required(client: openai.AsyncOpenAI):
544
choice = chat_completion.choices[0]
545
assert choice.finish_reason == "length"
546
assert len(choice.message.tool_calls) == 0
541
- assert choice.message.content == ""
tests/entrypoints/openai/test_tool_choice_content_none.py
new
+94
@@ -0,0 +1,94 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+import pytest
5
+
6
+from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
7
+from vllm.entrypoints.openai.engine.serving import OpenAIServing
8
+from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
9
+from vllm.parser.abstract_parser import DelegatingParser
10
+
11
+pytestmark = pytest.mark.skip_global_cleanup
12
+
13
+
14
+class _DummyDelegatingParser(DelegatingParser):
15
+ def is_reasoning_end(self, input_ids: list[int]) -> bool:
16
+ return False
17
+
18
+ def extract_content_ids(self, input_ids: list[int]) -> list[int]:
19
+ return input_ids
20
+
21
+ def extract_reasoning(self, model_output: str, request):
22
+ return None, model_output
23
+
24
+ def extract_reasoning_streaming(
25
+ self,
26
+ previous_text: str,
27
+ current_text: str,
28
+ delta_text: str,
29
+ previous_token_ids: list[int],
30
+ current_token_ids: list[int],
31
+ delta_token_ids: list[int],
32
+ ):
33
+ return None
34
+
35
+ def extract_tool_calls(self, model_output: str, request):
36
+ return None
37
+
38
+
39
+def test_parse_tool_calls_from_content_allows_named_tool_choice_with_none_content():
40
+ request = ChatCompletionRequest.model_validate(
41
+ {
42
+ "model": "test-model",
43
+ "messages": [{"role": "user", "content": "test"}],
44
+ "tools": [
45
+ {
46
+ "type": "function",
47
+ "function": {
48
+ "name": "get_weather",
49
+ "parameters": {"type": "object", "properties": {}},
50
+ },
51
+ }
52
+ ],
53
+ "tool_choice": {"type": "function", "function": {"name": "get_weather"}},
54
+ }
55
+ )
56
+
57
+ tool_calls, content = OpenAIServing._parse_tool_calls_from_content(
58
+ request=request,
59
+ tokenizer=None,
60
+ enable_auto_tools=True,
61
+ tool_parser_cls=None,
62
+ content=None,
63
+ )
64
+
65
+ assert content is None
66
+ assert tool_calls is not None
67
+ assert tool_calls == []
68
+
69
+
70
+def test_responses_parser_allows_named_tool_choice_with_none_content():
71
+ request = ResponsesRequest.model_validate(
72
+ {
73
+ "model": "test-model",
74
+ "input": "test",
75
+ "tools": [
76
+ {
77
+ "type": "function",
78
+ "name": "get_weather",
79
+ "parameters": {"type": "object", "properties": {}},
80
+ }
81
+ ],
82
+ "tool_choice": {"type": "function", "name": "get_weather"},
83
+ }
84
+ )
85
+ parser = _DummyDelegatingParser(tokenizer=None)
86
+
87
+ tool_calls, content = parser._parse_tool_calls(
88
+ request=request,
89
+ content=None,
90
+ enable_auto_tools=False,
91
+ )
92
+
93
+ assert content is None
94
+ assert tool_calls == []
vllm/entrypoints/openai/chat_completion/serving.py
+1
-1
@@ -1307,8 +1307,8 @@ class OpenAIServingChat(OpenAIServing):
1307
request.tool_choice
1308
and type(request.tool_choice) is ChatCompletionNamedToolChoiceParam
1309
):
1310
- assert tool_calls is not None and len(tool_calls) > 0
1310
tool_call_class_items = []
1311
+ tool_calls = tool_calls or []
1312
for idx, tc in enumerate(tool_calls):
1313
# Use native ID if available (e.g., Kimi K2),
1314
# otherwise generate ID with correct id_type
vllm/entrypoints/openai/engine/serving.py
+4
-2
@@ -638,8 +638,9 @@ class OpenAIServing:
638
and request.tool_choice
639
and isinstance(request.tool_choice, ToolChoiceFunction)
640
):
641
- assert content is not None
641
# Forced Function Call (Responses API)
642
+ if content is None:
643
+ return [], None
644
function_calls.append(
645
FunctionCall(name=request.tool_choice.name, arguments=content)
646
)
@@ -651,7 +652,8 @@ class OpenAIServing:
652
and (tool_parser_cls is None or tool_parser_cls.supports_required_and_named)
653
):
654
# Named function with standard JSON-based parsing
654
- assert content is not None
655
+ if content is None:
656
+ return [], None
657
function_calls.append(
658
FunctionCall(name=request.tool_choice.function.name, arguments=content)
659
)
vllm/parser/abstract_parser.py
+2
-1
@@ -459,7 +459,8 @@ class DelegatingParser(Parser):
459
(ToolChoiceFunction, ChatCompletionNamedToolChoiceParam),
460
):
461
# Forced Function Call
462
- assert content is not None
462
+ if content is None:
463
+ return [], None
464
function_calls.append(
465
FunctionCall(name=self._get_function_name(request), arguments=content)
466
)