[Bugfix][Frontend] Honor parallel_tool_calls=false in the Responses API (#59298)
Signed-off-by: sfeng33 <4florafeng@gmail.com>
Flora Feng committed
Sep 29, 2026 at 18:36 UTC
797505747f7847e4415fb1db604421810de1865e
4 files changed
+89
-17
tests/entrypoints/openai/responses/test_serving_responses.py
+56
@@ -1359,6 +1359,7 @@ def _make_simple_context_with_output(
1359
def _make_serving_instance(
1360
*,
1361
reasoning_parser: str = "",
1362
+ tool_parser: str | None = None,
1363
enable_per_request_metrics: bool = False,
1364
) -> OpenAIServingResponses:
1365
engine_client = MagicMock()
@@ -1379,6 +1380,8 @@ def _make_serving_instance(
1380
chat_template=None,
1381
chat_template_content_format="auto",
1382
reasoning_parser=reasoning_parser,
1383
+ enable_auto_tools=tool_parser is not None,
1384
+ tool_parser=tool_parser,
1385
enable_per_request_metrics=enable_per_request_metrics,
1386
)
1387
@@ -1977,3 +1980,56 @@ class TestAutoToolStreaming:
1980
assert len(function_done) == 1
1981
assert function_done[0].item.name == "get_weather"
1982
assert function_done[0].item.arguments == tool_args
1983
+
1984
+
1985
+@pytest.mark.asyncio
1986
+@pytest.mark.parametrize(("parallel_tool_calls", "num_calls"), [(False, 1), (None, 3)])
1987
+async def test_streaming_parallel_tool_calls(
1988
+ monkeypatch, parallel_tool_calls, num_calls
1989
+):
1990
+ """parallel_tool_calls=False keeps only the first call even when the grammar
1991
+ does not limit the model; None keeps every call."""
1992
+ monkeypatch.setattr(envs, "VLLM_USE_EXPERIMENTAL_PARSER_CONTEXT", False)
1993
+ serving = _make_serving_instance(tool_parser="hermes")
1994
+ request = ResponsesRequest(
1995
+ input="hi",
1996
+ tools=[{"type": "function", "name": "get_weather", "parameters": {}}],
1997
+ parallel_tool_calls=parallel_tool_calls,
1998
+ stream=True,
1999
+ )
2000
+ context = SimpleContext(
2001
+ response_parser=serving._make_response_parser(request, MagicMock(), {})
2002
+ )
2003
+ args = [f'{{"city": "{city}"}}' for city in ("Paris", "Berlin", "Tokyo")]
2004
+
2005
+ async def result_generator():
2006
+ for i, arg in enumerate(args):
2007
+ text = f'<tool_call>\n{{"name": "get_weather", "arguments": {arg}}}'
2008
+ context.append_output(_make_request_output(f"{text}\n</tool_call>\n", [i]))
2009
+ yield context
2010
+
2011
+ events = [
2012
+ event
2013
+ async for event in serving.responses_stream_generator(
2014
+ request=request,
2015
+ sampling_params=SamplingParams(max_tokens=64),
2016
+ result_generator=result_generator(),
2017
+ context=context,
2018
+ model_name="test-model",
2019
+ tokenizer=MagicMock(),
2020
+ request_metadata=RequestResponseMetadata(request_id="req"),
2021
+ )
2022
+ ]
2023
+
2024
+ assert [
2025
+ event.output_index
2026
+ for event in events
2027
+ if event.type == "response.output_item.added"
2028
+ ] == list(range(num_calls))
2029
+ assert all(getattr(event, "output_index", 0) < num_calls for event in events)
2030
+ assert [
2031
+ event.item.arguments
2032
+ for event in events
2033
+ if event.type == "response.output_item.done"
2034
+ ] == args[:num_calls]
2035
+ assert [item.arguments for item in events[-1].response.output] == args[:num_calls]
vllm/entrypoints/openai/responses/context.py
+6
-1
@@ -37,6 +37,9 @@ from vllm.entrypoints.openai.responses.utils import (
37
build_response_output_items,
38
construct_tool_dicts,
39
)
40
+from vllm.entrypoints.serve.utils.tool_calls_utils import (
41
+ maybe_filter_parallel_tool_calls,
42
+)
43
from vllm.outputs import RequestOutput
44
from vllm.parser.abstract_parser import Parser
45
from vllm.tokenizers import TokenizerLike
@@ -369,7 +372,9 @@ class ParsableContext(ConversationContext):
372
build_response_output_items(
373
reasoning=reasoning,
374
content=content,
372
- tool_calls=tool_calls,
375
+ tool_calls=maybe_filter_parallel_tool_calls(
376
+ tool_calls or [], self.request
377
+ ),
378
tools=self.request.tools,
379
)
380
)
vllm/entrypoints/openai/responses/serving.py
+5
-1
@@ -73,6 +73,9 @@ from vllm.entrypoints.openai.responses.utils import (
73
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
74
from vllm.entrypoints.serve.utils.api_utils import get_max_tokens
75
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
76
+from vllm.entrypoints.serve.utils.tool_calls_utils import (
77
+ maybe_filter_parallel_tool_calls,
78
+)
79
from vllm.exceptions import GenerationError, VLLMValidationError
80
from vllm.inputs import EngineInput
81
from vllm.logger import init_logger
@@ -1028,7 +1031,7 @@ class OpenAIServingResponses(GenerateBaseServing):
1031
return build_response_output_items(
1032
reasoning=reasoning,
1033
content=content,
1031
- tool_calls=tool_calls,
1034
+ tool_calls=maybe_filter_parallel_tool_calls(tool_calls or [], request),
1035
logprobs=logprobs,
1036
tools=request.tools,
1037
)
@@ -1231,6 +1234,7 @@ class OpenAIServingResponses(GenerateBaseServing):
1234
1235
if not delta_message:
1236
continue
1237
+ delta_message = maybe_filter_parallel_tool_calls(delta_message, request)
1238
1239
for dm in split_delta(delta_message):
1240
target_state, tool_call = processor.resolve_target_state(dm)
vllm/entrypoints/serve/utils/tool_calls_utils.py
+22
-15
@@ -2,35 +2,42 @@
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
from typing import TypeVar
4
5
+from vllm.entrypoints.generate.base.protocol import DeltaMessage, FunctionCall
6
from vllm.entrypoints.openai.chat_completion.protocol import (
7
ChatCompletionRequest,
8
ChatCompletionResponseChoice,
9
ChatCompletionResponseStreamChoice,
10
)
11
+from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
12
13
# Used internally
12
-_ChatCompletionResponseChoiceT = TypeVar(
13
- "_ChatCompletionResponseChoiceT",
14
+_ToolCallsContainerT = TypeVar(
15
+ "_ToolCallsContainerT",
16
ChatCompletionResponseChoice,
17
ChatCompletionResponseStreamChoice,
18
+ DeltaMessage,
19
+ list[FunctionCall],
20
)
21
22
23
def maybe_filter_parallel_tool_calls(
20
- choice: _ChatCompletionResponseChoiceT, request: ChatCompletionRequest
21
-) -> _ChatCompletionResponseChoiceT:
24
+ item: _ToolCallsContainerT,
25
+ request: ChatCompletionRequest | ResponsesRequest,
26
+) -> _ToolCallsContainerT:
27
"""Filter to first tool call only when parallel_tool_calls is explicitly False."""
28
if request.parallel_tool_calls is not False:
24
- return choice
29
+ return item
30
26
- if isinstance(choice, ChatCompletionResponseChoice) and choice.message.tool_calls:
27
- choice.message.tool_calls = choice.message.tool_calls[:1]
28
- elif (
29
- isinstance(choice, ChatCompletionResponseStreamChoice)
30
- and choice.delta.tool_calls
31
- ):
32
- choice.delta.tool_calls = [
33
- tool_call for tool_call in choice.delta.tool_calls if tool_call.index == 0
34
- ]
31
+ if isinstance(item, list):
32
+ return item[:1]
33
+ if isinstance(item, ChatCompletionResponseChoice):
34
+ if item.message.tool_calls:
35
+ item.message.tool_calls = item.message.tool_calls[:1]
36
+ return item
37
36
- return choice
38
+ delta = item.delta if isinstance(item, ChatCompletionResponseStreamChoice) else item
39
+ if delta.tool_calls:
40
+ delta.tool_calls = [
41
+ tool_call for tool_call in delta.tool_calls if tool_call.index == 0
42
+ ]
43
+ return item