A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 12, 13:30 UTC
[Frontend] Respect Chat Completion parallel_tool_calls param (#26233)
Signed-off-by: Ben Browning <bbrownin@redhat.com> Co-authored-by: Chauncey <chaunceyjiang@gmail.com>
Ben Browning committed
Nov 25, 2025 at 04:56 UTC
e1dd706cd1f2b008f6295a6a64634bf9e9b202c1
6 files changed
+102
-8
docs/serving/openai_compatible_server.md
+2
-1
@@ -49,7 +49,8 @@ We currently support the following OpenAI APIs:
49
- *Note: `suffix` parameter is not supported.*
50
- [Chat Completions API](#chat-api) (`/v1/chat/completions`)
51
- Only applicable to [text generation models](../models/generative_models.md) with a [chat template](../serving/openai_compatible_server.md#chat-template).
52
- - *Note: `parallel_tool_calls` and `user` parameters are ignored.*
52
+ - *Note: `user` parameter is ignored.*
53
+ - *Note:* Setting the `parallel_tool_calls` parameter to `false` ensures vLLM only returns zero or one tool call per request. Setting it to `true` (the default) allows returning more than one tool call per request. There is no guarantee more than one tool call will be returned if this is set to `true`, as that behavior is model dependent and not all models are designed to support parallel tool calls.
54
- [Embeddings API](#embeddings-api) (`/v1/embeddings`)
55
- Only applicable to [embedding models](../models/pooling_models.md).
56
- [Transcriptions API](#transcriptions-api) (`/v1/audio/transcriptions`)
tests/tool_use/test_parallel_tool_calls.py
+57
@@ -212,3 +212,60 @@ async def test_parallel_tool_calls_with_results(
212
assert finish_reason_count == 1
213
assert len(chunks)
214
assert "".join(chunks) == choice.message.content
215
+
216
+
217
+@pytest.mark.asyncio
218
+async def test_parallel_tool_calls_false(client: openai.AsyncOpenAI):
219
+ """
220
+ Ensure only one tool call is returned when parallel_tool_calls is False.
221
+ """
222
+
223
+ models = await client.models.list()
224
+ model_name: str = models.data[0].id
225
+ chat_completion = await client.chat.completions.create(
226
+ messages=MESSAGES_ASKING_FOR_PARALLEL_TOOLS,
227
+ temperature=0,
228
+ max_completion_tokens=200,
229
+ model=model_name,
230
+ tools=[WEATHER_TOOL, SEARCH_TOOL],
231
+ logprobs=False,
232
+ parallel_tool_calls=False,
233
+ )
234
+
235
+ stop_reason = chat_completion.choices[0].finish_reason
236
+ non_streamed_tool_calls = chat_completion.choices[0].message.tool_calls
237
+
238
+ # make sure only 1 tool call is present
239
+ assert len(non_streamed_tool_calls) == 1
240
+ assert stop_reason == "tool_calls"
241
+
242
+ # make the same request, streaming
243
+ stream = await client.chat.completions.create(
244
+ model=model_name,
245
+ messages=MESSAGES_ASKING_FOR_PARALLEL_TOOLS,
246
+ temperature=0,
247
+ max_completion_tokens=200,
248
+ tools=[WEATHER_TOOL, SEARCH_TOOL],
249
+ logprobs=False,
250
+ parallel_tool_calls=False,
251
+ stream=True,
252
+ )
253
+
254
+ finish_reason_count: int = 0
255
+ tool_call_id_count: int = 0
256
+
257
+ async for chunk in stream:
258
+ # if there's a finish reason make sure it's tools
259
+ if chunk.choices[0].finish_reason:
260
+ finish_reason_count += 1
261
+ assert chunk.choices[0].finish_reason == "tool_calls"
262
+
263
+ streamed_tool_calls = chunk.choices[0].delta.tool_calls
264
+ if streamed_tool_calls and len(streamed_tool_calls) > 0:
265
+ tool_call = streamed_tool_calls[0]
266
+ if tool_call.id:
267
+ tool_call_id_count += 1
268
+
269
+ # make sure only 1 streaming tool call is present
270
+ assert tool_call_id_count == 1
271
+ assert finish_reason_count == 1
vllm/entrypoints/openai/protocol.py
+2
-2
@@ -559,9 +559,9 @@ class ChatCompletionRequest(OpenAIBaseModel):
559
) = "none"
560
reasoning_effort: Literal["low", "medium", "high"] | None = None
561
include_reasoning: bool = True
562
+ parallel_tool_calls: bool | None = True
563
563
- # NOTE this will be ignored by vLLM -- the model determines the behavior
564
- parallel_tool_calls: bool | None = False
564
+ # NOTE this will be ignored by vLLM
565
user: str | None = None
566
567
# --8<-- [start:chat-completion-sampling-params]
vllm/entrypoints/openai/serving_chat.py
+3
@@ -55,6 +55,7 @@ from vllm.entrypoints.openai.serving_engine import OpenAIServing, clamp_prompt_l
55
from vllm.entrypoints.openai.serving_models import OpenAIServingModels
56
from vllm.entrypoints.openai.tool_parsers import ToolParser
57
from vllm.entrypoints.openai.tool_parsers.mistral_tool_parser import MistralToolCall
58
+from vllm.entrypoints.openai.utils import maybe_filter_parallel_tool_calls
59
from vllm.entrypoints.utils import get_max_tokens, should_include_usage
60
from vllm.inputs.data import TokensPrompt as EngineTokensPrompt
61
from vllm.logger import init_logger
@@ -1206,6 +1207,7 @@ class OpenAIServingChat(OpenAIServing):
1207
1208
finish_reason_sent[i] = True
1209
1210
+ choice_data = maybe_filter_parallel_tool_calls(choice_data, request)
1211
chunk = ChatCompletionStreamResponse(
1212
id=request_id,
1213
object=chunk_object_type,
@@ -1531,6 +1533,7 @@ class OpenAIServingChat(OpenAIServing):
1533
as_list(output.token_ids) if request.return_token_ids else None
1534
),
1535
)
1536
+ choice_data = maybe_filter_parallel_tool_calls(choice_data, request)
1537
1538
choices.append(choice_data)
1539
vllm/entrypoints/openai/serving_engine.py
+1
-5
@@ -296,11 +296,7 @@ class OpenAIServing:
296
parser = None
297
if not enable_auto_tools or tool_parser_name is None:
298
return parser
299
- logger.info(
300
- '"auto" tool choice has been enabled please note that while'
301
- " the parallel_tool_calls client option is preset for "
302
- "compatibility reasons, it will be ignored."
303
- )
299
+ logger.info('"auto" tool choice has been enabled.')
300
301
try:
302
if tool_parser_name == "pythonic" and self.model_config.model.startswith(
vllm/entrypoints/openai/utils.py
new
+37
@@ -0,0 +1,37 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+from typing import TypeVar
4
+
5
+from vllm.entrypoints.openai.protocol import (
6
+ ChatCompletionRequest,
7
+ ChatCompletionResponseChoice,
8
+ ChatCompletionResponseStreamChoice,
9
+)
10
+
11
+# Used internally
12
+_ChatCompletionResponseChoiceT = TypeVar(
13
+ "_ChatCompletionResponseChoiceT",
14
+ ChatCompletionResponseChoice,
15
+ ChatCompletionResponseStreamChoice,
16
+)
17
+
18
+
19
+def maybe_filter_parallel_tool_calls(
20
+ choice: _ChatCompletionResponseChoiceT, request: ChatCompletionRequest
21
+) -> _ChatCompletionResponseChoiceT:
22
+ """Filter to first tool call only when parallel_tool_calls is False."""
23
+
24
+ if request.parallel_tool_calls:
25
+ return choice
26
+
27
+ if isinstance(choice, ChatCompletionResponseChoice) and choice.message.tool_calls:
28
+ choice.message.tool_calls = choice.message.tool_calls[:1]
29
+ elif (
30
+ isinstance(choice, ChatCompletionResponseStreamChoice)
31
+ and choice.delta.tool_calls
32
+ ):
33
+ choice.delta.tool_calls = [
34
+ tool_call for tool_call in choice.delta.tool_calls if tool_call.index == 0
35
+ ]
36
+
37
+ return choice