A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 6, 20:30 UTC
[Bugfix] Fix crash when tool_choice=required exceeds max_tokens (#36841)
Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com>
Chauncey committed
Mar 12, 2026 at 18:28 UTC
5a71cdd76ebc4f55a7490e087d2a50bd892ab3bc
3 files changed
+36
-9
tests/entrypoints/openai/test_completion_with_function_calling.py
+24
@@ -514,3 +514,27 @@ async def test_inconsistent_tool_choice_and_tools(
514
],
515
tool_choice={},
516
)
517
+
518
+
519
+@pytest.mark.asyncio
520
+async def test_max_tokens_with_tool_choice_required(client: openai.AsyncOpenAI):
521
+ """ """
522
+ models = await client.models.list()
523
+ model_name: str = models.data[0].id
524
+
525
+ # This combination previously crashed the engine
526
+ chat_completion = await client.chat.completions.create(
527
+ messages=messages,
528
+ temperature=0,
529
+ max_completion_tokens=1,
530
+ model=model_name,
531
+ tools=tools,
532
+ tool_choice="required",
533
+ )
534
+ # When `tool_choice="required"` and the tokens of `tools` exceed `max_tokens`,
535
+ # both `tool_calls` and `content` should be empty.
536
+ # This behavior should be consistent with OpenAI.
537
+ choice = chat_completion.choices[0]
538
+ assert choice.finish_reason == "length"
539
+ assert len(choice.message.tool_calls) == 0
540
+ assert choice.message.content == ""
vllm/entrypoints/openai/chat_completion/serving.py
+1
-1
@@ -1507,7 +1507,7 @@ class OpenAIServingChat(OpenAIServing):
1507
1508
elif request.tool_choice and request.tool_choice == "required":
1509
tool_call_class_items = []
1510
- assert tool_calls is not None and len(tool_calls) > 0
1510
+ tool_calls = tool_calls or []
1511
for idx, tool_call in enumerate(tool_calls):
1512
# Use native ID if available,
1513
# otherwise generate ID with correct id_type
vllm/entrypoints/openai/engine/serving.py
+11
-8
@@ -1,6 +1,7 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
import asyncio
4
+import contextlib
5
import json
6
import time
7
from collections.abc import AsyncGenerator, Callable, Mapping, Sequence
@@ -13,7 +14,7 @@ from fastapi import Request
14
from openai.types.responses import (
15
ToolChoiceFunction,
16
)
16
-from pydantic import ConfigDict, TypeAdapter
17
+from pydantic import ConfigDict, TypeAdapter, ValidationError
18
from starlette.datastructures import Headers
19
20
import vllm.envs as envs
@@ -1125,17 +1126,19 @@ class OpenAIServing:
1126
)
1127
content = None # Clear content since tool is called.
1128
elif request.tool_choice == "required":
1128
- assert content is not None
1129
- tool_calls = TypeAdapter(list[FunctionDefinition]).validate_json(content)
1130
- function_calls.extend(
1131
- [
1129
+ tool_calls = []
1130
+ with contextlib.suppress(ValidationError):
1131
+ content = content or ""
1132
+ tool_calls = TypeAdapter(list[FunctionDefinition]).validate_json(
1133
+ content
1134
+ )
1135
+ for tool_call in tool_calls:
1136
+ function_calls.append(
1137
FunctionCall(
1138
name=tool_call.name,
1139
arguments=json.dumps(tool_call.parameters, ensure_ascii=False),
1140
)
1136
- for tool_call in tool_calls
1137
- ]
1138
- )
1141
+ )
1142
content = None # Clear content since tool is called.
1143
elif (
1144
tool_parser_cls