A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 07:15 UTC
[Bugfix]: Fix the incompatibility issue with tool_choice 'required' when Thinking is enabled (#19075)
Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com>
Chauncey committed
Jun 4, 2025 at 07:27 UTC
4de790fcad85abb0969da18bc9125889407c432a
2 files changed
+16
-4
tests/entrypoints/openai/test_completion_with_function_calling.py
+1
-1
@@ -9,7 +9,7 @@ import pytest_asyncio
9
from ...utils import RemoteOpenAIServer
10
11
# any model with a chat template should work here
12
-MODEL_NAME = "Qwen/Qwen2.5-1.5B-Instruct"
12
+MODEL_NAME = "Qwen/Qwen3-0.6B"
13
14
15
@pytest.fixture(scope="module")
vllm/entrypoints/openai/serving_chat.py
+15
-3
@@ -320,10 +320,13 @@ class OpenAIServingChat(OpenAIServing):
320
def extract_tool_call_required_streaming(
321
self,
322
previous_text: str,
323
- current_text: str,
323
+ current_text: Optional[str],
324
delta_text: str,
325
function_name_returned: bool,
326
) -> tuple[Optional[DeltaMessage], bool]:
327
+ if current_text is None or current_text == "":
328
+ # if the current text is empty, we cannot parse it
329
+ return None, function_name_returned
330
try:
331
obj = partial_json_parser.loads(current_text)
332
except partial_json_parser.core.exceptions.MalformedJSON:
@@ -650,10 +653,18 @@ class OpenAIServingChat(OpenAIServing):
653
current_text = previous_text + delta_text
654
fn_name_returned = function_name_returned[i]
655
656
+ if self.reasoning_parser:
657
+ _, content = \
658
+ reasoning_parser.extract_reasoning_content(
659
+ current_text,
660
+ request
661
+ )
662
+ else:
663
+ content = current_text
664
delta_message, function_name_returned[i] = (
665
self.extract_tool_call_required_streaming(
666
previous_text=previous_text,
656
- current_text=current_text,
667
+ current_text=content,
668
delta_text=delta_text,
669
function_name_returned=fn_name_returned))
670
@@ -981,8 +992,9 @@ class OpenAIServingChat(OpenAIServing):
992
993
# the fields of FunctionDefinition are a superset of the
994
# tool call outputs and can be used for parsing
995
+ assert content is not None
996
tool_calls = TypeAdapter(
985
- list[FunctionDefinition]).validate_json(output.text)
997
+ list[FunctionDefinition]).validate_json(content)
998
message = ChatMessage(
999
role=role,
1000
content="",