A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 11, 03:45 UTC
[BugFix]fix gpt-oss v1/completions response bug (#30608)
Signed-off-by: princepride <wangzhipeng628@gmail.com> Signed-off-by: 汪志鹏 <wangzhipeng628@gmail.com> Co-authored-by: Chauncey <chaunceyjiang@gmail.com> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk> Co-authored-by: bbrowning <bbrownin@redhat.com>
汪志鹏 committed
Dec 21, 2025 at 10:39 UTC
3e92b2b7acaa61335ecd7bea5eeed50388739194
4 files changed
+40
-14
tests/entrypoints/openai/test_response_api_with_harmony.py
+21
@@ -1,6 +1,7 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
import importlib
4
+import importlib.util
5
import json
6
import time
7
@@ -986,3 +987,23 @@ async def test_function_call_with_previous_input_messages(
987
assert (
988
"aquarius" in output_text or "otter" in output_text or "tuesday" in output_text
989
)
990
+
991
+
992
+@pytest.mark.asyncio
993
+@pytest.mark.parametrize("model_name", [MODEL_NAME])
994
+async def test_chat_truncation_content_not_null(client: OpenAI, model_name: str):
995
+ response = await client.chat.completions.create(
996
+ model=model_name,
997
+ messages=[{"role": "user", "content": "What is the role of AI in medicine?"}],
998
+ temperature=0.0,
999
+ max_tokens=250,
1000
+ )
1001
+
1002
+ choice = response.choices[0]
1003
+ assert choice.finish_reason == "length", (
1004
+ f"Expected finish_reason='length', got {choice.finish_reason}"
1005
+ )
1006
+ assert choice.message.content is not None, (
1007
+ "Content should not be None when truncated"
1008
+ )
1009
+ assert len(choice.message.content) > 0, "Content should not be empty"
tests/entrypoints/openai/test_serving_chat.py
+5
-10
@@ -955,7 +955,6 @@ class TestServingChatWithHarmony:
955
input_messages,
956
[
957
{"role": "system"},
958
- {"role": "developer"},
958
{"role": "user", "content": messages[0]["content"]},
959
],
960
)
@@ -983,7 +982,6 @@ class TestServingChatWithHarmony:
982
input_messages_2,
983
[
984
{"role": "system"},
986
- {"role": "developer"},
985
{"role": "user"},
986
# The analysis message should be dropped on subsequent inputs because
987
# of the subsequent assistant message to the final channel.
@@ -1043,7 +1041,7 @@ class TestServingChatWithHarmony:
1041
)
1042
1043
# Test the Harmony messages for the second turn's input
1046
- req_2 = ChatCompletionRequest(model=MODEL_NAME, messages=messages)
1044
+ req_2 = ChatCompletionRequest(model=MODEL_NAME, messages=messages, tools=tools)
1045
input_messages_2, _ = serving_chat._make_request_with_harmony(req_2)
1046
verify_harmony_messages(
1047
input_messages_2,
@@ -1124,7 +1122,7 @@ class TestServingChatWithHarmony:
1122
)
1123
1124
# Test the Harmony messages for the second turn's input
1127
- req_2 = ChatCompletionRequest(model=MODEL_NAME, messages=messages)
1125
+ req_2 = ChatCompletionRequest(model=MODEL_NAME, messages=messages, tools=tools)
1126
input_messages_2, _ = serving_chat._make_request_with_harmony(req_2)
1127
verify_harmony_messages(
1128
input_messages_2,
@@ -1205,7 +1203,7 @@ class TestServingChatWithHarmony:
1203
)
1204
1205
# Test the Harmony messages for the second turn's input
1208
- req_2 = ChatCompletionRequest(model=MODEL_NAME, messages=messages)
1206
+ req_2 = ChatCompletionRequest(model=MODEL_NAME, messages=messages, tools=tools)
1207
input_messages_2, _ = serving_chat._make_request_with_harmony(req_2)
1208
verify_harmony_messages(
1209
input_messages_2,
@@ -1255,7 +1253,7 @@ class TestServingChatWithHarmony:
1253
)
1254
1255
# Test the Harmony messages for the third turn's input
1258
- req_3 = ChatCompletionRequest(model=MODEL_NAME, messages=messages)
1256
+ req_3 = ChatCompletionRequest(model=MODEL_NAME, messages=messages, tools=tools)
1257
input_messages_3, _ = serving_chat._make_request_with_harmony(req_3)
1258
verify_harmony_messages(
1259
input_messages_3,
@@ -1318,7 +1316,7 @@ class TestServingChatWithHarmony:
1316
)
1317
1318
# Test the Harmony messages for the fourth turn's input
1321
- req_4 = ChatCompletionRequest(model=MODEL_NAME, messages=messages)
1319
+ req_4 = ChatCompletionRequest(model=MODEL_NAME, messages=messages, tools=tools)
1320
input_messages_4, _ = serving_chat._make_request_with_harmony(req_4)
1321
verify_harmony_messages(
1322
input_messages_4,
@@ -1374,7 +1372,6 @@ class TestServingChatWithHarmony:
1372
input_messages,
1373
[
1374
{"role": "system"},
1377
- {"role": "developer"},
1375
{"role": "user", "content": messages[0]["content"]},
1376
# The reasoning that would have resulted in an analysis message is
1377
# dropped because of a later assistant message to the final channel.
@@ -1406,7 +1403,6 @@ class TestServingChatWithHarmony:
1403
input_messages,
1404
[
1405
{"role": "system"},
1409
- {"role": "developer"},
1406
{"role": "user", "content": messages[0]["content"]},
1407
{
1408
"role": "assistant",
@@ -1436,7 +1432,6 @@ class TestServingChatWithHarmony:
1432
input_messages,
1433
[
1434
{"role": "system"},
1439
- {"role": "developer"},
1435
{"role": "user", "content": messages[0]["content"]},
1436
{
1437
"role": "assistant",
vllm/entrypoints/openai/serving_chat.py
+5
-4
@@ -1828,10 +1828,11 @@ class OpenAIServingChat(OpenAIServing):
1828
messages.append(sys_msg)
1829
1830
# Add developer message.
1831
- dev_msg = get_developer_message(
1832
- tools=request.tools if should_include_tools else None
1833
- )
1834
- messages.append(dev_msg)
1831
+ if request.tools:
1832
+ dev_msg = get_developer_message(
1833
+ tools=request.tools if should_include_tools else None
1834
+ )
1835
+ messages.append(dev_msg)
1836
1837
# Add user message.
1838
messages.extend(parse_chat_inputs_to_harmony_messages(request.messages))
vllm/tool_parsers/openai_tool_parser.py
+9
@@ -79,6 +79,15 @@ class OpenAIToolParser(ToolParser):
79
elif msg.channel == "commentary" and not msg.recipient:
80
commentary_content = msg_text
81
82
+ # Extract partial content from the parser state if the generation was truncated
83
+ if parser.current_content:
84
+ if parser.current_channel == "final":
85
+ final_content = parser.current_content
86
+ elif (
87
+ parser.current_channel == "commentary" and not parser.current_recipient
88
+ ):
89
+ commentary_content = parser.current_content
90
+
91
return ExtractedToolCallInformation(
92
tools_called=len(tool_calls) > 0,
93
tool_calls=tool_calls,