A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 8, 21:30 UTC
[CI][Bugfix]: return McpCall for built-in MCP tools in non-streaming mode (#32762)
Signed-off-by: Andreas Karatzas <akaratza@amd.com>
Andreas Karatzas committed
Feb 4, 2026 at 21:14 UTC
fb1270f1f821603402a8868e3067b3c3342455e7
6 files changed
+131
-28
tests/entrypoints/openai/responses/test_harmony.py
+14
-8
@@ -62,7 +62,7 @@ async def client(server):
62
async def test_basic(client: OpenAI, model_name: str):
63
response = await client.responses.create(
64
model=model_name,
65
- input="What is 13 * 24?",
65
+ input="What is 123 * 456?",
66
)
67
assert response is not None
68
print("response: ", response)
@@ -74,7 +74,7 @@ async def test_basic(client: OpenAI, model_name: str):
74
async def test_basic_with_instructions(client: OpenAI, model_name: str):
75
response = await client.responses.create(
76
model=model_name,
77
- input="What is 13 * 24?",
77
+ input="What is 123 * 456?",
78
instructions="Respond in Korean.",
79
)
80
assert response is not None
@@ -116,7 +116,7 @@ async def test_chat(client: OpenAI, model_name: str):
116
{"role": "system", "content": "Respond in Korean."},
117
{"role": "user", "content": "Hello!"},
118
{"role": "assistant", "content": "Hello! How can I help you today?"},
119
- {"role": "user", "content": "What is 13 * 24? Explain your answer."},
119
+ {"role": "user", "content": "What is 123 * 456? Explain your answer."},
120
],
121
)
122
assert response is not None
@@ -131,7 +131,7 @@ async def test_chat_with_input_type(client: OpenAI, model_name: str):
131
input=[
132
{
133
"role": "user",
134
- "content": [{"type": "input_text", "text": "What is 13*24?"}],
134
+ "content": [{"type": "input_text", "text": "What is 123 * 456?"}],
135
},
136
],
137
)
@@ -200,7 +200,7 @@ async def test_store(client: OpenAI, model_name: str):
200
for store in [True, False]:
201
response = await client.responses.create(
202
model=model_name,
203
- input="What is 13 * 24?",
203
+ input="What is 123 * 456?",
204
store=store,
205
)
206
assert response is not None
@@ -219,7 +219,7 @@ async def test_store(client: OpenAI, model_name: str):
219
async def test_background(client: OpenAI, model_name: str):
220
response = await client.responses.create(
221
model=model_name,
222
- input="What is 13 * 24?",
222
+ input="What is 123 * 456?",
223
background=True,
224
)
225
assert response is not None
@@ -256,7 +256,7 @@ async def test_background_cancel(client: OpenAI, model_name: str):
256
async def test_stateful_multi_turn(client: OpenAI, model_name: str):
257
response1 = await client.responses.create(
258
model=model_name,
259
- input="What is 13 * 24?",
259
+ input="What is 123 * 456?",
260
)
261
assert response1 is not None
262
assert response1.status == "completed"
@@ -361,7 +361,7 @@ async def test_streaming(client: OpenAI, model_name: str, background: bool):
361
# TODO: Add back when web search and code interpreter are available in CI
362
prompts = [
363
"tell me a story about a cat in 20 words",
364
- "What is 13 * 24? Use python to calculate the result.",
364
+ "What is 123 * 456? Use python to calculate the result.",
365
# "When did Jensen found NVIDIA? Search it and answer the year only.",
366
]
367
@@ -976,6 +976,9 @@ async def test_mcp_code_interpreter_streaming(client: OpenAI, model_name: str, s
976
977
@pytest.mark.asyncio
978
@pytest.mark.parametrize("model_name", [MODEL_NAME])
979
+@pytest.mark.dependency(
980
+ depends=["test_mcp_code_interpreter_streaming[openai/gpt-oss-20b]"]
981
+)
982
async def test_mcp_tool_multi_turn(client: OpenAI, model_name: str, server):
983
"""Test MCP tool calling across multiple turns.
984
@@ -1117,8 +1120,10 @@ async def test_function_call_with_previous_input_messages(
1120
model=model_name,
1121
input="What is the horoscope for Aquarius today?",
1122
tools=tools,
1123
+ temperature=0.0,
1124
extra_body={"enable_response_messages": True},
1125
stream=True,
1126
+ max_output_tokens=1000,
1127
)
1128
1129
response = None
@@ -1170,6 +1175,7 @@ async def test_function_call_with_previous_input_messages(
1175
stream_response_2 = await client.responses.create(
1176
model=model_name,
1177
tools=tools,
1178
+ temperature=0.0,
1179
input="",
1180
extra_body={
1181
"previous_input_messages": previous_messages,
tests/entrypoints/openai/responses/test_mcp_tools.py
+1
@@ -160,6 +160,7 @@ class TestMCPEnabled:
160
"No developer messages should be present with valid mcp tool"
161
)
162
163
+ @pytest.mark.flaky(reruns=3)
164
@pytest.mark.asyncio
165
@pytest.mark.parametrize("model_name", [MODEL_NAME])
166
async def test_mcp_tool_with_allowed_tools_star(
tests/entrypoints/openai/responses/test_parsable_context.py
+4
-4
@@ -53,7 +53,7 @@ async def client(server):
53
async def test_basic(client: OpenAI, model_name: str):
54
response = await client.responses.create(
55
model=model_name,
56
- input="What is 13 * 24?",
56
+ input="What is 123 * 456?",
57
)
58
assert response is not None
59
print("response: ", response)
@@ -164,7 +164,7 @@ async def test_function_call_first_turn(client: OpenAI, model_name: str):
164
async def test_mcp_tool_call(client: OpenAI, model_name: str):
165
response = await client.responses.create(
166
model=model_name,
167
- input="What is 13 * 24? Use python to calculate the result.",
167
+ input="What is 123 * 456? Use python to calculate the result.",
168
tools=[{"type": "code_interpreter", "container": {"type": "auto"}}],
169
extra_body={"enable_response_messages": True},
170
temperature=0.0,
@@ -179,12 +179,12 @@ async def test_mcp_tool_call(client: OpenAI, model_name: str):
179
assert response.output[2].type == "reasoning"
180
# make sure the correct math is in the final output
181
assert response.output[3].type == "message"
182
- assert "312" in response.output[3].content[0].text
182
+ assert "56088" in response.output[3].content[0].text
183
184
# test raw input_messages / output_messages
185
assert len(response.input_messages) == 1
186
assert len(response.output_messages) == 3
187
- assert "312" in response.output_messages[2]["message"]
187
+ assert "56088" in response.output_messages[2]["message"]
188
189
190
@pytest.mark.asyncio
tests/entrypoints/openai/responses/test_simple.py
+1
-1
@@ -34,7 +34,7 @@ async def client(server):
34
async def test_basic(client: OpenAI, model_name: str):
35
response = await client.responses.create(
36
model=model_name,
37
- input="What is 13 * 24?",
37
+ input="What is 123 * 456?",
38
)
39
assert response is not None
40
print("response: ", response)
tests/utils.py
+76
-2
@@ -197,12 +197,86 @@ class RemoteOpenAIServer:
197
return self
198
199
def __exit__(self, exc_type, exc_value, traceback):
200
+ pid = self.proc.pid
201
+ # Graceful shutdown
202
self.proc.terminate()
203
try:
202
- self.proc.wait(8)
204
+ self.proc.wait(timeout=15)
205
+ print(f"[RemoteOpenAIServer] Server {pid} terminated gracefully")
206
except subprocess.TimeoutExpired:
204
- # force kill if needed
207
+ print(
208
+ f"[RemoteOpenAIServer] Server {pid} did not respond "
209
+ "to SIGTERM, sending SIGKILL"
210
+ )
211
self.proc.kill()
212
+ try:
213
+ self.proc.wait(timeout=5)
214
+ print(f"[RemoteOpenAIServer] Server {pid} killed")
215
+ except subprocess.TimeoutExpired as err:
216
+ raise RuntimeError(
217
+ f"[RemoteOpenAIServer] Failed to kill server process {pid}"
218
+ ) from err
219
+ # Wait for GPU memory to be released
220
+ self._wait_for_gpu_memory_release()
221
+
222
+ def _get_gpu_memory_used(self) -> float | None:
223
+ """Get total GPU memory used across all visible devices in bytes."""
224
+ try:
225
+ if current_platform.is_rocm():
226
+ with _nvml():
227
+ handles = amdsmi_get_processor_handles()
228
+ total_used = 0
229
+ for handle in handles:
230
+ vram_info = amdsmi_get_gpu_vram_usage(handle)
231
+ total_used += vram_info["vram_used"]
232
+ return total_used
233
+ elif current_platform.is_cuda():
234
+ with _nvml():
235
+ total_used = 0
236
+ device_count = cuda_device_count_stateless()
237
+ for i in range(device_count):
238
+ handle = nvmlDeviceGetHandleByIndex(i)
239
+ mem_info = nvmlDeviceGetMemoryInfo(handle)
240
+ total_used += mem_info.used
241
+ return total_used
242
+ except Exception as e:
243
+ print(f"[RemoteOpenAIServer] Could not query GPU memory: {e}")
244
+ return None
245
+ return None
246
+
247
+ def _wait_for_gpu_memory_release(self, timeout: float = 30.0):
248
+ """Poll GPU memory until it stabilizes, indicating cleanup is complete."""
249
+ start = time.time()
250
+ prev_used: float | None = None
251
+ stable_count = 0
252
+
253
+ while time.time() - start < timeout:
254
+ used = self._get_gpu_memory_used()
255
+
256
+ if used is None:
257
+ return # Can't query, assume ok
258
+
259
+ if prev_used is not None and abs(used - prev_used) < 100 * 1024 * 1024:
260
+ stable_count += 1
261
+ if stable_count >= 3:
262
+ used_gb = used / 1e9
263
+ print(
264
+ f"[RemoteOpenAIServer] GPU memory stabilized "
265
+ f"at {used_gb:.2f} GB"
266
+ )
267
+ return
268
+ else:
269
+ stable_count = 0
270
+
271
+ prev_used = used
272
+ time.sleep(0.1)
273
+
274
+ last_reading = prev_used / 1e9 if prev_used is not None else 0.0
275
+ raise RuntimeError(
276
+ f"[RemoteOpenAIServer] GPU memory did not stabilize within {timeout}s. "
277
+ f"Last reading: {last_reading:.2f} GB. "
278
+ "Child processes may still be holding GPU memory."
279
+ )
280
281
def _poll(self) -> int | None:
282
"""Subclasses override this method to customize process polling"""
vllm/entrypoints/openai/parser/harmony_utils.py
+35
-13
@@ -68,6 +68,14 @@ MCP_BUILTIN_TOOLS: set[str] = {
68
"container",
69
}
70
71
+# Mapping from built-in tool recipient names to their MCP server labels.
72
+# This ensures consistency between streaming and non-streaming responses.
73
+_BUILTIN_TOOL_TO_MCP_SERVER_LABEL: dict[str, str] = {
74
+ "python": "code_interpreter",
75
+ "browser": "web_search_preview",
76
+ "container": "container",
77
+}
78
+
79
80
def has_custom_tools(tool_types: set[str]) -> bool:
81
"""
@@ -601,7 +609,13 @@ def _parse_mcp_recipient(recipient: str) -> tuple[str, str]:
609
610
def _parse_mcp_call(message: Message, recipient: str) -> list[ResponseOutputItem]:
611
"""Parse MCP calls into MCP call items."""
604
- server_label, tool_name = _parse_mcp_recipient(recipient)
612
+ # Handle built-in tools that need server_label mapping
613
+ if recipient in _BUILTIN_TOOL_TO_MCP_SERVER_LABEL:
614
+ server_label = _BUILTIN_TOOL_TO_MCP_SERVER_LABEL[recipient]
615
+ tool_name = recipient
616
+ else:
617
+ server_label, tool_name = _parse_mcp_recipient(recipient)
618
+
619
output_items = []
620
for content in message.content:
621
response_item = McpCall(
@@ -630,7 +644,7 @@ def parse_output_message(message: Message) -> list[ResponseOutputItem]:
644
recipient = message.recipient
645
646
if recipient is not None:
633
- # Browser tool calls
647
+ # Browser tool calls (browser.search, browser.open, browser.find)
648
if recipient.startswith("browser."):
649
output_items.append(_parse_browser_tool_call(message, recipient))
650
@@ -638,10 +652,8 @@ def parse_output_message(message: Message) -> list[ResponseOutputItem]:
652
elif message.channel == "commentary" and recipient.startswith("functions."):
653
output_items.extend(_parse_function_call(message, recipient))
654
641
- # Built-in tools are treated as reasoning
642
- elif recipient.startswith(("python", "browser", "container")):
643
- # Built-in tool recipients (python/browser/container)
644
- # generate reasoning output
655
+ # Built-in MCP tools (python, browser, container)
656
+ elif recipient in _BUILTIN_TOOL_TO_MCP_SERVER_LABEL:
657
output_items.extend(_parse_reasoning(message))
658
659
# All other recipients are MCP calls
@@ -688,13 +700,23 @@ def parse_remaining_state(parser: StreamableParser) -> list[ResponseOutputItem]:
700
status="in_progress",
701
)
702
]
691
- # Built-in tools (python, browser, container) should be treated as reasoning
692
- elif not (
693
- current_recipient.startswith("python")
694
- or current_recipient.startswith("browser")
695
- or current_recipient.startswith("container")
696
- ):
697
- # All other recipients are MCP calls
703
+ # Built-in MCP tools (python, browser, container)
704
+ elif current_recipient in _BUILTIN_TOOL_TO_MCP_SERVER_LABEL:
705
+ return [
706
+ ResponseReasoningItem(
707
+ id=f"rs_{random_uuid()}",
708
+ summary=[],
709
+ type="reasoning",
710
+ content=[
711
+ ResponseReasoningTextContent(
712
+ text=parser.current_content, type="reasoning_text"
713
+ )
714
+ ],
715
+ status=None,
716
+ )
717
+ ]
718
+ # All other recipients are MCP calls
719
+ else:
720
rid = random_uuid()
721
server_label, tool_name = _parse_mcp_recipient(current_recipient)
722
return [