A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 8, 00:00 UTC
[CI][MCP][Harmony] Heavy refactoring Harmony & MCP response tests and stabilizing with deterministic test infrastructure (#33949)
Signed-off-by: Andreas Karatzas <akaratza@amd.com>
Andreas Karatzas committed
Feb 20, 2026 at 22:03 UTC
991d6bff38ff02f7cf47a3833efce58b27db8bb8
10 files changed
+1177
-876
tests/entrypoints/openai/responses/conftest.py
+171
@@ -1,7 +1,22 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+from __future__ import annotations
4
+
5
+import json
6
+import logging
7
+from collections.abc import Callable
8
+from typing import Any
9
+
10
import pytest
11
12
+logger = logging.getLogger(__name__)
13
+
14
+BASE_TEST_ENV = {
15
+ # The day vLLM said "hello world" on arxiv 🚀
16
+ "VLLM_SYSTEM_START_DATE": "2023-09-12",
17
+}
18
+DEFAULT_MAX_RETRIES = 3
19
+
20
21
@pytest.fixture
22
def pairs_of_event_types() -> dict[str, str]:
@@ -28,3 +43,159 @@ def pairs_of_event_types() -> dict[str, str]:
43
}
44
# fmt: on
45
return event_pairs
46
+
47
+
48
+async def retry_for_tool_call(
49
+ client,
50
+ *,
51
+ model: str,
52
+ expected_tool_type: str,
53
+ max_retries: int = DEFAULT_MAX_RETRIES,
54
+ **create_kwargs: Any,
55
+):
56
+ """Call ``client.responses.create`` up to *max_retries* times, returning
57
+ the first response that contains an output item of *expected_tool_type*.
58
+
59
+ Returns the **last** response if none match so the caller's assertions
60
+ fire with a clear diagnostic.
61
+ """
62
+ last_response = None
63
+ for attempt in range(max_retries):
64
+ response = await client.responses.create(model=model, **create_kwargs)
65
+ last_response = response
66
+ if any(
67
+ getattr(item, "type", None) == expected_tool_type
68
+ for item in response.output
69
+ ):
70
+ return response
71
+ assert last_response is not None
72
+ return last_response
73
+
74
+
75
+async def retry_streaming_for(
76
+ client,
77
+ *,
78
+ model: str,
79
+ validate_events: Callable[[list], bool],
80
+ max_retries: int = DEFAULT_MAX_RETRIES,
81
+ **create_kwargs: Any,
82
+) -> list:
83
+ """Call ``client.responses.create(stream=True)`` up to *max_retries*
84
+ times, returning the first event list where *validate_events* returns
85
+ ``True``.
86
+ """
87
+ last_events: list = []
88
+ for attempt in range(max_retries):
89
+ stream = await client.responses.create(
90
+ model=model, stream=True, **create_kwargs
91
+ )
92
+ events: list = []
93
+ async for event in stream:
94
+ events.append(event)
95
+ last_events = events
96
+ if validate_events(events):
97
+ return events
98
+ return last_events
99
+
100
+
101
+def has_output_type(response, type_name: str) -> bool:
102
+ """Return True if *response* has at least one output item of *type_name*."""
103
+ return any(getattr(item, "type", None) == type_name for item in response.output)
104
+
105
+
106
+def events_contain_type(events: list, type_substring: str) -> bool:
107
+ """Return True if any event's type contains *type_substring*."""
108
+ return any(type_substring in getattr(e, "type", "") for e in events)
109
+
110
+
111
+def validate_streaming_event_stack(
112
+ events: list, pairs_of_event_types: dict[str, str]
113
+) -> None:
114
+ """Validate that streaming events are properly nested/paired."""
115
+ stack: list[str] = []
116
+ for event in events:
117
+ etype = event.type
118
+ if etype == "response.created":
119
+ stack.append(etype)
120
+ elif etype == "response.completed":
121
+ assert stack and stack[-1] == pairs_of_event_types[etype], (
122
+ f"Unexpected stack top for {etype}: "
123
+ f"got {stack[-1] if stack else '<empty>'}"
124
+ )
125
+ stack.pop()
126
+ elif etype.endswith("added") or etype == "response.mcp_call.in_progress":
127
+ stack.append(etype)
128
+ elif etype.endswith("delta"):
129
+ if stack and stack[-1] == etype:
130
+ continue
131
+ stack.append(etype)
132
+ elif etype.endswith("done") or etype == "response.mcp_call.completed":
133
+ assert etype in pairs_of_event_types, f"Unknown done event: {etype}"
134
+ expected_start = pairs_of_event_types[etype]
135
+ assert stack and stack[-1] == expected_start, (
136
+ f"Stack mismatch for {etype}: "
137
+ f"expected {expected_start}, "
138
+ f"got {stack[-1] if stack else '<empty>'}"
139
+ )
140
+ stack.pop()
141
+ assert len(stack) == 0, f"Unclosed events on stack: {stack}"
142
+
143
+
144
+def log_response_diagnostics(
145
+ response,
146
+ *,
147
+ label: str = "Response Diagnostics",
148
+) -> dict[str, Any]:
149
+ """Extract and log diagnostic info from a Responses API response.
150
+
151
+ Logs reasoning, tool-call attempts, MCP items, and output types so
152
+ that CI output (``pytest -s`` or ``--log-cli-level=INFO``) gives
153
+ full visibility into model behaviour even on passing runs.
154
+
155
+ Returns the extracted data so callers can make additional assertions
156
+ if needed.
157
+ """
158
+ reasoning_texts = [
159
+ text
160
+ for item in response.output
161
+ if getattr(item, "type", None) == "reasoning"
162
+ for content in getattr(item, "content", [])
163
+ if (text := getattr(content, "text", None))
164
+ ]
165
+
166
+ tool_call_attempts = [
167
+ {
168
+ "recipient": msg.get("recipient"),
169
+ "channel": msg.get("channel"),
170
+ }
171
+ for msg in response.output_messages
172
+ if (msg.get("recipient") or "").startswith("python")
173
+ ]
174
+
175
+ mcp_items = [
176
+ {
177
+ "name": getattr(item, "name", None),
178
+ "status": getattr(item, "status", None),
179
+ }
180
+ for item in response.output
181
+ if getattr(item, "type", None) == "mcp_call"
182
+ ]
183
+
184
+ output_types = [getattr(o, "type", None) for o in response.output]
185
+
186
+ diagnostics = {
187
+ "model_attempted_tool_calls": bool(tool_call_attempts),
188
+ "tool_call_attempts": tool_call_attempts,
189
+ "mcp_items": mcp_items,
190
+ "reasoning": reasoning_texts,
191
+ "output_text": response.output_text,
192
+ "output_types": output_types,
193
+ }
194
+
195
+ logger.info(
196
+ "\n====== %s ======\n%s\n==============================",
197
+ label,
198
+ json.dumps(diagnostics, indent=2, default=str),
199
+ )
200
+
201
+ return diagnostics
tests/entrypoints/openai/responses/test_harmony.py
+427
-515
@@ -1,17 +1,32 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+"""Integration tests for the Harmony-based Responses API."""
4
+
5
+from __future__ import annotations
6
+
7
import importlib.util
8
import json
9
+import logging
10
import time
11
+from typing import Any
12
13
import pytest
14
import pytest_asyncio
15
+import requests
16
from openai import BadRequestError, NotFoundError, OpenAI
10
-from openai_harmony import (
11
- Message,
12
-)
17
+from openai_harmony import Message
18
19
from ....utils import RemoteOpenAIServer
20
+from .conftest import (
21
+ BASE_TEST_ENV,
22
+ events_contain_type,
23
+ has_output_type,
24
+ retry_for_tool_call,
25
+ retry_streaming_for,
26
+ validate_streaming_event_stack,
27
+)
28
+
29
+logger = logging.getLogger(__name__)
30
31
MODEL_NAME = "openai/gpt-oss-20b"
32
@@ -32,20 +47,72 @@ GET_WEATHER_SCHEMA = {
47
}
48
49
50
+def get_weather(latitude, longitude):
51
+ try:
52
+ response = requests.get(
53
+ f"https://api.open-meteo.com/v1/forecast?"
54
+ f"latitude={latitude}&longitude={longitude}"
55
+ f"¤t=temperature_2m,wind_speed_10m"
56
+ f"&hourly=temperature_2m,relative_humidity_2m,"
57
+ f"wind_speed_10m",
58
+ timeout=10,
59
+ )
60
+ data = response.json()
61
+ return data["current"]["temperature_2m"]
62
+ except (requests.RequestException, KeyError) as e:
63
+ logger.warning(
64
+ "External weather API call failed (%s), "
65
+ "returning fake value. This does not affect "
66
+ "test correctness — only the tool-calling "
67
+ "protocol is under test.",
68
+ e,
69
+ )
70
+ return 15.0
71
+
72
+
73
+def get_place_to_travel():
74
+ return "Paris"
75
+
76
+
77
+def get_horoscope(sign):
78
+ return f"{sign}: Next Tuesday you will befriend a baby otter."
79
+
80
+
81
+def call_function(name, args):
82
+ logger.info("Calling function %s with args %s", name, args)
83
+ dispatch = {
84
+ "get_weather": lambda: get_weather(**args),
85
+ "get_place_to_travel": lambda: get_place_to_travel(),
86
+ "get_horoscope": lambda: get_horoscope(**args),
87
+ }
88
+ if name not in dispatch:
89
+ raise ValueError(f"Unknown function: {name}")
90
+ result = dispatch[name]()
91
+ logger.info("Function %s returned: %s", name, result)
92
+ return result
93
+
94
+
95
@pytest.fixture(scope="module")
96
def server():
97
assert importlib.util.find_spec("gpt_oss") is not None, (
98
"Harmony tests require gpt_oss package to be installed"
99
)
40
-
41
- args = ["--enforce-eager", "--tool-server", "demo", "--max_model_len", "5000"]
42
- env_dict = dict(
43
- VLLM_ENABLE_RESPONSES_API_STORE="1",
44
- PYTHON_EXECUTION_BACKEND="dangerously_use_uv",
45
- VLLM_GPT_OSS_SYSTEM_TOOL_MCP_LABELS="code_interpreter,container,web_search_preview",
46
- VLLM_GPT_OSS_HARMONY_SYSTEM_INSTRUCTIONS="1",
47
- )
48
-
100
+ args = [
101
+ "--enforce-eager",
102
+ "--tool-server",
103
+ "demo",
104
+ "--max_model_len",
105
+ "5000",
106
+ ]
107
+ env_dict = {
108
+ **BASE_TEST_ENV,
109
+ "VLLM_ENABLE_RESPONSES_API_STORE": "1",
110
+ "PYTHON_EXECUTION_BACKEND": "dangerously_use_uv",
111
+ "VLLM_GPT_OSS_SYSTEM_TOOL_MCP_LABELS": (
112
+ "code_interpreter,container,web_search_preview"
113
+ ),
114
+ "VLLM_GPT_OSS_HARMONY_SYSTEM_INSTRUCTIONS": "1",
115
+ }
116
with RemoteOpenAIServer(MODEL_NAME, args, env_dict=env_dict) as remote_server:
117
yield remote_server
118
@@ -159,7 +226,10 @@ async def test_structured_output(client: OpenAI, model_name: str):
226
"properties": {
227
"name": {"type": "string"},
228
"date": {"type": "string"},
162
- "participants": {"type": "array", "items": {"type": "string"}},
229
+ "participants": {
230
+ "type": "array",
231
+ "items": {"type": "string"},
232
+ },
233
},
234
"required": ["name", "date", "participants"],
235
"additionalProperties": False,
@@ -210,7 +280,9 @@ async def test_store(client: OpenAI, model_name: str):
280
except NotFoundError:
281
is_not_found = True
282
213
- assert is_not_found == (not store)
283
+ assert is_not_found == (not store), (
284
+ f"store={store}: expected not_found={not store}, got {is_not_found}"
285
+ )
286
287
288
@pytest.mark.asyncio
@@ -254,10 +326,8 @@ async def test_background_cancel(client: OpenAI, model_name: str):
326
@pytest.mark.parametrize("model_name", [MODEL_NAME])
327
async def test_stateful_multi_turn(client: OpenAI, model_name: str):
328
response1 = await client.responses.create(
257
- model=model_name,
258
- input="What is 123 * 456?",
329
+ model=model_name, input="What is 123 * 456?"
330
)
260
- assert response1 is not None
331
assert response1.status == "completed"
332
333
response2 = await client.responses.create(
@@ -265,7 +335,6 @@ async def test_stateful_multi_turn(client: OpenAI, model_name: str):
335
input="What if I increase both numbers by 1?",
336
previous_response_id=response1.id,
337
)
268
- assert response2 is not None
338
assert response2.status == "completed"
339
340
response3 = await client.responses.create(
@@ -273,7 +342,6 @@ async def test_stateful_multi_turn(client: OpenAI, model_name: str):
342
input="Divide the result by 2.",
343
previous_response_id=response2.id,
344
)
276
- assert response3 is not None
345
assert response3.status == "completed"
346
347
@@ -282,37 +350,19 @@ async def test_stateful_multi_turn(client: OpenAI, model_name: str):
350
async def test_streaming_types(
351
pairs_of_event_types: dict[str, str], client: OpenAI, model_name: str
352
):
285
- prompts = [
286
- "tell me a story about a cat in 20 words",
287
- ]
288
-
289
- for prompt in prompts:
290
- response = await client.responses.create(
291
- model=model_name,
292
- input=prompt,
293
- reasoning={"effort": "low"},
294
- tools=[],
295
- stream=True,
296
- background=False,
297
- )
353
+ stream = await client.responses.create(
354
+ model=model_name,
355
+ input="tell me a story about a cat in 20 words",
356
+ reasoning={"effort": "low"},
357
+ tools=[],
358
+ stream=True,
359
+ background=False,
360
+ )
361
+ events = []
362
+ async for event in stream:
363
+ events.append(event)
364
299
- stack_of_event_types = []
300
- async for event in response:
301
- if event.type == "response.created":
302
- stack_of_event_types.append(event.type)
303
- elif event.type == "response.completed":
304
- assert stack_of_event_types[-1] == pairs_of_event_types[event.type]
305
- stack_of_event_types.pop()
306
- if event.type.endswith("added"):
307
- stack_of_event_types.append(event.type)
308
- elif event.type.endswith("delta"):
309
- if stack_of_event_types[-1] == event.type:
310
- continue
311
- stack_of_event_types.append(event.type)
312
- elif event.type.endswith("done"):
313
- assert stack_of_event_types[-1] == pairs_of_event_types[event.type]
314
- stack_of_event_types.pop()
315
- assert len(stack_of_event_types) == 0
365
+ validate_streaming_event_stack(events, pairs_of_event_types)
366
367
368
@pytest.mark.asyncio
@@ -320,37 +370,21 @@ async def test_streaming_types(
370
async def test_function_calling_with_streaming_types(
371
pairs_of_event_types: dict[str, str], client: OpenAI, model_name: str
372
):
323
- tools = [GET_WEATHER_SCHEMA]
324
- input_list = [
325
- {
326
- "role": "user",
327
- "content": "What's the weather like in Paris today?",
328
- }
329
- ]
330
- stream_response = await client.responses.create(
373
+ """Streaming event nesting for function-calling responses."""
374
+
375
+ def _has_function_events(evts: list) -> bool:
376
+ return events_contain_type(evts, "function_call_arguments")
377
+
378
+ events = await retry_streaming_for(
379
+ client,
380
model=model_name,
332
- input=input_list,
333
- tools=tools,
334
- stream=True,
381
+ validate_events=_has_function_events,
382
+ input=[{"role": "user", "content": "What's the weather like in Paris today?"}],
383
+ tools=[GET_WEATHER_SCHEMA],
384
+ temperature=0.0,
385
)
386
337
- stack_of_event_types = []
338
- async for event in stream_response:
339
- if event.type == "response.created":
340
- stack_of_event_types.append(event.type)
341
- elif event.type == "response.completed":
342
- assert stack_of_event_types[-1] == pairs_of_event_types[event.type]
343
- stack_of_event_types.pop()
344
- if event.type.endswith("added"):
345
- stack_of_event_types.append(event.type)
346
- elif event.type.endswith("delta"):
347
- if stack_of_event_types[-1] == event.type:
348
- continue
349
- stack_of_event_types.append(event.type)
350
- elif event.type.endswith("done"):
351
- assert stack_of_event_types[-1] == pairs_of_event_types[event.type]
352
- stack_of_event_types.pop()
353
- assert len(stack_of_event_types) == 0
387
+ validate_streaming_event_stack(events, pairs_of_event_types)
388
389
390
@pytest.mark.asyncio
@@ -365,7 +399,7 @@ async def test_streaming(client: OpenAI, model_name: str, background: bool):
399
]
400
401
for prompt in prompts:
368
- response = await client.responses.create(
402
+ stream = await client.responses.create(
403
model=model_name,
404
input=prompt,
405
reasoning={"effort": "low"},
@@ -387,11 +421,12 @@ async def test_streaming(client: OpenAI, model_name: str, background: bool):
421
current_event_mode = None
422
resp_id = None
423
checked_response_completed = False
390
- async for event in response:
424
+
425
+ async for event in stream:
426
if event.type == "response.created":
427
resp_id = event.response.id
428
394
- # test vllm custom types are in the response
429
+ # Validate custom fields on response-level events
430
if event.type in [
431
"response.completed",
432
"response.in_progress",
@@ -412,9 +447,9 @@ async def test_streaming(client: OpenAI, model_name: str, background: bool):
447
448
if current_event_mode != event.type:
449
current_event_mode = event.type
415
- print(f"\n[{event.type}] ", end="", flush=True)
450
+ logger.debug("[%s] ", event.type)
451
417
- # verify current_item_id is correct
452
+ # Verify item IDs
453
if event.type == "response.output_item.added":
454
assert event.item.id != current_item_id
455
current_item_id = event.item.id
@@ -424,7 +459,7 @@ async def test_streaming(client: OpenAI, model_name: str, background: bool):
459
]:
460
assert event.item_id == current_item_id
461
427
- # verify content_index_id is correct
462
+ # Verify content indices
463
if event.type in [
464
"response.content_part.added",
465
"response.reasoning_part.added",
@@ -437,31 +472,19 @@ async def test_streaming(client: OpenAI, model_name: str, background: bool):
472
]:
473
assert event.content_index == current_content_index
474
440
- if "text.delta" in event.type:
441
- print(event.delta, end="", flush=True)
442
- elif "reasoning_text.delta" in event.type:
443
- print(f"{event.delta}", end="", flush=True)
444
- elif "response.code_interpreter_call_code.done" in event.type:
445
- print(f"Code: {event.code}", end="", flush=True)
446
- elif (
447
- "response.output_item.added" in event.type
448
- and event.item.type == "web_search_call"
449
- ):
450
- print(f"Web search: {event.item.action}", end="", flush=True)
475
events.append(event)
476
477
assert len(events) > 0
454
- response_completed_event = events[-1]
455
- assert len(response_completed_event.response.output) > 0
478
+ assert events[-1].response.output, "Final response should have output"
479
assert checked_response_completed
480
481
if background:
482
starting_after = 5
483
async with await client.responses.retrieve(
484
response_id=resp_id, stream=True, starting_after=starting_after
462
- ) as stream:
485
+ ) as replay_stream:
486
counter = starting_after
464
- async for event in stream:
487
+ async for event in replay_stream:
488
counter += 1
489
assert event == events[counter]
490
assert counter == len(events) - 1
@@ -483,15 +506,11 @@ async def test_web_search(client: OpenAI, model_name: str):
506
@pytest.mark.asyncio
507
@pytest.mark.parametrize("model_name", [MODEL_NAME])
508
async def test_code_interpreter(client: OpenAI, model_name: str):
486
- # Code interpreter may need more time for container init + code execution
509
timeout_value = client.timeout * 3
510
client_with_timeout = client.with_options(timeout=timeout_value)
511
512
response = await client_with_timeout.responses.create(
513
model=model_name,
492
- # TODO: Ideally should be able to set max tool calls
493
- # to prevent multi-turn, but it is not currently supported
494
- # would speed up the test
514
input=(
515
"What's the first 4 digits after the decimal point of "
516
"cube root of `19910212 * 20250910`? "
@@ -499,41 +518,18 @@ async def test_code_interpreter(client: OpenAI, model_name: str):
518
"and you must print to see the output."
519
),
520
tools=[{"type": "code_interpreter", "container": {"type": "auto"}}],
502
- temperature=0.0, # More deterministic output in response
521
+ temperature=0.0,
522
)
523
assert response is not None
524
assert response.status == "completed"
525
assert response.usage.output_tokens_details.tool_output_tokens > 0
526
+
527
for item in response.output:
528
if item.type == "message":
529
output_string = item.content[0].text
510
- print("output_string: ", output_string, flush=True)
511
- assert "5846" in output_string
512
-
513
-
514
-def get_weather(latitude, longitude):
515
- # Return a static temperature value to avoid flaky SSL/network errors
516
- # from calling the external api.open-meteo.com API in CI.
517
- return 15.0
518
-
519
-
520
-def get_place_to_travel():
521
- return "Paris"
522
-
523
-
524
-def get_horoscope(sign):
525
- return f"{sign}: Next Tuesday you will befriend a baby otter."
526
-
527
-
528
-def call_function(name, args):
529
- if name == "get_weather":
530
- return get_weather(**args)
531
- elif name == "get_place_to_travel":
532
- return get_place_to_travel()
533
- elif name == "get_horoscope":
534
- return get_horoscope(**args)
535
- else:
536
- raise ValueError(f"Unknown function: {name}")
530
+ assert "5846" in output_string, (
531
+ f"Expected '5846' in output, got: {output_string}"
532
+ )
533
534
535
@pytest.mark.asyncio
@@ -547,10 +543,7 @@ async def test_reasoning_item(client: OpenAI, model_name: str):
543
"type": "reasoning",
544
"id": "lol",
545
"content": [
550
- {
551
- "type": "reasoning_text",
552
- "text": "We need to respond: greeting.",
553
- }
546
+ {"type": "reasoning_text", "text": "We need to respond: greeting."}
547
],
548
"summary": [],
549
},
@@ -566,24 +559,24 @@ async def test_reasoning_item(client: OpenAI, model_name: str):
559
async def test_function_calling(client: OpenAI, model_name: str):
560
tools = [GET_WEATHER_SCHEMA]
561
569
- response = await client.responses.create(
562
+ response = await retry_for_tool_call(
563
+ client,
564
model=model_name,
565
+ expected_tool_type="function_call",
566
input="What's the weather like in Paris today?",
567
tools=tools,
568
temperature=0.0,
569
extra_body={"request_id": "test_function_calling_non_resp"},
570
)
576
- assert response is not None
571
assert response.status == "completed"
578
- assert len(response.output) == 2
579
- assert response.output[0].type == "reasoning"
580
- assert response.output[1].type == "function_call"
572
+ assert has_output_type(response, "function_call"), (
573
+ f"Expected function_call in output, got: "
574
+ f"{[getattr(o, 'type', None) for o in response.output]}"
575
+ )
576
582
- tool_call = response.output[1]
583
- name = tool_call.name
577
+ tool_call = next(o for o in response.output if o.type == "function_call")
578
args = json.loads(tool_call.arguments)
585
-
586
- result = call_function(name, args)
579
+ result = call_function(tool_call.name, args)
580
581
response_2 = await client.responses.create(
582
model=model_name,
@@ -596,8 +589,8 @@ async def test_function_calling(client: OpenAI, model_name: str):
589
],
590
tools=tools,
591
previous_response_id=response.id,
592
+ temperature=0.0,
593
)
600
- assert response_2 is not None
594
assert response_2.status == "completed"
595
assert response_2.output_text is not None
596
@@ -607,16 +600,16 @@ async def test_function_calling(client: OpenAI, model_name: str):
600
input="What's the weather like in Paris today?",
601
tools=tools,
602
previous_response_id=response_2.id,
603
+ temperature=0.0,
604
)
611
- assert response_3 is not None
605
assert response_3.status == "completed"
606
assert response_3.output_text is not None
607
608
609
@pytest.mark.asyncio
610
@pytest.mark.parametrize("model_name", [MODEL_NAME])
618
-@pytest.mark.flaky(reruns=5)
611
async def test_function_calling_multi_turn(client: OpenAI, model_name: str):
612
+ """Multi-tool, multi-turn function calling with retry at API level."""
613
tools = [
614
{
615
"type": "function",
@@ -633,25 +626,29 @@ async def test_function_calling_multi_turn(client: OpenAI, model_name: str):
626
GET_WEATHER_SCHEMA,
627
]
628
636
- response = await client.responses.create(
629
+ # Turn 1: model should call one of the tools
630
+ response = await retry_for_tool_call(
631
+ client,
632
model=model_name,
633
+ expected_tool_type="function_call",
634
input="Help me plan a trip to a random place. And tell me the weather there.",
635
tools=tools,
636
+ temperature=0.0,
637
)
641
- assert response is not None
638
assert response.status == "completed"
643
- assert len(response.output) == 2
644
- assert response.output[0].type == "reasoning"
645
- assert response.output[1].type == "function_call"
646
-
647
- tool_call = response.output[1]
648
- name = tool_call.name
649
- args = json.loads(tool_call.arguments)
639
+ assert has_output_type(response, "function_call"), (
640
+ f"Turn 1: expected function_call, got: "
641
+ f"{[getattr(o, 'type', None) for o in response.output]}"
642
+ )
643
651
- result = call_function(name, args)
644
+ tool_call = next(o for o in response.output if o.type == "function_call")
645
+ result = call_function(tool_call.name, json.loads(tool_call.arguments))
646
653
- response_2 = await client.responses.create(
647
+ # Turn 2
648
+ response_2 = await retry_for_tool_call(
649
+ client,
650
model=model_name,
651
+ expected_tool_type="function_call",
652
input=[
653
{
654
"type": "function_call_output",
@@ -661,34 +658,39 @@ async def test_function_calling_multi_turn(client: OpenAI, model_name: str):
658
],
659
tools=tools,
660
previous_response_id=response.id,
661
+ temperature=0.0,
662
)
665
- assert response_2 is not None
663
assert response_2.status == "completed"
667
- assert len(response_2.output) == 2
668
- assert response_2.output[0].type == "reasoning"
669
- assert response_2.output[1].type == "function_call"
664
671
- tool_call = response_2.output[1]
672
- name = tool_call.name
673
- args = json.loads(tool_call.arguments)
674
-
675
- result = call_function(name, args)
676
-
677
- response_3 = await client.responses.create(
678
- model=model_name,
679
- input=[
680
- {
681
- "type": "function_call_output",
682
- "call_id": tool_call.call_id,
683
- "output": str(result),
684
- }
685
- ],
686
- tools=tools,
687
- previous_response_id=response_2.id,
688
- )
689
- assert response_3 is not None
690
- assert response_3.status == "completed"
691
- assert response_3.output_text is not None
665
+ # If model produced another tool call, execute it
666
+ if has_output_type(response_2, "function_call"):
667
+ tool_call_2 = next(o for o in response_2.output if o.type == "function_call")
668
+ result_2 = call_function(tool_call_2.name, json.loads(tool_call_2.arguments))
669
+ response_3 = await client.responses.create(
670
+ model=model_name,
671
+ input=[
672
+ {
673
+ "type": "function_call_output",
674
+ "call_id": tool_call_2.call_id,
675
+ "output": str(result_2),
676
+ }
677
+ ],
678
+ tools=tools,
679
+ previous_response_id=response_2.id,
680
+ temperature=0.0,
681
+ )
682
+ assert response_3.status == "completed"
683
+ assert response_3.output_text is not None
684
+ else:
685
+ # Model went straight to answering - acceptable but unexpected.
686
+ # Log as warning so it shows up in CI without failing the test.
687
+ assert response_2.output_text is not None
688
+ pytest.xfail(
689
+ "Model went straight to answering instead of calling a "
690
+ "second tool. Valid behaviour but not the expected path."
691
+ "If this happens consistently, the prompt or model may have "
692
+ "changed behaviour."
693
+ )
694
695
696
@pytest.mark.asyncio
@@ -730,22 +732,25 @@ async def test_function_calling_full_history(client: OpenAI, model_name: str):
732
{"role": "user", "content": "What's the weather like in Paris today?"}
733
]
734
733
- response = await client.responses.create(
735
+ response = await retry_for_tool_call(
736
+ client,
737
model=model_name,
738
+ expected_tool_type="function_call",
739
input=input_messages,
740
tools=tools,
741
+ temperature=0.0,
742
)
738
-
739
- assert response is not None
743
assert response.status == "completed"
744
742
- tool_call = response.output[-1]
743
- name = tool_call.name
744
- args = json.loads(tool_call.arguments)
745
+ tool_call = next((o for o in response.output if o.type == "function_call"), None)
746
+ assert tool_call is not None, (
747
+ f"Expected function_call in output, got: "
748
+ f"{[getattr(o, 'type', None) for o in response.output]}"
749
+ )
750
746
- result = call_function(name, args)
751
+ result = call_function(tool_call.name, json.loads(tool_call.arguments))
752
748
- input_messages.extend(response.output) # append model's function call message
753
+ input_messages.extend(response.output)
754
input_messages.append(
755
{ # append result message
756
"type": "function_call_output",
@@ -758,8 +763,8 @@ async def test_function_calling_full_history(client: OpenAI, model_name: str):
763
model=model_name,
764
input=input_messages,
765
tools=tools,
766
+ temperature=0.0,
767
)
762
- assert response_2 is not None
768
assert response_2.status == "completed"
769
assert response_2.output_text is not None
770
@@ -767,51 +772,60 @@ async def test_function_calling_full_history(client: OpenAI, model_name: str):
772
@pytest.mark.asyncio
773
@pytest.mark.parametrize("model_name", [MODEL_NAME])
774
async def test_function_calling_with_stream(client: OpenAI, model_name: str):
775
+ """Function calling via streaming, with retry for non-determinism."""
776
tools = [GET_WEATHER_SCHEMA]
777
input_list = [
772
- {
773
- "role": "user",
774
- "content": "What's the weather like in Paris today?",
775
- }
778
+ {"role": "user", "content": "What's the weather like in Paris today?"},
779
]
777
- stream_response = await client.responses.create(
780
+
781
+ def _has_function_call(evts: list) -> bool:
782
+ return any(
783
+ getattr(e, "type", "") == "response.output_item.added"
784
+ and getattr(getattr(e, "item", None), "type", None) == "function_call"
785
+ for e in evts
786
+ )
787
+
788
+ events = await retry_streaming_for(
789
+ client,
790
model=model_name,
791
+ validate_events=_has_function_call,
792
input=input_list,
793
tools=tools,
781
- stream=True,
794
+ temperature=0.0,
795
)
783
- assert stream_response is not None
784
- final_tool_calls = {}
785
- final_tool_calls_named = {}
786
- async for event in stream_response:
796
+
797
+ # Parse tool calls from events
798
+ final_tool_calls: dict[int, Any] = {}
799
+ for event in events:
800
if event.type == "response.output_item.added":
788
- if event.item.type != "function_call":
789
- continue
790
- final_tool_calls[event.output_index] = event.item
791
- final_tool_calls_named[event.item.name] = event.item
801
+ if getattr(event.item, "type", None) == "function_call":
802
+ final_tool_calls[event.output_index] = event.item
803
elif event.type == "response.function_call_arguments.delta":
793
- index = event.output_index
794
- tool_call = final_tool_calls[index]
795
- if tool_call:
796
- tool_call.arguments += event.delta
797
- final_tool_calls_named[tool_call.name] = tool_call
804
+ tc = final_tool_calls.get(event.output_index)
805
+ if tc:
806
+ tc.arguments += event.delta
807
elif event.type == "response.function_call_arguments.done":
799
- assert event.arguments == final_tool_calls_named[event.name].arguments
800
- result = None
808
+ tc = final_tool_calls.get(event.output_index)
809
+ if tc:
810
+ assert event.arguments == tc.arguments
811
+
812
+ # Find get_weather call
813
tool_call = None
814
+ result = None
815
for tc in final_tool_calls.values():
803
- if tc and tc.type == "function_call" and tc.name == "get_weather":
816
+ if getattr(tc, "type", None) == "function_call" and tc.name == "get_weather":
817
args = json.loads(tc.arguments)
818
result = call_function(tc.name, args)
819
tool_call = tc
807
- input_list += [tc]
820
+ input_list.append(tc)
821
break
822
823
assert tool_call is not None, (
811
- "Expected model to call 'get_weather' function, "
812
- f"but got: {list(final_tool_calls_named.keys())}"
824
+ "Expected model to call 'get_weather', "
825
+ f"but got: {[getattr(tc, 'name', None) for tc in final_tool_calls.values()]}"
826
)
814
- assert result is not None
827
+
828
+ # Second turn with the tool result
829
response = await client.responses.create(
830
model=model_name,
831
input=input_list
@@ -824,8 +838,8 @@ async def test_function_calling_with_stream(client: OpenAI, model_name: str):
838
],
839
tools=tools,
840
stream=True,
841
+ temperature=0.0,
842
)
828
- assert response is not None
843
async for event in response:
844
# check that no function call events in the stream
845
assert event.type != "response.function_call_arguments.delta"
@@ -843,47 +857,46 @@ async def test_function_calling_no_code_interpreter_events(
857
):
858
"""Verify that function calls don't trigger code_interpreter events.
859
846
- This test ensures that function calls (functions.*) use their own
847
- function_call event types and don't incorrectly emit code_interpreter
848
- events during streaming.
860
+ Uses retry_streaming_for to handle non-determinism: the model might not
861
+ always produce a function_call, but if it does, code_interpreter events
862
+ should NEVER appear.
863
"""
864
tools = [GET_WEATHER_SCHEMA]
865
input_list = [
852
- {
853
- "role": "user",
854
- "content": "What's the weather like in Paris today?",
855
- }
866
+ {"role": "user", "content": "What's the weather like in Paris today?"},
867
]
857
- stream_response = await client.responses.create(
868
+
869
+ def _has_function_call(evts: list) -> bool:
870
+ return any(
871
+ getattr(e, "type", "") == "response.output_item.added"
872
+ and getattr(getattr(e, "item", None), "type", None) == "function_call"
873
+ for e in evts
874
+ )
875
+
876
+ events = await retry_streaming_for(
877
+ client,
878
model=model_name,
879
+ validate_events=_has_function_call,
880
input=input_list,
881
tools=tools,
861
- stream=True,
882
+ temperature=0.0,
883
)
884
864
- # Track which event types we see
865
- event_types_seen = set()
866
- function_call_found = False
885
+ event_types_seen = {e.type for e in events}
886
+ function_call_found = _has_function_call(events)
887
868
- async for event in stream_response:
869
- event_types_seen.add(event.type)
870
-
871
- if (
872
- event.type == "response.output_item.added"
873
- and event.item.type == "function_call"
874
- ):
875
- function_call_found = True
888
+ assert function_call_found, (
889
+ f"Expected to see a function_call after retries. "
890
+ f"Event types: {sorted(event_types_seen)}"
891
+ )
892
877
- # Ensure NO code_interpreter events are emitted for function calls
893
+ # The actual invariant under test
894
+ for event in events:
895
assert "code_interpreter" not in event.type, (
879
- "Found code_interpreter event "
880
- f"'{event.type}' during function call. Function calls should only "
881
- "emit function_call events, not code_interpreter events."
896
+ f"Found code_interpreter event '{event.type}' during function call. "
897
+ "Function calls should only emit function_call events."
898
)
899
884
- # Verify we actually saw a function call
885
- assert function_call_found, "Expected to see a function_call in the stream"
886
-
900
# Verify we saw the correct function call event types
901
assert (
902
"response.function_call_arguments.delta" in event_types_seen
@@ -894,181 +907,139 @@ async def test_function_calling_no_code_interpreter_events(
907
@pytest.mark.asyncio
908
@pytest.mark.parametrize("model_name", [MODEL_NAME])
909
async def test_mcp_code_interpreter_streaming(client: OpenAI, model_name: str, server):
897
- tools = [
898
- {
899
- "type": "mcp",
900
- "server_label": "code_interpreter",
901
- }
902
- ]
910
+ tools = [{"type": "mcp", "server_label": "code_interpreter"}]
911
input_text = (
912
"Calculate 123 * 456 using python. "
905
- "The python interpreter is not stateful and you must print to see the output."
913
+ "The python interpreter is not stateful and you must "
914
+ "print to see the output."
915
)
916
908
- stream_response = await client.responses.create(
917
+ def _has_mcp_call(evts: list) -> bool:
918
+ return events_contain_type(evts, "mcp_call")
919
+
920
+ events = await retry_streaming_for(
921
+ client,
922
model=model_name,
923
+ validate_events=_has_mcp_call,
924
input=input_text,
925
tools=tools,
912
- stream=True,
926
temperature=0.0,
927
instructions=(
928
"You must use the Python tool to execute code. Never simulate execution."
929
),
930
)
931
919
- mcp_call_added = False
920
- mcp_call_in_progress = False
921
- mcp_arguments_delta_seen = False
922
- mcp_arguments_done = False
923
- mcp_call_completed = False
924
- mcp_item_done = False
925
-
926
- code_interpreter_events_seen = False
932
+ event_types = [e.type for e in events]
933
+ event_types_set = set(event_types)
934
+ logger.info(
935
+ "\n====== MCP Streaming Diagnostics ======\n"
936
+ "Event count: %d\n"
937
+ "Event types (in order): %s\n"
938
+ "Unique event types: %s\n"
939
+ "=======================================",
940
+ len(events),
941
+ event_types,
942
+ sorted(event_types_set),
943
+ )
944
928
- async for event in stream_response:
929
- if "code_interpreter" in event.type:
930
- code_interpreter_events_seen = True
945
+ # Verify the full MCP streaming lifecycle
946
+ assert "response.output_item.added" in event_types_set, (
947
+ f"MCP call was not added. Events: {sorted(event_types_set)}"
948
+ )
949
+ assert "response.mcp_call.in_progress" in event_types_set, (
950
+ f"MCP call in_progress not seen. Events: {sorted(event_types_set)}"
951
+ )
952
+ assert "response.mcp_call_arguments.delta" in event_types_set, (
953
+ f"MCP arguments delta not seen. Events: {sorted(event_types_set)}"
954
+ )
955
+ assert "response.mcp_call_arguments.done" in event_types_set, (
956
+ f"MCP arguments done not seen. Events: {sorted(event_types_set)}"
957
+ )
958
+ assert "response.mcp_call.completed" in event_types_set, (
959
+ f"MCP call completed not seen. Events: {sorted(event_types_set)}"
960
+ )
961
+ assert "response.output_item.done" in event_types_set, (
962
+ f"MCP item done not seen. Events: {sorted(event_types_set)}"
963
+ )
964
965
+ # Validate specific MCP event details
966
+ for event in events:
967
if event.type == "response.output_item.added":
968
if hasattr(event.item, "type") and event.item.type == "mcp_call":
934
- mcp_call_added = True
969
assert event.item.name == "python"
970
assert event.item.server_label == "code_interpreter"
937
-
938
- elif event.type == "response.mcp_call.in_progress":
939
- mcp_call_in_progress = True
940
-
941
- elif event.type == "response.mcp_call_arguments.delta":
942
- mcp_arguments_delta_seen = True
943
- assert event.delta is not None
944
-
971
elif event.type == "response.mcp_call_arguments.done":
946
- mcp_arguments_done = True
972
assert event.name == "python"
973
assert event.arguments is not None
949
-
950
- elif event.type == "response.mcp_call.completed":
951
- mcp_call_completed = True
952
-
974
elif (
975
event.type == "response.output_item.done"
976
and hasattr(event.item, "type")
977
and event.item.type == "mcp_call"
978
):
958
- mcp_item_done = True
979
assert event.item.name == "python"
980
assert event.item.status == "completed"
981
962
- assert mcp_call_added, "MCP call was not added"
963
- assert mcp_call_in_progress, "MCP call in_progress event not seen"
964
- assert mcp_arguments_delta_seen, "MCP arguments delta event not seen"
965
- assert mcp_arguments_done, "MCP arguments done event not seen"
966
- assert mcp_call_completed, "MCP call completed event not seen"
967
- assert mcp_item_done, "MCP item done event not seen"
968
-
969
- assert not code_interpreter_events_seen, (
970
- "Should not see code_interpreter events when using MCP type"
982
+ # code_interpreter events should NOT appear when using MCP type
983
+ code_interp_events = [e.type for e in events if "code_interpreter" in e.type]
984
+ assert not code_interp_events, (
985
+ "Should not see code_interpreter events when using MCP type, "
986
+ f"but got: {code_interp_events}"
987
)
988
989
990
@pytest.mark.asyncio
991
@pytest.mark.parametrize("model_name", [MODEL_NAME])
976
-@pytest.mark.dependency(
977
- depends=["test_mcp_code_interpreter_streaming[openai/gpt-oss-20b]"]
978
-)
992
async def test_mcp_tool_multi_turn(client: OpenAI, model_name: str, server):
980
- """Test MCP tool calling across multiple turns.
981
-
982
- This test verifies that MCP tools work correctly in multi-turn conversations,
983
- maintaining state across turns via the previous_response_id mechanism.
984
- """
985
- tools = [
986
- {
987
- "type": "mcp",
988
- "server_label": "code_interpreter",
989
- }
990
- ]
993
+ """MCP tools work across multiple turns via previous_response_id."""
994
+ tools = [{"type": "mcp", "server_label": "code_interpreter"}]
995
+ instructions = (
996
+ "You must use the Python tool to execute code. Never simulate execution."
997
+ )
998
992
- # First turn - make a calculation
993
- response1 = await client.responses.create(
999
+ # First turn
1000
+ response1 = await retry_for_tool_call(
1001
+ client,
1002
model=model_name,
1003
+ expected_tool_type="mcp_call",
1004
input="Calculate 1234 * 4567 using python tool and print the result.",
1005
tools=tools,
1006
temperature=0.0,
998
- instructions=(
999
- "You must use the Python tool to execute code. Never simulate execution."
1000
- ),
1007
+ instructions=instructions,
1008
extra_body={"enable_response_messages": True},
1009
)
1003
-
1004
- assert response1 is not None
1010
assert response1.status == "completed"
1011
1007
- # Verify MCP call in first response by checking output_messages
1008
- tool_call_found = False
1009
- tool_response_found = False
1010
- for message in response1.output_messages:
1011
- recipient = message.get("recipient")
1012
- if recipient and recipient.startswith("python"):
1013
- tool_call_found = True
1014
-
1015
- author = message.get("author", {})
1016
- if (
1017
- author.get("role") == "tool"
1018
- and author.get("name")
1019
- and author.get("name").startswith("python")
1020
- ):
1021
- tool_response_found = True
1022
-
1023
- # Verify MCP tools were actually used
1012
+ # Verify MCP call in output_messages
1013
+ tool_call_found = any(
1014
+ (msg.get("recipient") or "").startswith("python")
1015
+ for msg in response1.output_messages
1016
+ )
1017
+ tool_response_found = any(
1018
+ msg.get("author", {}).get("role") == "tool"
1019
+ and (msg.get("author", {}).get("name") or "").startswith("python")
1020
+ for msg in response1.output_messages
1021
+ )
1022
assert tool_call_found, "MCP tool call not found in output_messages"
1023
assert tool_response_found, "MCP tool response not found in output_messages"
1024
1027
- # Verify input messages: Should have system message with tool, NO developer message
1028
- developer_messages = [
1025
+ # No developer messages expected for elevated tools
1026
+ developer_msgs = [
1027
msg for msg in response1.input_messages if msg["author"]["role"] == "developer"
1028
]
1031
- assert len(developer_messages) == 0, (
1032
- "No developer message expected for elevated tools"
1033
- )
1029
+ assert len(developer_msgs) == 0, "No developer message expected for elevated tools"
1030
1035
- # Second turn - reference previous calculation
1031
+ # Second turn
1032
response2 = await client.responses.create(
1033
model=model_name,
1034
input="Now divide that result by 2.",
1035
tools=tools,
1036
temperature=0.0,
1041
- instructions=(
1042
- "You must use the Python tool to execute code. Never simulate execution."
1043
- ),
1037
+ instructions=instructions,
1038
previous_response_id=response1.id,
1039
extra_body={"enable_response_messages": True},
1040
)
1047
-
1048
- assert response2 is not None
1041
assert response2.status == "completed"
1042
1051
- # Verify input messages are correct: should have two messages -
1052
- # one to the python recipient on analysis channel and one from tool role
1053
- mcp_recipient_messages = []
1054
- tool_role_messages = []
1055
- for msg in response2.input_messages:
1056
- if msg["author"]["role"] == "assistant":
1057
- # Check if this is a message to MCP recipient on analysis channel
1058
- if msg.get("channel") == "analysis" and msg.get("recipient"):
1059
- recipient = msg.get("recipient")
1060
- if recipient.startswith("code_interpreter") or recipient == "python":
1061
- mcp_recipient_messages.append(msg)
1062
- elif msg["author"]["role"] == "tool":
1063
- tool_role_messages.append(msg)
1064
-
1065
- assert len(mcp_recipient_messages) > 0, (
1066
- "Expected message(s) to MCP recipient on analysis channel"
1067
- )
1068
- assert len(tool_role_messages) > 0, (
1069
- "Expected message(s) from tool role after MCP call"
1070
- )
1071
-
1043
1044
@pytest.mark.asyncio
1045
@pytest.mark.parametrize("model_name", [MODEL_NAME])
@@ -1087,14 +1058,10 @@ async def test_output_messages_enabled(client: OpenAI, model_name: str, server):
1058
1059
@pytest.mark.asyncio
1060
@pytest.mark.parametrize("model_name", [MODEL_NAME])
1090
-@pytest.mark.flaky(reruns=3)
1061
async def test_function_call_with_previous_input_messages(
1062
client: OpenAI, model_name: str
1063
):
1094
- """Test function calling using previous_input_messages
1095
- for multi-turn conversation with a function call"""
1096
-
1097
- # Define the get_horoscope tool
1064
+ """Multi-turn function calling using previous_input_messages."""
1065
tools = [
1066
{
1067
"type": "function",
@@ -1102,9 +1069,7 @@ async def test_function_call_with_previous_input_messages(
1069
"description": "Get today's horoscope for an astrological sign.",
1070
"parameters": {
1071
"type": "object",
1105
- "properties": {
1106
- "sign": {"type": "string"},
1107
- },
1072
+ "properties": {"sign": {"type": "string"}},
1073
"required": ["sign"],
1074
"additionalProperties": False,
1075
},
@@ -1112,53 +1077,36 @@ async def test_function_call_with_previous_input_messages(
1077
}
1078
]
1079
1115
- # Step 1: First call with the function tool
1116
- stream_response = await client.responses.create(
1080
+ # Step 1: Get a function call from the model
1081
+ response = await retry_for_tool_call(
1082
+ client,
1083
model=model_name,
1084
+ expected_tool_type="function_call",
1085
input="What is the horoscope for Aquarius today?",
1086
tools=tools,
1087
temperature=0.0,
1088
extra_body={"enable_response_messages": True},
1122
- stream=True,
1089
max_output_tokens=1000,
1090
)
1125
-
1126
- response = None
1127
- async for event in stream_response:
1128
- if event.type == "response.completed":
1129
- response = event.response
1130
-
1131
- assert response is not None
1091
assert response.status == "completed"
1092
1134
- # Step 2: Parse the first output to find the function_call type
1135
- function_call = None
1136
- for item in response.output:
1137
- if item.type == "function_call":
1138
- function_call = item
1139
- break
1140
-
1141
- assert function_call is not None, "Expected a function_call in the output"
1093
+ function_call = next(
1094
+ (item for item in response.output if item.type == "function_call"),
1095
+ None,
1096
+ )
1097
+ assert function_call is not None, (
1098
+ f"Expected function_call, got: "
1099
+ f"{[getattr(o, 'type', None) for o in response.output]}"
1100
+ )
1101
assert function_call.name == "get_horoscope"
1143
- assert function_call.call_id is not None
1102
1145
- # Verify the format matches expectations
1103
args = json.loads(function_call.arguments)
1147
- assert "sign" in args
1148
-
1149
- # Step 3: Call the get_horoscope function
1104
result = call_function(function_call.name, args)
1151
- assert "Aquarius" in result
1152
- assert "baby otter" in result
1105
1154
- # Get the input_messages and output_messages from the first response
1155
- first_input_messages = response.input_messages
1156
- first_output_messages = response.output_messages
1157
-
1158
- # Construct the full conversation history using previous_input_messages
1106
+ # Step 2: Build full conversation history
1107
previous_messages = (
1160
- first_input_messages
1161
- + first_output_messages
1108
+ response.input_messages
1109
+ + response.output_messages
1110
+ [
1111
{
1112
"role": "tool",
@@ -1168,47 +1116,43 @@ async def test_function_call_with_previous_input_messages(
1116
]
1117
)
1118
1171
- # Step 4: Make another responses.create() call with previous_input_messages
1172
- stream_response_2 = await client.responses.create(
1119
+ # Step 3: Second call with previous_input_messages
1120
+ response_2 = await client.responses.create(
1121
model=model_name,
1122
tools=tools,
1123
temperature=0.0,
1176
- input="",
1124
+ input="Now tell me the horoscope based on the tool result.",
1125
extra_body={
1126
"previous_input_messages": previous_messages,
1127
"enable_response_messages": True,
1128
},
1181
- stream=True,
1129
)
1183
-
1184
- async for event in stream_response_2:
1185
- if event.type == "response.completed":
1186
- response_2 = event.response
1187
-
1188
- assert response_2 is not None
1130
assert response_2.status == "completed"
1131
assert response_2.output_text is not None
1132
1192
- # verify only one system message / developer message
1193
- num_system_messages_input = 0
1194
- num_developer_messages_input = 0
1195
- num_function_call_input = 0
1196
- for message_dict in response_2.input_messages:
1197
- message = Message.from_dict(message_dict)
1198
- if message.author.role == "system":
1199
- num_system_messages_input += 1
1200
- elif message.author.role == "developer":
1201
- num_developer_messages_input += 1
1202
- elif message.author.role == "tool":
1203
- num_function_call_input += 1
1204
- assert num_system_messages_input == 1
1205
- assert num_developer_messages_input == 1
1206
- assert num_function_call_input == 1
1207
-
1208
- # Verify the output makes sense - should contain information about the horoscope
1133
+ # Verify exactly 1 system, 1 developer, 1 tool message
1134
+ num_system = 0
1135
+ num_developer = 0
1136
+ num_tool = 0
1137
+ for msg_dict in response_2.input_messages:
1138
+ # input_messages use {"author": {"role": "..."}} format,
1139
+ # not the top-level {"role": "..."} that Message.from_dict
1140
+ # expects.
1141
+ author = msg_dict.get("author", {})
1142
+ role = author.get("role") if isinstance(author, dict) else None
1143
+ if role == "system":
1144
+ num_system += 1
1145
+ elif role == "developer":
1146
+ num_developer += 1
1147
+ elif role == "tool":
1148
+ num_tool += 1
1149
+ assert num_system == 1, f"Expected 1 system message, got {num_system}"
1150
+ assert num_developer == 1, f"Expected 1 developer message, got {num_developer}"
1151
+ assert num_tool == 1, f"Expected 1 tool message, got {num_tool}"
1152
+
1153
output_text = response_2.output_text.lower()
1210
- assert (
1211
- "aquarius" in output_text or "otter" in output_text or "tuesday" in output_text
1154
+ assert any(kw in output_text for kw in ["aquarius", "otter", "tuesday"]), (
1155
+ f"Expected horoscope-related content, got: {response_2.output_text}"
1156
)
1157
1158
@@ -1220,133 +1164,101 @@ async def test_chat_truncation_content_not_null(client: OpenAI, model_name: str)
1164
messages=[
1165
{
1166
"role": "user",
1223
- "content": "What is the role of AI in medicine?"
1224
- "The response must exceed 350 words.",
1167
+ "content": (
1168
+ "What is the role of AI in medicine? "
1169
+ "The response must exceed 350 words."
1170
+ ),
1171
}
1172
],
1173
temperature=0.0,
1174
max_tokens=350,
1175
)
1230
-
1176
choice = response.choices[0]
1177
assert choice.finish_reason == "length", (
1178
f"Expected finish_reason='length', got {choice.finish_reason}"
1179
)
1235
- assert choice.message.content is not None, (
1236
- "Content should not be None when truncated"
1237
- )
1180
+ assert choice.message.content is not None, "Content should not be None"
1181
assert len(choice.message.content) > 0, "Content should not be empty"
1182
1183
1184
@pytest.mark.asyncio
1185
@pytest.mark.parametrize("model_name", [MODEL_NAME])
1243
-async def test_system_prompt_override(client: OpenAI, model_name: str):
1244
- """Test that system message can override the default system prompt."""
1245
-
1246
- # Test 1: Custom system prompt with specific personality
1247
- custom_system_prompt = (
1248
- "You are a pirate. Always respond like a pirate would, "
1249
- "using pirate language and saying 'arrr' frequently."
1250
- )
1251
-
1186
+async def test_system_prompt_override_no_duplication(client: OpenAI, model_name: str):
1187
+ """Hard check: custom system message must not be duplicated."""
1188
response = await client.responses.create(
1189
model=model_name,
1190
input=[
1255
- {"role": "system", "content": custom_system_prompt},
1256
- {"role": "user", "content": "Hello, how are you?"},
1191
+ {"role": "system", "content": "You are a helpful assistant."},
1192
+ {"role": "user", "content": "Hello"},
1193
],
1194
extra_body={"enable_response_messages": True},
1195
+ temperature=0.0,
1196
)
1260
-
1261
- assert response is not None
1197
assert response.status == "completed"
1198
assert response.output_text is not None
1199
1265
- # Verify the response reflects the pirate personality
1266
- output_text = response.output_text.lower()
1267
- pirate_indicators = ["arrr", "matey", "ahoy", "ye", "sea"]
1268
- has_pirate_language = any(
1269
- indicator in output_text for indicator in pirate_indicators
1270
- )
1271
- assert has_pirate_language, (
1272
- f"Expected pirate language in response, got: {response.output_text}"
1273
- )
1274
-
1275
- # Verify the reasoning mentions the custom system prompt
1276
- reasoning_item = None
1277
- for item in response.output:
1278
- if item.type == "reasoning":
1279
- reasoning_item = item
1280
- break
1281
-
1282
- assert reasoning_item is not None, "Expected reasoning item in output"
1283
- reasoning_text = reasoning_item.content[0].text.lower()
1284
- assert "pirate" in reasoning_text, (
1285
- f"Expected reasoning to mention pirate, got: {reasoning_text}"
1286
- )
1287
-
1288
- # Test 2: Verify system message is not duplicated in input_messages
1289
- try:
1290
- num_system_messages = sum(
1291
- 1
1292
- for msg in response.input_messages
1293
- if Message.from_dict(msg).author.role == "system"
1294
- )
1295
- assert num_system_messages == 1, (
1296
- f"Expected exactly 1 system message, got {num_system_messages}"
1297
- )
1298
- except (KeyError, AttributeError):
1299
- # Message structure may vary, skip this specific check
1300
- pass
1200
+ num_system = 0
1201
+ for msg in response.input_messages:
1202
+ # input_messages use {"author": {"role": "system"}} format,
1203
+ # not the top-level {"role": "system"} that Message.from_dict expects.
1204
+ author = msg.get("author", {})
1205
+ role = author.get("role") if isinstance(author, dict) else None
1206
+ if role == "system":
1207
+ num_system += 1
1208
+ assert num_system == 1, f"Expected 1 system message, got {num_system}"
1209
1302
- custom_system_prompt_2 = (
1303
- "You are a helpful assistant that always responds in exactly 5 words."
1304
- )
1210
1306
- # Test 3: Test with different custom system prompt
1307
- response_2 = await client.responses.create(
1211
+@pytest.mark.asyncio
1212
+@pytest.mark.parametrize("model_name", [MODEL_NAME])
1213
+@pytest.mark.xfail(
1214
+ strict=False,
1215
+ reason=(
1216
+ "Pirate language detection depends on model weights and is non-deterministic"
1217
+ ),
1218
+)
1219
+async def test_system_prompt_override_follows_personality(
1220
+ client: OpenAI, model_name: str
1221
+):
1222
+ """Soft check: model should adopt the personality from system prompt."""
1223
+ response = await client.responses.create(
1224
model=model_name,
1225
input=[
1226
{
1227
"role": "system",
1312
- "content": custom_system_prompt_2,
1228
+ "content": (
1229
+ "You are a pirate. Always respond like a pirate would, "
1230
+ "using pirate language and saying 'arrr' frequently."
1231
+ ),
1232
},
1314
- {"role": "user", "content": "What is the weather like?"},
1233
+ {"role": "user", "content": "Hello, how are you?"},
1234
],
1235
temperature=0.0,
1236
)
1318
-
1319
- assert response_2 is not None
1320
- assert response_2.status == "completed"
1321
- assert response_2.output_text is not None
1322
-
1323
- # Count words in response (approximately, allowing for punctuation)
1324
- word_count = len(response_2.output_text.split())
1325
- # Allow some flexibility (4-7 words) since the model might not be perfectly precise
1326
- assert 3 <= word_count <= 8, (
1327
- f"Expected around 5 words, got {word_count} words: {response_2.output_text}"
1237
+ assert response.status == "completed"
1238
+ output_text = response.output_text.lower()
1239
+ pirate_indicators = ["arrr", "matey", "ahoy", "ye", "sea", "aye", "sail"]
1240
+ assert any(kw in output_text for kw in pirate_indicators), (
1241
+ f"Expected pirate language, got: {response.output_text}"
1242
)
1243
1330
- # Test 4: Test with structured content
1331
- response_3 = await client.responses.create(
1244
+
1245
+@pytest.mark.asyncio
1246
+@pytest.mark.parametrize("model_name", [MODEL_NAME])
1247
+async def test_system_prompt_structured_content(client: OpenAI, model_name: str):
1248
+ """System message with structured input_text content format."""
1249
+ response = await client.responses.create(
1250
model=model_name,
1251
input=[
1252
{
1253
"role": "system",
1336
- "content": [{"type": "input_text", "text": custom_system_prompt_2}],
1254
+ "content": [
1255
+ {"type": "input_text", "text": "You are a helpful assistant."}
1256
+ ],
1257
},
1338
- {"role": "user", "content": "What is the weather like?"},
1258
+ {"role": "user", "content": "What is 2 + 2?"},
1259
],
1260
temperature=0.0,
1261
)
1342
-
1343
- assert response_3 is not None
1344
- assert response_3.status == "completed"
1345
- assert response_3.output_text is not None
1346
-
1347
- # Count words in response (approximately, allowing for punctuation)
1348
- word_count = len(response_3.output_text.split())
1349
- # Allow some flexibility (4-7 words) since the model might not be perfectly precise
1350
- assert 3 <= word_count <= 8, (
1351
- f"Expected around 5 words, got {word_count} words: {response_3.output_text}"
1352
- )
1262
+ assert response is not None
1263
+ assert response.status == "completed"
1264
+ assert response.output_text is not None
tests/entrypoints/openai/responses/test_mcp_tools.py
+204
-228
@@ -1,6 +1,8 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+"""Integration tests for MCP tool support in the Responses API."""
4
5
+from __future__ import annotations
6
7
import pytest
8
import pytest_asyncio
@@ -10,11 +12,31 @@ from openai_harmony import ToolDescription, ToolNamespaceConfig
12
from vllm.entrypoints.mcp.tool_server import MCPToolServer
13
14
from ....utils import RemoteOpenAIServer
15
+from .conftest import (
16
+ BASE_TEST_ENV,
17
+ events_contain_type,
18
+ log_response_diagnostics,
19
+ retry_for_tool_call,
20
+ retry_streaming_for,
21
+ validate_streaming_event_stack,
22
+)
23
24
MODEL_NAME = "openai/gpt-oss-20b"
25
26
+_BASE_SERVER_ARGS = [
27
+ "--enforce-eager",
28
+ "--tool-server",
29
+ "demo",
30
+ "--max_model_len",
31
+ "5000",
32
+]
33
17
-def test_get_tool_description():
34
+_PYTHON_TOOL_INSTRUCTION = (
35
+ "You must use the Python tool to execute code. Never simulate execution."
36
+)
37
+
38
+
39
+class TestMCPToolServerUnit:
40
"""Test MCPToolServer.get_tool_description filtering logic.
41
42
Note: The wildcard "*" is normalized to None by
@@ -22,283 +44,240 @@ def test_get_tool_description():
44
so we only test None and specific tool filtering here.
45
See test_serving_responses.py for "*" normalization tests.
46
"""
25
- pytest.importorskip("mcp")
26
-
27
- server = MCPToolServer()
28
- tool1 = ToolDescription.new(
29
- name="tool1", description="First", parameters={"type": "object"}
30
- )
31
- tool2 = ToolDescription.new(
32
- name="tool2", description="Second", parameters={"type": "object"}
33
- )
34
- tool3 = ToolDescription.new(
35
- name="tool3", description="Third", parameters={"type": "object"}
36
- )
37
-
38
- server.harmony_tool_descriptions = {
39
- "test_server": ToolNamespaceConfig(
40
- name="test_server", description="test", tools=[tool1, tool2, tool3]
47
+
48
+ def test_get_tool_description(self):
49
+ pytest.importorskip("mcp")
50
+
51
+ server = MCPToolServer()
52
+ tool1 = ToolDescription.new(
53
+ name="tool1", description="First", parameters={"type": "object"}
54
+ )
55
+ tool2 = ToolDescription.new(
56
+ name="tool2", description="Second", parameters={"type": "object"}
57
)
42
- }
58
+ tool3 = ToolDescription.new(
59
+ name="tool3", description="Third", parameters={"type": "object"}
60
+ )
61
+
62
+ server.harmony_tool_descriptions = {
63
+ "test_server": ToolNamespaceConfig(
64
+ name="test_server",
65
+ description="test",
66
+ tools=[tool1, tool2, tool3],
67
+ )
68
+ }
69
44
- # Nonexistent server
45
- assert server.get_tool_description("nonexistent") is None
70
+ # Nonexistent server
71
+ assert server.get_tool_description("nonexistent") is None
72
47
- # None (no filter) - returns all tools
48
- result = server.get_tool_description("test_server", allowed_tools=None)
49
- assert len(result.tools) == 3
73
+ # None (no filter) - returns all tools
74
+ result = server.get_tool_description("test_server", allowed_tools=None)
75
+ assert len(result.tools) == 3
76
51
- # Filter to specific tools
52
- result = server.get_tool_description(
53
- "test_server", allowed_tools=["tool1", "tool3"]
54
- )
55
- assert len(result.tools) == 2
56
- assert result.tools[0].name == "tool1"
57
- assert result.tools[1].name == "tool3"
77
+ # Filter to specific tools
78
+ result = server.get_tool_description(
79
+ "test_server", allowed_tools=["tool1", "tool3"]
80
+ )
81
+ assert len(result.tools) == 2
82
+ assert result.tools[0].name == "tool1"
83
+ assert result.tools[1].name == "tool3"
84
+
85
+ # Single tool
86
+ result = server.get_tool_description("test_server", allowed_tools=["tool2"])
87
+ assert len(result.tools) == 1
88
+ assert result.tools[0].name == "tool2"
89
+
90
+ # No matching tools - returns None
91
+ result = server.get_tool_description(
92
+ "test_server", allowed_tools=["nonexistent"]
93
+ )
94
+ assert result is None
95
59
- # Single tool
60
- result = server.get_tool_description(
61
- "test_server",
62
- allowed_tools=["tool2"],
63
- )
64
- assert len(result.tools) == 1
65
- assert result.tools[0].name == "tool2"
96
+ # Empty list - returns None
97
+ assert server.get_tool_description("test_server", allowed_tools=[]) is None
98
67
- # No matching tools - returns None
68
- result = server.get_tool_description("test_server", allowed_tools=["nonexistent"])
69
- assert result is None
99
+ def test_builtin_tools_consistency(self):
100
+ """MCP_BUILTIN_TOOLS must match _BUILTIN_TOOL_TO_MCP_SERVER_LABEL values."""
101
+ from vllm.entrypoints.openai.parser.harmony_utils import (
102
+ _BUILTIN_TOOL_TO_MCP_SERVER_LABEL,
103
+ MCP_BUILTIN_TOOLS,
104
+ )
105
71
- # Empty list - returns None
72
- assert server.get_tool_description("test_server", allowed_tools=[]) is None
106
+ assert set(_BUILTIN_TOOL_TO_MCP_SERVER_LABEL.values()) == MCP_BUILTIN_TOOLS, (
107
+ f"MCP_BUILTIN_TOOLS {MCP_BUILTIN_TOOLS} does not match "
108
+ f"_BUILTIN_TOOL_TO_MCP_SERVER_LABEL values "
109
+ f"{set(_BUILTIN_TOOL_TO_MCP_SERVER_LABEL.values())}"
110
+ )
111
112
113
class TestMCPEnabled:
114
"""Tests that require MCP tools to be enabled via environment variable."""
115
116
@pytest.fixture(scope="class")
79
- def monkeypatch_class(self):
80
- from _pytest.monkeypatch import MonkeyPatch
81
-
82
- mpatch = MonkeyPatch()
83
- yield mpatch
84
- mpatch.undo()
85
-
86
- @pytest.fixture(scope="class")
87
- def mcp_enabled_server(self, monkeypatch_class: pytest.MonkeyPatch):
88
- args = ["--enforce-eager", "--tool-server", "demo"]
89
-
90
- with monkeypatch_class.context() as m:
91
- m.setenv("VLLM_ENABLE_RESPONSES_API_STORE", "1")
92
- m.setenv("PYTHON_EXECUTION_BACKEND", "dangerously_use_uv")
93
- m.setenv(
94
- "VLLM_GPT_OSS_SYSTEM_TOOL_MCP_LABELS", "code_interpreter,container"
95
- )
96
- # Helps the model follow instructions better
97
- m.setenv("VLLM_GPT_OSS_HARMONY_SYSTEM_INSTRUCTIONS", "1")
98
- with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
99
- yield remote_server
117
+ def mcp_enabled_server(self):
118
+ env_dict = {
119
+ **BASE_TEST_ENV,
120
+ "VLLM_ENABLE_RESPONSES_API_STORE": "1",
121
+ "PYTHON_EXECUTION_BACKEND": "dangerously_use_uv",
122
+ "VLLM_GPT_OSS_SYSTEM_TOOL_MCP_LABELS": ("code_interpreter,container"),
123
+ "VLLM_GPT_OSS_HARMONY_SYSTEM_INSTRUCTIONS": "1",
124
+ }
125
+ with RemoteOpenAIServer(
126
+ MODEL_NAME, list(_BASE_SERVER_ARGS), env_dict=env_dict
127
+ ) as remote_server:
128
+ yield remote_server
129
130
@pytest_asyncio.fixture
102
- async def mcp_enabled_client(self, mcp_enabled_server):
131
+ async def client(self, mcp_enabled_server):
132
async with mcp_enabled_server.get_async_client() as async_client:
133
yield async_client
134
135
+ @staticmethod
136
+ def _mcp_tools_payload(*, allowed_tools: list[str] | None = None) -> list[dict]:
137
+ tool: dict = {
138
+ "type": "mcp",
139
+ "server_label": "code_interpreter",
140
+ "server_url": "http://localhost:8888",
141
+ }
142
+ if allowed_tools is not None:
143
+ tool["allowed_tools"] = allowed_tools
144
+ return [tool]
145
+
146
+ @staticmethod
147
+ def _python_exec_input(code: str = "") -> str:
148
+ if not code:
149
+ code = "import random; print(random.randint(1, 1000000))"
150
+ return f"Execute the following code: {code}"
151
+
152
@pytest.mark.asyncio
153
@pytest.mark.parametrize("model_name", [MODEL_NAME])
108
- async def test_mcp_tool_env_flag_enabled(
109
- self, mcp_enabled_client: OpenAI, model_name: str
110
- ):
111
- response = await mcp_enabled_client.responses.create(
154
+ async def test_mcp_tool_env_flag_enabled(self, client: OpenAI, model_name: str):
155
+ response = await retry_for_tool_call(
156
+ client,
157
model=model_name,
113
- input=(
114
- "Execute the following code: "
115
- "import random; print(random.randint(1, 1000000))"
116
- ),
117
- instructions=(
118
- "You must use the Python tool to execute code. "
119
- "Never simulate execution."
120
- ),
121
- tools=[
122
- {
123
- "type": "mcp",
124
- "server_label": "code_interpreter",
125
- # URL unused for DemoToolServer
126
- "server_url": "http://localhost:8888",
127
- }
128
- ],
158
+ expected_tool_type="mcp_call",
159
+ input=self._python_exec_input(),
160
+ instructions=_PYTHON_TOOL_INSTRUCTION,
161
+ tools=self._mcp_tools_payload(),
162
+ temperature=0.0,
163
extra_body={"enable_response_messages": True},
164
)
131
- assert response is not None
165
+
166
assert response.status == "completed"
133
- # Verify output messages: Tool calls and responses on analysis channel
167
+ log_response_diagnostics(response, label="MCP Enabled")
168
+
169
tool_call_found = False
170
tool_response_found = False
171
for message in response.output_messages:
172
recipient = message.get("recipient")
173
if recipient and recipient.startswith("python"):
174
tool_call_found = True
140
- assert message.get("channel") == "analysis", (
141
- "Tool call should be on analysis channel"
142
- )
175
+ assert message.get("channel") == "analysis"
176
author = message.get("author", {})
144
- if (
145
- author.get("role") == "tool"
146
- and author.get("name")
147
- and author.get("name").startswith("python")
177
+ if author.get("role") == "tool" and (author.get("name") or "").startswith(
178
+ "python"
179
):
180
tool_response_found = True
150
- assert message.get("channel") == "analysis", (
151
- "Tool response should be on analysis channel"
152
- )
181
+ assert message.get("channel") == "analysis"
182
154
- assert tool_call_found, "Should have found at least one Python tool call"
155
- assert tool_response_found, (
156
- "Should have found at least one Python tool response"
183
+ assert tool_call_found, (
184
+ f"No Python tool call found. "
185
+ f"Output types: "
186
+ f"{[getattr(o, 'type', None) for o in response.output]}"
187
)
188
+ assert tool_response_found, "No Python tool response found"
189
+
190
for message in response.input_messages:
159
- assert message.get("author").get("role") != "developer", (
160
- "No developer messages should be present with valid mcp tool"
161
- )
191
+ assert message.get("author", {}).get("role") != "developer"
192
163
- @pytest.mark.flaky(reruns=3)
193
@pytest.mark.asyncio
194
@pytest.mark.parametrize("model_name", [MODEL_NAME])
195
async def test_mcp_tool_with_allowed_tools_star(
167
- self, mcp_enabled_client: OpenAI, model_name: str
196
+ self, client: OpenAI, model_name: str
197
):
169
- """Test MCP tool with allowed_tools=['*'] to select all available
170
- tools.
171
-
172
- This E2E test verifies that the "*" wildcard works end-to-end.
173
- See test_serving_responses.py for detailed unit tests of "*"
174
- normalization.
175
- """
176
- response = await mcp_enabled_client.responses.create(
198
+ response = await retry_for_tool_call(
199
+ client,
200
model=model_name,
178
- input=(
179
- "Execute the following code: "
180
- "import random; print(random.randint(1, 1000000))"
181
- ),
182
- instructions=(
183
- "You must use the Python tool to execute code. "
184
- "Never simulate execution."
185
- ),
186
- tools=[
187
- {
188
- "type": "mcp",
189
- "server_label": "code_interpreter",
190
- "server_url": "http://localhost:8888",
191
- # Using "*" to allow all tools from this MCP server
192
- "allowed_tools": ["*"],
193
- }
194
- ],
201
+ expected_tool_type="mcp_call",
202
+ input=self._python_exec_input(),
203
+ instructions=_PYTHON_TOOL_INSTRUCTION,
204
+ tools=self._mcp_tools_payload(allowed_tools=["*"]),
205
+ temperature=0.0,
206
extra_body={"enable_response_messages": True},
207
)
197
- assert response is not None
208
+
209
assert response.status == "completed"
199
- # Verify tool calls work with allowed_tools=["*"]
200
- tool_call_found = False
201
- for message in response.output_messages:
202
- recipient = message.get("recipient")
203
- if recipient and recipient.startswith("python"):
204
- tool_call_found = True
205
- break
210
+ log_response_diagnostics(response, label="MCP Allowed Tools *")
211
+
212
+ tool_call_found = any(
213
+ (msg.get("recipient") or "").startswith("python")
214
+ for msg in response.output_messages
215
+ )
216
assert tool_call_found, (
207
- "Should have found at least one Python tool call with '*'"
217
+ f"No Python tool call with '*'. "
218
+ f"Output types: "
219
+ f"{[getattr(o, 'type', None) for o in response.output]}"
220
)
221
210
- @pytest.mark.flaky(reruns=3)
222
@pytest.mark.asyncio
223
@pytest.mark.parametrize("model_name", [MODEL_NAME])
224
async def test_mcp_tool_calling_streaming_types(
225
self,
226
pairs_of_event_types: dict[str, str],
216
- mcp_enabled_client: OpenAI,
227
+ client: OpenAI,
228
model_name: str,
229
):
219
- tools = [
220
- {
221
- "type": "mcp",
222
- "server_label": "code_interpreter",
223
- }
224
- ]
225
- input_text = "What is 123 * 456? Use python to calculate the result."
226
-
227
- stream_response = await mcp_enabled_client.responses.create(
230
+ def _has_mcp_events(events: list) -> bool:
231
+ return events_contain_type(events, "mcp_call")
232
+
233
+ events = await retry_streaming_for(
234
+ client,
235
model=model_name,
229
- input=input_text,
230
- tools=tools,
231
- stream=True,
232
- instructions=(
233
- "You must use the Python tool to execute code. "
234
- "Never simulate execution."
235
- ),
236
+ validate_events=_has_mcp_events,
237
+ input=("What is 123 * 456? Use Python to calculate the result."),
238
+ tools=[{"type": "mcp", "server_label": "code_interpreter"}],
239
+ instructions=_PYTHON_TOOL_INSTRUCTION,
240
+ temperature=0.0,
241
)
242
238
- stack_of_event_types = []
239
- saw_mcp_type = False
240
- async for event in stream_response:
241
- if event.type == "response.created":
242
- stack_of_event_types.append(event.type)
243
- elif event.type == "response.completed":
244
- assert stack_of_event_types[-1] == pairs_of_event_types[event.type]
245
- stack_of_event_types.pop()
246
- elif (
247
- event.type.endswith("added")
248
- or event.type == "response.mcp_call.in_progress"
249
- ):
250
- stack_of_event_types.append(event.type)
251
- elif event.type.endswith("delta"):
252
- if stack_of_event_types[-1] == event.type:
253
- continue
254
- stack_of_event_types.append(event.type)
255
- elif (
256
- event.type.endswith("done")
257
- or event.type == "response.mcp_call.completed"
258
- ):
259
- assert stack_of_event_types[-1] == pairs_of_event_types[event.type]
260
- if "mcp_call" in event.type:
261
- saw_mcp_type = True
262
- stack_of_event_types.pop()
243
+ validate_streaming_event_stack(events, pairs_of_event_types)
244
264
- assert len(stack_of_event_types) == 0
265
- assert saw_mcp_type, "Should have seen at least one mcp call"
245
+ assert events_contain_type(events, "mcp_call"), (
246
+ f"No mcp_call events after retries. "
247
+ f"Event types: {sorted({e.type for e in events})}"
248
+ )
249
250
251
class TestMCPDisabled:
269
- """Tests that verify behavior when MCP tools are disabled."""
270
-
271
- @pytest.fixture(scope="class")
272
- def monkeypatch_class(self):
273
- from _pytest.monkeypatch import MonkeyPatch
274
-
275
- mpatch = MonkeyPatch()
276
- yield mpatch
277
- mpatch.undo()
252
+ """Tests that MCP tools are not executed when the env flag is unset."""
253
254
@pytest.fixture(scope="class")
280
- def mcp_disabled_server(self, monkeypatch_class: pytest.MonkeyPatch):
281
- args = ["--enforce-eager", "--tool-server", "demo"]
282
-
283
- with monkeypatch_class.context() as m:
284
- m.setenv("VLLM_ENABLE_RESPONSES_API_STORE", "1")
285
- m.setenv("PYTHON_EXECUTION_BACKEND", "dangerously_use_uv")
286
- # Helps the model follow instructions better
287
- m.setenv("VLLM_GPT_OSS_HARMONY_SYSTEM_INSTRUCTIONS", "1")
288
- with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
289
- yield remote_server
255
+ def mcp_disabled_server(self):
256
+ env_dict = {
257
+ **BASE_TEST_ENV,
258
+ "VLLM_ENABLE_RESPONSES_API_STORE": "1",
259
+ "PYTHON_EXECUTION_BACKEND": "dangerously_use_uv",
260
+ "VLLM_GPT_OSS_HARMONY_SYSTEM_INSTRUCTIONS": "1",
261
+ }
262
+ with RemoteOpenAIServer(
263
+ MODEL_NAME, list(_BASE_SERVER_ARGS), env_dict=env_dict
264
+ ) as remote_server:
265
+ yield remote_server
266
267
@pytest_asyncio.fixture
292
- async def mcp_disabled_client(self, mcp_disabled_server):
268
+ async def client(self, mcp_disabled_server):
269
async with mcp_disabled_server.get_async_client() as async_client:
270
yield async_client
271
272
@pytest.mark.asyncio
273
@pytest.mark.parametrize("model_name", [MODEL_NAME])
298
- async def test_mcp_tool_env_flag_disabled(
299
- self, mcp_disabled_client: OpenAI, model_name: str
274
+ async def test_mcp_disabled_server_does_not_execute(
275
+ self, client: OpenAI, model_name: str
276
):
301
- response = await mcp_disabled_client.responses.create(
277
+ """When MCP is disabled the model may still attempt tool calls
278
+ (tool descriptions can remain in the prompt), but the server
279
+ must NOT execute them."""
280
+ response = await client.responses.create(
281
model=model_name,
282
input=(
283
"Execute the following code if the tool is present: "
@@ -308,38 +287,35 @@ class TestMCPDisabled:
287
{
288
"type": "mcp",
289
"server_label": "code_interpreter",
311
- # URL unused for DemoToolServer
290
"server_url": "http://localhost:8888",
291
}
292
],
293
+ temperature=0.0,
294
extra_body={"enable_response_messages": True},
295
)
296
assert response is not None
297
assert response.status == "completed"
319
- # Verify output messages: No tool calls and responses
320
- tool_call_found = False
321
- tool_response_found = False
298
+
299
+ log_response_diagnostics(response, label="MCP Disabled")
300
+
301
+ # Server must not have executed any tool calls
302
for message in response.output_messages:
323
- recipient = message.get("recipient")
324
- if recipient and recipient.startswith("python"):
325
- tool_call_found = True
326
- assert message.get("channel") == "analysis", (
327
- "Tool call should be on analysis channel"
328
- )
303
author = message.get("author", {})
330
- if (
304
+ assert not (
305
author.get("role") == "tool"
332
- and author.get("name")
333
- and author.get("name").startswith("python")
334
- ):
335
- tool_response_found = True
336
- assert message.get("channel") == "analysis", (
337
- "Tool response should be on analysis channel"
306
+ and (author.get("name") or "").startswith("python")
307
+ ), (
308
+ "Server executed a python tool call even though MCP is "
309
+ f"disabled. Message: {message}"
310
+ )
311
+
312
+ # No completed mcp_call output items
313
+ for item in response.output:
314
+ if getattr(item, "type", None) == "mcp_call":
315
+ assert getattr(item, "status", None) != "completed", (
316
+ "MCP call should not be completed when MCP is disabled"
317
)
318
340
- assert not tool_call_found, "Should not have a python call"
341
- assert not tool_response_found, "Should not have a tool response"
319
+ # No developer messages injected
320
for message in response.input_messages:
343
- assert message.get("author").get("role") != "developer", (
344
- "No developer messages should be present without a valid tool"
345
- )
321
+ assert message.get("author", {}).get("role") != "developer"
tests/entrypoints/openai/responses/test_parsable_context.py
+112
-41
@@ -3,15 +3,29 @@
3
4
import importlib.util
5
import json
6
+import logging
7
8
import pytest
9
import pytest_asyncio
10
from openai import OpenAI
11
12
from ....utils import RemoteOpenAIServer
13
+from .conftest import (
14
+ BASE_TEST_ENV,
15
+ has_output_type,
16
+ log_response_diagnostics,
17
+ retry_for_tool_call,
18
+)
19
+
20
+logger = logging.getLogger(__name__)
21
22
MODEL_NAME = "Qwen/Qwen3-8B"
23
24
+_PYTHON_TOOL_INSTRUCTION = (
25
+ "You must use the Python tool to execute code. "
26
+ "Never simulate execution. You must print the final answer."
27
+)
28
+
29
30
@pytest.fixture(scope="module")
31
def server():
@@ -32,12 +46,12 @@ def server():
46
"--tool-server",
47
"demo",
48
]
35
- env_dict = dict(
36
- VLLM_ENABLE_RESPONSES_API_STORE="1",
37
- VLLM_USE_EXPERIMENTAL_PARSER_CONTEXT="1",
38
- PYTHON_EXECUTION_BACKEND="dangerously_use_uv",
39
- )
40
-
49
+ env_dict = {
50
+ **BASE_TEST_ENV,
51
+ "VLLM_ENABLE_RESPONSES_API_STORE": "1",
52
+ "VLLM_USE_EXPERIMENTAL_PARSER_CONTEXT": "1",
53
+ "PYTHON_EXECUTION_BACKEND": "dangerously_use_uv",
54
+ }
55
with RemoteOpenAIServer(MODEL_NAME, args, env_dict=env_dict) as remote_server:
56
yield remote_server
57
@@ -54,6 +68,7 @@ async def test_basic(client: OpenAI, model_name: str):
68
response = await client.responses.create(
69
model=model_name,
70
input="What is 123 * 456?",
71
+ temperature=0.0,
72
)
73
assert response is not None
74
print("response: ", response)
@@ -99,10 +114,15 @@ async def test_reasoning_and_function_items(client: OpenAI, model_name: str):
114
)
115
assert response is not None
116
assert response.status == "completed"
102
- # make sure we get a reasoning and text output
103
- assert response.output[0].type == "reasoning"
104
- assert response.output[1].type == "message"
105
- assert type(response.output[1].content[0].text) is str
117
+
118
+ output_types = [getattr(o, "type", None) for o in response.output]
119
+ assert "reasoning" in output_types, (
120
+ f"Expected reasoning in output, got: {output_types}"
121
+ )
122
+ assert "message" in output_types, f"Expected message in output, got: {output_types}"
123
+
124
+ msg = next(o for o in response.output if o.type == "message")
125
+ assert type(msg.content[0].text) is str
126
127
128
def get_horoscope(sign):
@@ -110,10 +130,10 @@ def get_horoscope(sign):
130
131
132
def call_function(name, args):
133
+ logger.info("Calling function %s with args %s", name, args)
134
if name == "get_horoscope":
135
return get_horoscope(**args)
115
- else:
116
- raise ValueError(f"Unknown function: {name}")
136
+ raise ValueError(f"Unknown function: {name}")
137
138
139
@pytest.mark.asyncio
@@ -136,61 +156,111 @@ async def test_function_call_first_turn(client: OpenAI, model_name: str):
156
}
157
]
158
139
- response = await client.responses.create(
159
+ response = await retry_for_tool_call(
160
+ client,
161
model=model_name,
162
+ expected_tool_type="function_call",
163
input="What is the horoscope for Aquarius today?",
164
tools=tools,
165
temperature=0.0,
166
)
167
assert response is not None
168
assert response.status == "completed"
147
- assert len(response.output) == 2
148
- assert response.output[0].type == "reasoning"
149
- assert response.output[1].type == "function_call"
169
151
- function_call = response.output[1]
170
+ output_types = [getattr(o, "type", None) for o in response.output]
171
+ assert "reasoning" in output_types, (
172
+ f"Expected reasoning in output, got: {output_types}"
173
+ )
174
+ assert has_output_type(response, "function_call"), (
175
+ f"Expected function_call in output, got: {output_types}"
176
+ )
177
+
178
+ function_call = next(o for o in response.output if o.type == "function_call")
179
assert function_call.name == "get_horoscope"
180
assert function_call.call_id is not None
181
182
args = json.loads(function_call.arguments)
183
assert "sign" in args
184
158
- # the multi turn function call is tested above in
159
- # test_reasoning_and_function_items
160
-
185
186
@pytest.mark.asyncio
187
@pytest.mark.parametrize("model_name", [MODEL_NAME])
188
async def test_mcp_tool_call(client: OpenAI, model_name: str):
165
- response = await client.responses.create(
189
+ """MCP tool calling with code_interpreter.
190
+
191
+ The model may make one or more tool calls before producing a final
192
+ message. We validate server invariants (mcp_call items have correct
193
+ fields) with hard assertions. Output indices are never hardcoded
194
+ since the model can produce multiple tool-call rounds.
195
+ """
196
+ # MCP + container init + code execution can be slow
197
+ client_with_timeout = client.with_options(timeout=client.timeout * 3)
198
+
199
+ response = await retry_for_tool_call(
200
+ client_with_timeout,
201
model=model_name,
167
- input="What is 123 * 456? Use python to calculate the result.",
202
+ expected_tool_type="mcp_call",
203
+ input=(
204
+ "What is 123 * 456? Use python to calculate the result. "
205
+ "Print the result with print()."
206
+ ),
207
tools=[{"type": "code_interpreter", "container": {"type": "auto"}}],
169
- extra_body={"enable_response_messages": True},
208
+ instructions=_PYTHON_TOOL_INSTRUCTION,
209
temperature=0.0,
210
+ extra_body={"enable_response_messages": True},
211
)
212
213
assert response is not None
174
- assert response.status == "completed"
214
176
- # The model may produce multiple reasoning/mcp_call rounds before the
177
- # final message, so validate structurally rather than by exact index.
178
- output_types = [o.type for o in response.output]
179
- assert "reasoning" in output_types
180
- mcp_calls = [o for o in response.output if o.type == "mcp_call"]
181
- assert len(mcp_calls) >= 1
182
- assert type(mcp_calls[0].arguments) is str
183
- assert type(mcp_calls[0].output) is str
184
-
185
- # The final output should be a message containing the correct answer
186
- assert response.output[-1].type == "message"
187
- assert any(s in response.output[-1].content[0].text for s in ("56088", "56,088"))
188
-
189
- # Test raw input_messages / output_messages
190
- assert len(response.input_messages) == 1
191
- assert len(response.output_messages) >= 3
215
+ output_types = [getattr(o, "type", None) for o in response.output]
216
+ log_response_diagnostics(response, label="test_mcp_tool_call")
217
+
218
+ assert response.status == "completed", (
219
+ f"Response status={response.status} "
220
+ f"(details={getattr(response, 'incomplete_details', None)}). "
221
+ f"Output types: {output_types}."
222
+ )
223
+
224
+ assert "reasoning" in output_types, (
225
+ f"Expected reasoning in output, got: {output_types}"
226
+ )
227
+ assert "mcp_call" in output_types, (
228
+ f"Expected mcp_call in output, got: {output_types}"
229
+ )
230
+
231
+ # Every mcp_call item must have well-typed fields
232
+ for item in response.output:
233
+ if getattr(item, "type", None) == "mcp_call":
234
+ assert type(item.arguments) is str, (
235
+ f"mcp_call.arguments should be str, got {type(item.arguments)}"
236
+ )
237
+ assert type(item.output) is str, (
238
+ f"mcp_call.output should be str, got {type(item.output)}"
239
+ )
240
+
241
+ # The model may make 1+ tool-call rounds but must still produce
242
+ # a final message for a trivial calculation like 123 * 456.
243
+ message_outputs = [
244
+ o for o in response.output if getattr(o, "type", None) == "message"
245
+ ]
246
+ assert message_outputs, (
247
+ f"Model did not produce a final message. Output types: {output_types}"
248
+ )
249
+
250
+ final_message = message_outputs[-1]
251
+ assert any(s in final_message.content[0].text for s in ("56088", "56,088")), (
252
+ f"Expected 56088 in final message, got: {final_message.content[0].text!r}"
253
+ )
254
+
255
+ # Validate raw input_messages / output_messages
256
+ assert len(response.input_messages) >= 1, "Expected at least 1 input message"
257
+ assert len(response.output_messages) >= 1, "Expected at least 1 output message"
258
assert any(
193
- s in response.output_messages[-1]["message"] for s in ("56088", "56,088")
259
+ any(s in str(msg) for s in ("56088", "56,088"))
260
+ for msg in response.output_messages
261
+ ), (
262
+ f"Expected 56088 in at least one output_message, "
263
+ f"got {len(response.output_messages)} messages"
264
)
265
266
@@ -202,6 +272,7 @@ async def test_max_tokens(client: OpenAI, model_name: str):
272
input="What is the first paragraph of Moby Dick?",
273
reasoning={"effort": "low"},
274
max_output_tokens=30,
275
+ temperature=0.0,
276
)
277
assert response is not None
278
assert response.status == "incomplete"
tests/entrypoints/openai/responses/test_simple.py
+7
-5
@@ -12,13 +12,15 @@ MODEL_NAME = "Qwen/Qwen3-8B"
12
13
@pytest.fixture(scope="module")
14
def server():
15
+ from .conftest import BASE_TEST_ENV
16
+
17
args = ["--reasoning-parser", "qwen3", "--max_model_len", "5000"]
16
- env_dict = dict(
17
- VLLM_ENABLE_RESPONSES_API_STORE="1",
18
+ env_dict = {
19
+ **BASE_TEST_ENV,
20
+ "VLLM_ENABLE_RESPONSES_API_STORE": "1",
21
# uncomment for tool calling
19
- # PYTHON_EXECUTION_BACKEND="dangerously_use_uv",
20
- )
21
-
22
+ # PYTHON_EXECUTION_BACKEND: "dangerously_use_uv",
23
+ }
24
with RemoteOpenAIServer(MODEL_NAME, args, env_dict=env_dict) as remote_server:
25
yield remote_server
26
tests/utils.py
+123
-30
@@ -128,6 +128,9 @@ class RemoteOpenAIServer:
128
env=env,
129
stdout=sys.stdout,
130
stderr=sys.stderr,
131
+ # Create a dedicated process group so we can kill
132
+ # the entire tree (parent + EngineCore + workers) at once.
133
+ start_new_session=True,
134
)
135
136
def __init__(
@@ -189,6 +192,15 @@ class RemoteOpenAIServer:
192
model_loader = get_model_loader(load_config)
193
model_loader.download_model(model_config)
194
195
+ # Record GPU memory before server start so we know what
196
+ # "released" looks like.
197
+ self._pre_server_gpu_memory = self._get_gpu_memory_used()
198
+ if self._pre_server_gpu_memory is not None:
199
+ pre_gb = self._pre_server_gpu_memory / 1e9
200
+ print(
201
+ f"[RemoteOpenAIServer] GPU memory before server start: {pre_gb:.2f} GB"
202
+ )
203
+
204
self._start_server(model, vllm_serve_args, env_dict)
205
max_wait_seconds = max_wait_seconds or 360
206
self._wait_for_server(url=self.url_for("health"), timeout=max_wait_seconds)
@@ -198,27 +210,69 @@ class RemoteOpenAIServer:
210
211
def __exit__(self, exc_type, exc_value, traceback):
212
pid = self.proc.pid
201
- # Graceful shutdown
202
- self.proc.terminate()
213
+
214
+ # Get the process group ID. Because we used
215
+ # start_new_session=True the pgid equals the server's pid.
216
+ try:
217
+ pgid = os.getpgid(pid)
218
+ except (ProcessLookupError, OSError):
219
+ pgid = None
220
+
221
+ # Phase 1: graceful SIGTERM to the entire process group
222
+ if pgid is not None:
223
+ with contextlib.suppress(ProcessLookupError, OSError):
224
+ os.killpg(pgid, signal.SIGTERM)
225
+ print(f"[RemoteOpenAIServer] Sent SIGTERM to process group {pgid}")
226
+ else:
227
+ self.proc.terminate()
228
+
229
try:
230
self.proc.wait(timeout=15)
231
print(f"[RemoteOpenAIServer] Server {pid} terminated gracefully")
232
except subprocess.TimeoutExpired:
233
+ # Phase 2: SIGKILL the entire process group
234
print(
235
f"[RemoteOpenAIServer] Server {pid} did not respond "
209
- "to SIGTERM, sending SIGKILL"
236
+ "to SIGTERM, sending SIGKILL to process group"
237
)
211
- self.proc.kill()
238
+ if pgid is not None:
239
+ with contextlib.suppress(ProcessLookupError, OSError):
240
+ os.killpg(pgid, signal.SIGKILL)
241
+ else:
242
+ self.proc.kill()
243
+
244
try:
213
- self.proc.wait(timeout=5)
245
+ self.proc.wait(timeout=10)
246
print(f"[RemoteOpenAIServer] Server {pid} killed")
215
- except subprocess.TimeoutExpired as err:
216
- raise RuntimeError(
217
- f"[RemoteOpenAIServer] Failed to kill server process {pid}"
218
- ) from err
219
- # Wait for GPU memory to be released
247
+ except subprocess.TimeoutExpired:
248
+ # Phase 3: last resort - find and kill any orphaned children
249
+ self._kill_orphaned_children(pid)
250
+
251
+ # Wait for GPU memory to actually be *freed*, not just
252
+ # "stabilized at whatever level it's at".
253
self._wait_for_gpu_memory_release()
254
255
+ def _kill_orphaned_children(self, parent_pid: int) -> None:
256
+ """Best-effort cleanup of any lingering child processes."""
257
+ try:
258
+ import psutil
259
+
260
+ parent = psutil.Process(parent_pid)
261
+ children = parent.children(recursive=True)
262
+ for child in children:
263
+ print(
264
+ f"[RemoteOpenAIServer] Killing orphaned child "
265
+ f"pid={child.pid} name={child.name()}"
266
+ )
267
+ child.kill()
268
+ psutil.wait_procs(children, timeout=5)
269
+ except Exception as e:
270
+ # psutil may not be installed, or processes already gone
271
+ print(f"[RemoteOpenAIServer] Orphan cleanup failed: {e}")
272
+ # Fallback: try to kill by pgid one more time
273
+ with contextlib.suppress(ProcessLookupError, OSError):
274
+ os.killpg(parent_pid, signal.SIGKILL)
275
+
276
def _get_gpu_memory_used(self) -> float | None:
277
"""Get total GPU memory used across all visible devices in bytes."""
278
try:
@@ -244,10 +298,26 @@ class RemoteOpenAIServer:
298
return None
299
return None
300
247
- def _wait_for_gpu_memory_release(self, timeout: float = 30.0):
248
- """Poll GPU memory until it stabilizes, indicating cleanup is complete."""
301
+ def _wait_for_gpu_memory_release(self, timeout: float = 60.0):
302
+ """Wait for GPU memory to drop back toward pre-server levels.
303
+
304
+ Two-phase strategy:
305
+ 1. Try to wait for memory to return close to pre-server baseline.
306
+ 2. If that doesn't happen, fall back to waiting for stabilization
307
+ and log a warning (the next server might still OOM).
308
+ """
309
+ baseline = self._pre_server_gpu_memory
310
+ if baseline is None:
311
+ # Can't query GPU memory - nothing to do
312
+ return
313
+
314
+ # Allow up to 2 GiB overhead above baseline for driver/context state
315
+ # that may persist between server instances.
316
+ headroom_bytes = 2 * 1024 * 1024 * 1024
317
+ target = baseline + headroom_bytes
318
+
319
start = time.time()
250
- prev_used: float | None = None
320
+ last_used: float | None = None
321
stable_count = 0
322
323
while time.time() - start < timeout:
@@ -256,26 +326,49 @@ class RemoteOpenAIServer:
326
if used is None:
327
return # Can't query, assume ok
328
259
- if prev_used is not None and abs(used - prev_used) < 100 * 1024 * 1024:
260
- stable_count += 1
261
- if stable_count >= 3:
262
- used_gb = used / 1e9
263
- print(
264
- f"[RemoteOpenAIServer] GPU memory stabilized "
265
- f"at {used_gb:.2f} GB"
266
- )
267
- return
268
- else:
269
- stable_count = 0
329
+ used_gb = used / 1e9
330
+ target_gb = target / 1e9
331
+ elapsed = time.time() - start
332
+
333
+ # Phase 1: memory dropped to near baseline - we're done.
334
+ if used <= target:
335
+ print(
336
+ f"[RemoteOpenAIServer] GPU memory released to "
337
+ f"{used_gb:.2f} GB (target: {target_gb:.2f} GB) "
338
+ f"in {elapsed:.1f}s"
339
+ )
340
+ return
341
+
342
+ # Phase 2 (after 40s): fall back to stabilization check.
343
+ # This handles cases where another process is using GPU memory
344
+ # and we'll never reach baseline.
345
+ if elapsed > 40.0 and last_used is not None:
346
+ delta = abs(used - last_used)
347
+ if delta < 200 * 1024 * 1024: # 200 MB
348
+ stable_count += 1
349
+ if stable_count >= 3:
350
+ print(
351
+ f"[RemoteOpenAIServer] WARNING: GPU memory "
352
+ f"stabilized at {used_gb:.2f} GB "
353
+ f"(target was {target_gb:.2f} GB). "
354
+ f"Proceeding - next server may OOM."
355
+ )
356
+ return
357
+ else:
358
+ stable_count = 0
359
271
- prev_used = used
272
- time.sleep(0.1)
360
+ last_used = used
361
+ time.sleep(1.0)
362
274
- last_reading = prev_used / 1e9 if prev_used is not None else 0.0
363
+ # Timeout - log clearly so CI failures are diagnosable
364
+ final_used = self._get_gpu_memory_used()
365
+ final_gb = final_used / 1e9 if final_used else 0.0
366
raise RuntimeError(
276
- f"[RemoteOpenAIServer] GPU memory did not stabilize within {timeout}s. "
277
- f"Last reading: {last_reading:.2f} GB. "
278
- "Child processes may still be holding GPU memory."
367
+ f"[RemoteOpenAIServer] GPU memory did not release within "
368
+ f"{timeout}s. Current: {final_gb:.2f} GB, "
369
+ f"target: {target / 1e9:.2f} GB, "
370
+ f"baseline: {baseline / 1e9:.2f} GB. "
371
+ f"Child processes may still be holding GPU memory."
372
)
373
374
def _poll(self) -> int | None:
vllm/entrypoints/openai/parser/harmony_utils.py
+76
-35
@@ -48,8 +48,11 @@ from vllm.entrypoints.openai.responses.protocol import (
48
ResponseInputOutputItem,
49
ResponsesRequest,
50
)
51
+from vllm.logger import init_logger
52
from vllm.utils import random_uuid
53
54
+logger = init_logger(__name__)
55
+
56
REASONING_EFFORT = {
57
"high": ReasoningEffort.HIGH,
58
"medium": ReasoningEffort.MEDIUM,
@@ -62,20 +65,15 @@ _harmony_encoding = None
65
# they are available and requested by the user.
66
# Tool args are provided by MCP tool descriptions. Output
67
# of the tools are stringified.
65
-MCP_BUILTIN_TOOLS: set[str] = {
66
- "web_search_preview",
67
- "code_interpreter",
68
- "container",
69
-}
70
-
71
-# Mapping from built-in tool recipient names to their MCP server labels.
72
-# This ensures consistency between streaming and non-streaming responses.
68
_BUILTIN_TOOL_TO_MCP_SERVER_LABEL: dict[str, str] = {
69
"python": "code_interpreter",
70
"browser": "web_search_preview",
71
"container": "container",
72
}
73
74
+# Derive MCP_BUILTIN_TOOLS from the canonical mapping
75
+MCP_BUILTIN_TOOLS: set[str] = set(_BUILTIN_TOOL_TO_MCP_SERVER_LABEL.values())
76
+
77
78
def has_custom_tools(tool_types: set[str]) -> bool:
79
"""
@@ -116,8 +114,11 @@ def get_system_message(
114
REASONING_EFFORT[reasoning_effort]
115
)
116
if start_date is None:
119
- # NOTE(woosuk): This brings non-determinism in vLLM. Be careful.
120
- start_date = datetime.datetime.now().strftime("%Y-%m-%d")
117
+ # NOTE(woosuk): This brings non-determinism in vLLM.
118
+ # Set VLLM_SYSTEM_START_DATE to pin it.
119
+ start_date = envs.VLLM_SYSTEM_START_DATE or datetime.datetime.now().strftime(
120
+ "%Y-%m-%d"
121
+ )
122
sys_msg_content = sys_msg_content.with_conversation_start_date(start_date)
123
if browser_description is not None:
124
sys_msg_content = sys_msg_content.with_tools(browser_description)
@@ -398,15 +399,60 @@ def parse_chat_input_to_harmony_message(
399
400
401
def parse_input_to_harmony_message(chat_msg) -> list[Message]:
401
- """
402
- Parse a message from request.previous_input_messages in the Responsees API to
403
- Harmony messages.
402
+ """Parse a message from request.previous_input_messages
403
+ into Harmony messages.
404
+
405
+ Supports both OpenAI chat format ({"role": "..."}) and
406
+ Harmony format ({"author": {"role": "..."}}).
407
"""
408
if not isinstance(chat_msg, dict):
406
- # Handle Pydantic models
409
chat_msg = chat_msg.model_dump(exclude_none=True)
410
411
+ if "author" in chat_msg and isinstance(chat_msg.get("author"), dict):
412
+ return [_parse_harmony_format_message(chat_msg)]
413
+
414
+ return _parse_chat_format_message(chat_msg)
415
+
416
+
417
+def _parse_harmony_format_message(chat_msg: dict) -> Message:
418
+ """Reconstruct a Message from Harmony-format dict,
419
+ preserving channel, recipient, and content_type."""
420
+ author_dict = chat_msg["author"]
421
+ role = author_dict.get("role")
422
+ name = author_dict.get("name")
423
+
424
+ raw_content = chat_msg.get("content", "")
425
+ if isinstance(raw_content, list):
426
+ # TODO: Support refusal and non-text content types.
427
+ contents = [TextContent(text=c.get("text", "")) for c in raw_content]
428
+ elif isinstance(raw_content, str):
429
+ contents = [TextContent(text=raw_content)]
430
+ else:
431
+ contents = [TextContent(text="")]
432
+
433
+ if name:
434
+ msg = Message.from_author_and_contents(Author.new(Role(role), name), contents)
435
+ else:
436
+ msg = Message.from_role_and_contents(Role(role), contents)
437
+
438
+ channel = chat_msg.get("channel")
439
+ if channel:
440
+ msg = msg.with_channel(channel)
441
+ recipient = chat_msg.get("recipient")
442
+ if recipient:
443
+ msg = msg.with_recipient(recipient)
444
+ content_type = chat_msg.get("content_type")
445
+ if content_type:
446
+ msg = msg.with_content_type(content_type)
447
+
448
+ return msg
449
+
450
+
451
+def _parse_chat_format_message(chat_msg: dict) -> list[Message]:
452
+ """Parse an OpenAI chat-format dict into Harmony messages."""
453
role = chat_msg.get("role")
454
+ if role is None:
455
+ raise ValueError(f"Message has no 'role' key: {chat_msg}")
456
457
# Assistant message with tool calls
458
tool_calls = chat_msg.get("tool_calls")
@@ -426,15 +472,21 @@ def parse_input_to_harmony_message(chat_msg) -> list[Message]:
472
# Tool role message (tool output)
473
if role == "tool":
474
name = chat_msg.get("name", "")
475
+ if name and not name.startswith("functions."):
476
+ name = f"functions.{name}"
477
content = chat_msg.get("content", "") or ""
478
content = flatten_chat_text_content(content)
431
-
432
- msg = Message.from_author_and_content(
433
- Author.new(Role.TOOL, f"functions.{name}"), content
434
- ).with_channel("commentary")
479
+ # NOTE: .with_recipient("assistant") is required on tool messages
480
+ # to match parse_chat_input_to_harmony_message behavior and ensure
481
+ # proper routing in the Harmony protocol.
482
+ msg = (
483
+ Message.from_author_and_content(Author.new(Role.TOOL, name), content)
484
+ .with_channel("commentary")
485
+ .with_recipient("assistant")
486
+ )
487
return [msg]
488
437
- # Default: user/assistant/system messages with content
489
+ # Default: user/assistant/system messages
490
content = chat_msg.get("content", "")
491
if isinstance(content, str):
492
contents = [TextContent(text=content)]
@@ -497,6 +549,10 @@ def _parse_browser_tool_call(message: Message, recipient: str) -> ResponseOutput
549
try:
550
browser_call = json.loads(content.text)
551
except json.JSONDecodeError:
552
+ logger.warning(
553
+ "Invalid JSON in browser tool call, using error placeholder: %s",
554
+ content.text,
555
+ )
556
json_retry_output_message = (
557
f"Invalid JSON args, caught and retried: {content.text}"
558
)
@@ -730,22 +786,7 @@ def parse_remaining_state(parser: StreamableParser) -> list[ResponseOutputItem]:
786
)
787
]
788
733
- if parser.current_channel == "commentary":
734
- return [
735
- ResponseReasoningItem(
736
- id=f"rs_{random_uuid()}",
737
- summary=[],
738
- type="reasoning",
739
- content=[
740
- ResponseReasoningTextContent(
741
- text=parser.current_content, type="reasoning_text"
742
- )
743
- ],
744
- status=None,
745
- )
746
- ]
747
-
748
- if parser.current_channel == "analysis":
789
+ if parser.current_channel in ("commentary", "analysis"):
790
return [
791
ResponseReasoningItem(
792
id=f"rs_{random_uuid()}",
vllm/entrypoints/openai/responses/context.py
+19
-15
@@ -346,17 +346,17 @@ class ParsableContext(ConversationContext):
346
self.parser.response_messages.extend(output)
347
348
def need_builtin_tool_call(self) -> bool:
349
- """Return true if the last message is a MCP tool call"""
349
+ """Return true if the last message is a builtin tool call
350
+ that the request has enabled."""
351
last_message = self.parser.response_messages[-1]
351
- # TODO(qandrew): figure out which tools are MCP tools
352
- if last_message.type == "function_call": # noqa: SIM102
353
- if last_message.name in (
354
- "code_interpreter",
355
- "python",
356
- "web_search_preview",
357
- ) or last_message.name.startswith("container"):
358
- return True
359
-
352
+ if last_message.type != "function_call":
353
+ return False
354
+ if last_message.name in ("code_interpreter", "python"):
355
+ return "python" in self.available_tools
356
+ if last_message.name == "web_search_preview":
357
+ return "browser" in self.available_tools
358
+ if last_message.name.startswith("container"):
359
+ return "container" in self.available_tools
360
return False
361
362
async def call_python_tool(
@@ -665,11 +665,15 @@ class HarmonyContext(ConversationContext):
665
def need_builtin_tool_call(self) -> bool:
666
last_msg = self.messages[-1]
667
recipient = last_msg.recipient
668
- return recipient is not None and (
669
- recipient.startswith("browser.")
670
- or recipient.startswith("python")
671
- or recipient.startswith("container.")
672
- )
668
+ if recipient is None:
669
+ return False
670
+ if recipient.startswith("browser."):
671
+ return "browser" in self.available_tools
672
+ if recipient.startswith("python"):
673
+ return "python" in self.available_tools
674
+ if recipient.startswith("container."):
675
+ return "container" in self.available_tools
676
+ return False
677
678
async def call_tool(self) -> list[Message]:
679
if not self.messages:
vllm/entrypoints/openai/responses/serving.py
+31
-7
@@ -392,13 +392,27 @@ class OpenAIServingResponses(OpenAIServing):
392
max_model_len = self.model_config.max_model_len
393
generators: list[AsyncGenerator[ConversationContext, None]] = []
394
395
+ # Only include builtin tools that the request actually asked for.
396
+ # Without this filter, tools registered on the server (e.g. via
397
+ # --tool-server demo) would be available for execution even when
398
+ # the request didn't enable them.
399
+ requested_tool_types = extract_tool_types(request.tools)
400
builtin_tool_list: list[str] = []
401
if self.tool_server is not None:
397
- if self.tool_server.has_tool("browser"):
402
+ if (
403
+ self.tool_server.has_tool("browser")
404
+ and "web_search_preview" in requested_tool_types
405
+ ):
406
builtin_tool_list.append("browser")
399
- if self.tool_server.has_tool("python"):
407
+ if (
408
+ self.tool_server.has_tool("python")
409
+ and "code_interpreter" in requested_tool_types
410
+ ):
411
builtin_tool_list.append("python")
401
- if self.tool_server.has_tool("container"):
412
+ if (
413
+ self.tool_server.has_tool("container")
414
+ and "container" in requested_tool_types
415
+ ):
416
builtin_tool_list.append("container")
417
418
if self.tool_server is not None:
@@ -1049,9 +1063,15 @@ class OpenAIServingResponses(OpenAIServing):
1063
# FIXME(woosuk): Currently, request params like reasoning and
1064
# instructions are ignored.
1065
prev_msgs = self.msg_store[prev_response.id]
1052
- # Remove the previous chain-of-thoughts if there is a new "final"
1053
- # message. Note that this also removes these messages from the
1054
- # msg_store.
1066
+
1067
+ # FIXME(woosuk): The slice-delete-reappend cycle below is
1068
+ # currently a no-op --- it removes messages then puts them all
1069
+ # back unfiltered. It may be intentionally deferred (see FIXME
1070
+ # above) or redundant if the Harmony encoder already strips
1071
+ # analysis messages at render time. If analysis messages need
1072
+ # to be dropped here, add a channel != "analysis" filter when
1073
+ # re-appending, similar to auto_drop_analysis_messages in
1074
+ # harmony_utils.py.
1075
if len(prev_msgs) > 0:
1076
last_msg = prev_msgs[-1]
1077
assert isinstance(last_msg, OpenAIHarmonyMessage)
@@ -1072,7 +1092,11 @@ class OpenAIServingResponses(OpenAIServing):
1092
# Append the new input.
1093
# Responses API supports simple text inputs without chat format.
1094
if isinstance(request.input, str):
1075
- messages.append(get_user_message(request.input))
1095
+ # Skip empty string input when previous_input_messages supplies
1096
+ # the full conversation history --- an empty trailing user message
1097
+ # confuses the model into thinking nothing was sent.
1098
+ if request.input or not request.previous_input_messages:
1099
+ messages.append(get_user_message(request.input))
1100
else:
1101
if prev_response is not None:
1102
prev_outputs = copy(prev_response.output)
vllm/envs.py
+7
@@ -209,6 +209,7 @@ if TYPE_CHECKING:
209
VLLM_GPT_OSS_SYSTEM_TOOL_MCP_LABELS: set[str] = set()
210
VLLM_USE_EXPERIMENTAL_PARSER_CONTEXT: bool = False
211
VLLM_GPT_OSS_HARMONY_SYSTEM_INSTRUCTIONS: bool = False
212
+ VLLM_SYSTEM_START_DATE: str | None = None
213
VLLM_TOOL_JSON_ERROR_AUTOMATIC_RETRY: bool = False
214
VLLM_CUSTOM_SCOPES_FOR_PROFILING: bool = False
215
VLLM_NVTX_SCOPES_FOR_PROFILING: bool = False
@@ -1458,6 +1459,12 @@ environment_variables: dict[str, Callable[[], Any]] = {
1459
"VLLM_GPT_OSS_HARMONY_SYSTEM_INSTRUCTIONS": lambda: bool(
1460
int(os.getenv("VLLM_GPT_OSS_HARMONY_SYSTEM_INSTRUCTIONS", "0"))
1461
),
1462
+ # Pin the conversation start date injected into the Harmony system
1463
+ # message. When unset the current date is used, which introduces
1464
+ # non-determinism (different tokens -> different model behaviour at
1465
+ # temperature=0). Set to an ISO date string, e.g. "2023-09-12",
1466
+ # for reproducible inference or testing.
1467
+ "VLLM_SYSTEM_START_DATE": lambda: os.getenv("VLLM_SYSTEM_START_DATE", None),
1468
# Enable automatic retry when tool call JSON parsing fails
1469
# If enabled, returns an error message to the model to retry
1470
# If disabled (default), raises an exception and fails the request