@tduncan / Nvidia-Optimization-Stack-VLLM / commits / e8b358b4c0

[Bugfix][Responses] Add tests for Chat Completions Responses API Render Parity (#50334)

Signed-off-by: Yifan Zong <yzong@redhat.com> Co-authored-by: Ben Browning <56071+bbrowning@users.noreply.github.com>

yzong-rh committed Jul 31, 2026 at 15:45 UTC e8b358b4c0df5cf136072747c97a95526bd59df1
8 files changed +521 -43
tests/entrypoints/openai/responses/test_parsable_context.py
+6
@@ -186,6 +186,12 @@ async def test_function_call_first_turn(client: OpenAI, model_name: str):
186
187 @pytest.mark.asyncio
188 @pytest.mark.parametrize("model_name", [MODEL_NAME])
189 +@pytest.mark.xfail(
190 + reason=(
191 + "MCP tools are not properly supported: tool name parameters "
192 + "are not extracted for prompt rendering."
193 + ),
194 +)
195 async def test_mcp_tool_call(client: OpenAI, model_name: str):
196 """MCP tool calling with code_interpreter.
197
tests/entrypoints/openai/responses/test_responses_utils.py
+1 -22
@@ -20,7 +20,6 @@ from vllm.entrypoints.openai.responses.utils import (
20 _construct_message_from_response_item,
21 construct_chat_messages_with_tool_call,
22 construct_input_messages,
23 - convert_tool_responses_to_completions_format,
23 should_continue_final_message,
24 )
25
@@ -116,27 +115,7 @@ def make_function_call_output(
115
116
117 class TestResponsesUtils:
119 - """Tests for convert_tool_responses_to_completions_format function."""
120 -
121 - def test_convert_tool_responses_to_completions_format(self):
122 - """Test basic conversion of a flat tool schema to nested format."""
123 - input_tool = {
124 - "type": "function",
125 - "name": "get_weather",
126 - "description": "Get the current weather in a given location",
127 - "parameters": {
128 - "type": "object",
129 - "properties": {
130 - "location": {"type": "string"},
131 - "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]},
132 - },
133 - "required": ["location", "unit"],
134 - },
135 - }
136 -
137 - result = convert_tool_responses_to_completions_format(input_tool)
138 -
139 - assert result == {"type": "function", "function": input_tool}
118 + """Tests for Responses API utils."""
119
120 def test_construct_chat_messages_with_tool_call(self):
121 """Test construction of chat messages with tool calls."""
tests/entrypoints/openai/test_render_parity.py new
+479
@@ -0,0 +1,479 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +"""Cross-API HF render-input parity tests.
4 +
5 +Drives the real Chat Completions and Responses prep paths and captures the
6 +arguments each passes to ``BaseRenderer.render_chat_async`` (via shared
7 +``OnlineRenderer.preprocess_chat``):
8 +
9 + - conversation messages
10 + - ChatParams fields that affect the HF template / media path
11 + (template, content format, template kwargs including tools,
12 + media_io_kwargs, mm_processor_kwargs)
13 +
14 +``TokenizeParams``, ``prompt_extras``, Harmony / GPT-OSS, prefill / continue,
15 +prompt cache salt, and truncation are out of scope for this file.
16 +"""
17 +
18 +from __future__ import annotations
19 +
20 +from dataclasses import dataclass, field
21 +from typing import Any
22 +from unittest.mock import MagicMock
23 +
24 +import pytest
25 +from openai.types.shared import Reasoning
26 +
27 +from vllm.config.multimodal import MultiModalConfig
28 +from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
29 +from vllm.entrypoints.openai.engine.protocol import ErrorResponse
30 +from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
31 +from vllm.entrypoints.openai.responses.serving import OpenAIServingResponses
32 +from vllm.inputs import tokens_input
33 +from vllm.renderers.online_renderer import OnlineRenderer
34 +from vllm.renderers.params import ChatParams
35 +
36 +_MODEL = "test-model"
37 +_USER = [{"role": "user", "content": "Hello"}]
38 +
39 +_WEATHER_PARAMETERS = {
40 + "type": "object",
41 + "properties": {"location": {"type": "string"}},
42 + "required": ["location"],
43 +}
44 +
45 +_CHAT_WEATHER_TOOL = {
46 + "type": "function",
47 + "function": {
48 + "name": "get_weather",
49 + "description": "Get the weather",
50 + "parameters": _WEATHER_PARAMETERS,
51 + },
52 +}
53 +
54 +_RESPONSES_WEATHER_TOOL = {
55 + "type": "function",
56 + "name": "get_weather",
57 + "description": "Get the weather",
58 + "parameters": _WEATHER_PARAMETERS,
59 +}
60 +
61 +
62 +@dataclass
63 +class MockHFConfig:
64 + model_type: str = "llama"
65 +
66 +
67 +@dataclass
68 +class MockModelConfig:
69 + task = "generate"
70 + runner_type = "generate"
71 + model = _MODEL
72 + tokenizer = _MODEL
73 + trust_remote_code = False
74 + tokenizer_mode = "auto"
75 + max_model_len = 100
76 + tokenizer_revision = None
77 + multimodal_config = MultiModalConfig()
78 + hf_config = MockHFConfig()
79 + hf_text_config = MockHFConfig()
80 + logits_processors: list[str] | None = None
81 + diff_sampling_param: dict | None = None
82 + allowed_local_media_path: str = ""
83 + allowed_media_domains: list[str] | None = None
84 + encoder_config = None
85 + generation_config: str = "auto"
86 + override_generation_config: dict[str, Any] = field(default_factory=dict)
87 + media_io_kwargs: dict[str, dict[str, Any]] = field(default_factory=dict)
88 + skip_tokenizer_init = False
89 + is_encoder_decoder: bool = False
90 + is_multimodal_model: bool = False
91 + renderer_num_workers: int = 1
92 + enable_prompt_embeds: bool = False
93 +
94 + def get_diff_sampling_param(self):
95 + return self.diff_sampling_param or {}
96 +
97 +
98 +@dataclass(frozen=True)
99 +class CapturedRenderInputs:
100 + """Args observed at ``render_chat_async`` (HF render boundary)."""
101 +
102 + messages: list[Any]
103 + chat_params: ChatParams
104 +
105 +
106 +class RenderCapture:
107 + """Install a ``render_chat_async`` stub and record its HF-bound inputs."""
108 +
109 + def __init__(self, online_renderer: OnlineRenderer) -> None:
110 + self.online_renderer = online_renderer
111 + self.captured: CapturedRenderInputs | None = None
112 +
113 + async def fake_render_chat_async(
114 + conversations,
115 + chat_params,
116 + tok_params=None,
117 + *,
118 + prompt_extras=None,
119 + skip_mm_cache=False,
120 + ):
121 + assert len(conversations) == 1
122 + self.captured = CapturedRenderInputs(
123 + messages=list(conversations[0]),
124 + chat_params=chat_params,
125 + )
126 + return [list(conversations[0])], [
127 + tokens_input(prompt_token_ids=[0]),
128 + ]
129 +
130 + online_renderer.renderer.render_chat_async = fake_render_chat_async
131 +
132 + def take(self) -> CapturedRenderInputs:
133 + assert self.captured is not None
134 + captured = self.captured
135 + self.captured = None
136 + return captured
137 +
138 +
139 +async def _capture_chat(
140 + online_renderer: OnlineRenderer,
141 + request: ChatCompletionRequest,
142 +) -> CapturedRenderInputs:
143 + capture = RenderCapture(online_renderer)
144 + result = await online_renderer.render_chat(request)
145 + assert not isinstance(result, ErrorResponse), result
146 + return capture.take()
147 +
148 +
149 +async def _capture_responses(
150 + serving: OpenAIServingResponses,
151 + request: ResponsesRequest,
152 +) -> CapturedRenderInputs:
153 + capture = RenderCapture(serving.online_renderer)
154 + await serving._make_request(request, prev_response=None)
155 + return capture.take()
156 +
157 +
158 +async def _assert_parity(
159 + online_renderer: OnlineRenderer,
160 + serving: OpenAIServingResponses,
161 + *,
162 + chat_kwargs: dict[str, Any],
163 + responses_kwargs: dict[str, Any],
164 +) -> None:
165 + """Build paired requests, capture HF render inputs, and assert equality."""
166 + chat_req = ChatCompletionRequest(model=_MODEL, **chat_kwargs)
167 + responses_req = ResponsesRequest(model=_MODEL, **responses_kwargs)
168 + chat = await _capture_chat(online_renderer, chat_req)
169 + responses = await _capture_responses(serving, responses_req)
170 +
171 + assert chat.messages == responses.messages
172 +
173 + chat_params = chat.chat_params
174 + responses_params = responses.chat_params
175 + assert chat_params.chat_template == responses_params.chat_template
176 + assert (
177 + chat_params.chat_template_content_format
178 + == responses_params.chat_template_content_format
179 + )
180 + assert chat_params.media_io_kwargs == responses_params.media_io_kwargs
181 + assert chat_params.mm_processor_kwargs == responses_params.mm_processor_kwargs
182 + assert dict(chat_params.chat_template_kwargs) == dict(
183 + responses_params.chat_template_kwargs
184 + )
185 +
186 +
187 +def _weather_tools(
188 + *, overrides: dict[str, Any] | None = None
189 +) -> tuple[list[dict], list[dict]]:
190 + """Return (chat_tools, responses_tools) for the shared weather function."""
191 + overrides = overrides or {}
192 + chat_tool = {
193 + "type": "function",
194 + "function": {
195 + "name": "get_weather",
196 + "description": "Get the weather",
197 + "parameters": _WEATHER_PARAMETERS,
198 + **overrides,
199 + },
200 + }
201 + responses_tool = {
202 + "type": "function",
203 + "name": "get_weather",
204 + "description": "Get the weather",
205 + "parameters": _WEATHER_PARAMETERS,
206 + **overrides,
207 + }
208 + return [chat_tool], [responses_tool]
209 +
210 +
211 +@pytest.fixture
212 +def model_config() -> MockModelConfig:
213 + return MockModelConfig()
214 +
215 +
216 +@pytest.fixture
217 +def online_renderer(model_config: MockModelConfig, request) -> OnlineRenderer:
218 + exclude_tools_when_tool_choice_none = getattr(request, "param", False)
219 + renderer = MagicMock()
220 + # Non-Mistral stub; only needed so render_chat / preprocess_chat can
221 + # inspect tokenizer type before render_chat_async is mocked.
222 + renderer.tokenizer = MagicMock()
223 +
224 + return OnlineRenderer(
225 + model_config=model_config, # type: ignore[arg-type]
226 + renderer=renderer,
227 + request_logger=None,
228 + chat_template=None,
229 + chat_template_content_format="auto",
230 + enable_auto_tools=True,
231 + tool_parser="openai",
232 + exclude_tools_when_tool_choice_none=exclude_tools_when_tool_choice_none,
233 + )
234 +
235 +
236 +@pytest.fixture
237 +def serving_responses(
238 + model_config: MockModelConfig,
239 + online_renderer: OnlineRenderer,
240 +) -> OpenAIServingResponses:
241 + engine_client = MagicMock()
242 + engine_client.model_config = model_config
243 + engine_client.renderer = online_renderer.renderer
244 + engine_client.input_processor = MagicMock()
245 + engine_client.vllm_config = MagicMock()
246 +
247 + return OpenAIServingResponses(
248 + engine_client=engine_client,
249 + models=MagicMock(),
250 + online_renderer=online_renderer,
251 + request_logger=None,
252 + chat_template=online_renderer.chat_template,
253 + chat_template_content_format=online_renderer.chat_template_content_format,
254 + enable_auto_tools=True,
255 + tool_parser="openai",
256 + )
257 +
258 +
259 +@pytest.mark.asyncio
260 +class TestConversationRenderParity:
261 + async def test_multiturn_tool_calling(self, online_renderer, serving_responses):
262 + """System/instructions, tool-call turn, and a follow-up user message."""
263 + await _assert_parity(
264 + online_renderer,
265 + serving_responses,
266 + chat_kwargs={
267 + "messages": [
268 + {"role": "system", "content": "Be helpful."},
269 + {"role": "user", "content": "Weather in NYC?"},
270 + {
271 + "role": "assistant",
272 + "tool_calls": [
273 + {
274 + "id": "call_1",
275 + "type": "function",
276 + "function": {
277 + "name": "get_weather",
278 + "arguments": '{"location":"NYC"}',
279 + },
280 + }
281 + ],
282 + },
283 + {
284 + "role": "tool",
285 + "tool_call_id": "call_1",
286 + "content": "72F",
287 + },
288 + {"role": "user", "content": "Thanks"},
289 + ],
290 + "tools": [_CHAT_WEATHER_TOOL],
291 + "tool_choice": "auto",
292 + },
293 + responses_kwargs={
294 + "instructions": "Be helpful.",
295 + "input": [
296 + {"role": "user", "content": "Weather in NYC?"},
297 + {
298 + "type": "function_call",
299 + "call_id": "call_1",
300 + "name": "get_weather",
301 + "arguments": '{"location":"NYC"}',
302 + },
303 + {
304 + "type": "function_call_output",
305 + "call_id": "call_1",
306 + "output": "72F",
307 + },
308 + {"role": "user", "content": "Thanks"},
309 + ],
310 + "tools": [_RESPONSES_WEATHER_TOOL],
311 + "tool_choice": "auto",
312 + },
313 + )
314 +
315 +
316 +@pytest.mark.asyncio
317 +class TestToolsRenderParity:
318 + @pytest.mark.parametrize(
319 + "chat_tool_choice,responses_tool_choice",
320 + [
321 + ("auto", "auto"),
322 + ("required", "required"),
323 + (
324 + {"type": "function", "function": {"name": "get_weather"}},
325 + {"type": "function", "name": "get_weather"},
326 + ),
327 + ],
328 + ids=["auto", "required", "named"],
329 + )
330 + async def test_tools_with_tool_choice(
331 + self,
332 + online_renderer,
333 + serving_responses,
334 + chat_tool_choice,
335 + responses_tool_choice,
336 + ):
337 + chat_tools, responses_tools = _weather_tools()
338 + await _assert_parity(
339 + online_renderer,
340 + serving_responses,
341 + chat_kwargs={
342 + "messages": _USER,
343 + "tools": chat_tools,
344 + "tool_choice": chat_tool_choice,
345 + },
346 + responses_kwargs={
347 + "input": _USER,
348 + "tools": responses_tools,
349 + "tool_choice": responses_tool_choice,
350 + },
351 + )
352 +
353 + @pytest.mark.parametrize(
354 + "online_renderer",
355 + [False, True],
356 + indirect=True,
357 + ids=["include_tools", "exclude_tools"],
358 + )
359 + async def test_tools_with_tool_choice_none(
360 + self, online_renderer, serving_responses
361 + ):
362 + chat_tools, responses_tools = _weather_tools()
363 + await _assert_parity(
364 + online_renderer,
365 + serving_responses,
366 + chat_kwargs={
367 + "messages": _USER,
368 + "tools": chat_tools,
369 + "tool_choice": "none",
370 + },
371 + responses_kwargs={
372 + "input": _USER,
373 + "tools": responses_tools,
374 + "tool_choice": "none",
375 + },
376 + )
377 +
378 + async def test_function_tool_optional_fields(
379 + self, online_renderer, serving_responses
380 + ):
381 + """Optional FunctionDefinition fields dump the same on both APIs."""
382 + chat_tools, responses_tools = _weather_tools(
383 + overrides={
384 + "strict": True,
385 + "defer_loading": False,
386 + "unrelated_extra": "should_be_ignored",
387 + }
388 + )
389 + await _assert_parity(
390 + online_renderer,
391 + serving_responses,
392 + chat_kwargs={
393 + "messages": _USER,
394 + "tools": chat_tools,
395 + "tool_choice": "auto",
396 + },
397 + responses_kwargs={
398 + "input": _USER,
399 + "tools": responses_tools,
400 + "tool_choice": "auto",
401 + },
402 + )
403 +
404 +
405 +@pytest.mark.asyncio
406 +class TestReasoningRenderParity:
407 + @pytest.mark.parametrize(
408 + "effort",
409 + ["none", "minimal", "low", "medium", "high", "xhigh"],
410 + )
411 + async def test_reasoning_effort(
412 + self, online_renderer, serving_responses, effort: str
413 + ):
414 + await _assert_parity(
415 + online_renderer,
416 + serving_responses,
417 + chat_kwargs={
418 + "messages": _USER,
419 + "reasoning_effort": effort,
420 + "tool_choice": "none",
421 + },
422 + responses_kwargs={
423 + "input": _USER,
424 + "reasoning": Reasoning(effort=effort),
425 + "tool_choice": "none",
426 + },
427 + )
428 +
429 + async def test_explicit_enable_thinking_not_overridden(
430 + self, online_renderer, serving_responses
431 + ):
432 + await _assert_parity(
433 + online_renderer,
434 + serving_responses,
435 + chat_kwargs={
436 + "messages": _USER,
437 + "reasoning_effort": "high",
438 + "chat_template_kwargs": {"enable_thinking": False},
439 + "tool_choice": "none",
440 + },
441 + responses_kwargs={
442 + "input": _USER,
443 + "reasoning": Reasoning(effort="high"),
444 + "chat_template_kwargs": {"enable_thinking": False},
445 + "tool_choice": "none",
446 + },
447 + )
448 +
449 +
450 +@pytest.mark.asyncio
451 +class TestTemplateKwargsRenderParity:
452 + async def test_passthrough_fields(self, online_renderer, serving_responses):
453 + """chat_template_kwargs / media_io_kwargs / mm_processor_kwargs parity."""
454 + await _assert_parity(
455 + online_renderer,
456 + serving_responses,
457 + chat_kwargs={
458 + "messages": [
459 + {"role": "system", "content": "Be helpful."},
460 + {"role": "user", "content": "Hello"},
461 + ],
462 + "tools": [_CHAT_WEATHER_TOOL],
463 + "tool_choice": "auto",
464 + "reasoning_effort": "medium",
465 + "chat_template_kwargs": {"custom_flag": True},
466 + "media_io_kwargs": {"image": {"max_pixels": 512}},
467 + "mm_processor_kwargs": {"num_crops": 2},
468 + },
469 + responses_kwargs={
470 + "instructions": "Be helpful.",
471 + "input": _USER,
472 + "tools": [_RESPONSES_WEATHER_TOOL],
473 + "tool_choice": "auto",
474 + "reasoning": Reasoning(effort="medium"),
475 + "chat_template_kwargs": {"custom_flag": True},
476 + "media_io_kwargs": {"image": {"max_pixels": 512}},
477 + "mm_processor_kwargs": {"num_crops": 2},
478 + },
479 + )
vllm/entrypoints/openai/chat_completion/protocol.py
+1
@@ -179,6 +179,7 @@ class ChatCompletionToolsParam(OpenAIBaseModel):
179 @model_serializer(mode="wrap")
180 def _serialize(self, handler):
181 data = handler(self)
182 + data = {k: v for k, v in data.items() if k in type(self).model_fields}
183 if self.defer_loading is None:
184 data.pop("defer_loading", None)
185 return data
vllm/entrypoints/openai/engine/protocol.py
+1
@@ -296,6 +296,7 @@ class FunctionDefinition(OpenAIBaseModel):
296 @model_serializer(mode="wrap")
297 def _serialize(self, handler):
298 data = handler(self)
299 + data = {k: v for k, v in data.items() if k in type(self).model_fields}
300 if self.strict is None:
301 data.pop("strict", None)
302 if self.defer_loading is None:
vllm/entrypoints/openai/responses/serving.py
+7 -1
@@ -610,7 +610,13 @@ class OpenAIServingResponses(GenerateBaseServing):
610 request: ResponsesRequest,
611 prev_response: ResponsesResponse | None,
612 ):
613 - tool_dicts = construct_tool_dicts(request.tools, request.tool_choice)
613 + tool_dicts = construct_tool_dicts(
614 + request.tools,
615 + request.tool_choice,
616 + exclude_tools_when_tool_choice_none=(
617 + self.online_renderer.exclude_tools_when_tool_choice_none
618 + ),
619 + )
620 # Construct the input messages.
621 messages = construct_input_messages(
622 request_instructions=request.instructions,
vllm/entrypoints/openai/responses/utils.py
+25 -19
@@ -31,8 +31,11 @@ from openai.types.responses.tool import Tool
31
32 from vllm import envs
33 from vllm.entrypoints.chat_utils import make_tool_call_id
34 -from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionMessageParam
35 -from vllm.entrypoints.openai.engine.protocol import FunctionCall
34 +from vllm.entrypoints.openai.chat_completion.protocol import (
35 + ChatCompletionMessageParam,
36 + ChatCompletionToolsParam,
37 +)
38 +from vllm.entrypoints.openai.engine.protocol import FunctionCall, FunctionDefinition
39 from vllm.entrypoints.openai.responses.protocol import ResponseInputOutputItem
40 from vllm.logger import init_logger
41 from vllm.tool_parsers.utils import (
@@ -363,27 +366,30 @@ def extract_tool_types(tools: list[Tool]) -> set[str]:
366 return tool_types
367
368
366 -def convert_tool_responses_to_completions_format(tool: dict) -> dict:
369 +def convert_tool_responses_to_completions_format(
370 + tool: dict,
371 +) -> ChatCompletionToolsParam:
372 """
368 - Convert a flat tool schema:
373 + Convert a flat Responses tool schema:
374 {"type": "function", "name": "...", "description": "...", "parameters": {...}}
370 - into:
371 - {"type": "function", "function": {...}}
375 + into a Chat Completions tool param for chat-template rendering.
376 """
373 - return {
374 - "type": "function",
375 - "function": tool,
376 - }
377 + return ChatCompletionToolsParam(
378 + type="function",
379 + function=FunctionDefinition.model_validate(
380 + {k: v for k, v in tool.items() if k != "type"}
381 + ),
382 + )
383
384
385 def construct_tool_dicts(
380 - tools: list[Tool], tool_choice: ToolChoice
386 + tools: list[Tool],
387 + tool_choice: ToolChoice,
388 + exclude_tools_when_tool_choice_none: bool = False,
389 ) -> list[dict[str, Any]] | None:
382 - if not tools or (tool_choice == "none"):
383 - tool_dicts = None
384 - else:
385 - tool_dicts = [
386 - convert_tool_responses_to_completions_format(tool)
387 - for tool in iter_response_function_tool_dicts(tools)
388 - ]
389 - return tool_dicts
390 + if not tools or (tool_choice == "none" and exclude_tools_when_tool_choice_none):
391 + return None
392 + return [
393 + convert_tool_responses_to_completions_format(tool).model_dump()
394 + for tool in iter_response_function_tool_dicts(tools)
395 + ]
vllm/tool_parsers/utils.py
+1 -1
@@ -215,7 +215,7 @@ def iter_response_function_tool_dicts(
215 namespace, namespaced_tool.name
216 )
217 function_tools.append(tool_dict)
218 - else:
218 + elif isinstance(tool, FunctionTool):
219 function_tools.append(tool.model_dump())
220 return function_tools
221