| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | """Cross-API HF render-input parity tests. |
| 4 | |
| 5 | Drives the real Chat Completions and Responses prep paths and captures the |
| 6 | arguments each passes to ``BaseRenderer.render_chat_async`` (via shared |
| 7 | ``OnlineRenderer.preprocess_chat``): |
| 8 | |
| 9 | - conversation messages |
| 10 | - ChatParams fields that affect the HF template / media path |
| 11 | (template, content format, template kwargs including tools, |
| 12 | media_io_kwargs, mm_processor_kwargs) |
| 13 | |
| 14 | ``TokenizeParams``, ``prompt_extras``, Harmony / GPT-OSS, prefill / continue, |
| 15 | prompt cache salt, and truncation are out of scope for this file. |
| 16 | """ |
| 17 | |
| 18 | from __future__ import annotations |
| 19 | |
| 20 | from dataclasses import dataclass, field |
| 21 | from typing import Any |
| 22 | from unittest.mock import MagicMock |
| 23 | |
| 24 | import pytest |
| 25 | from openai.types.shared import Reasoning |
| 26 | |
| 27 | from vllm.config.multimodal import MultiModalConfig |
| 28 | from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest |
| 29 | from vllm.entrypoints.openai.responses.protocol import ResponsesRequest |
| 30 | from vllm.entrypoints.openai.responses.serving import OpenAIServingResponses |
| 31 | from vllm.entrypoints.serve.engine.protocol import ErrorResponse |
| 32 | from vllm.exceptions import VLLMValidationError |
| 33 | from vllm.inputs import tokens_input |
| 34 | from vllm.renderers.online_renderer import OnlineRenderer |
| 35 | from vllm.renderers.params import ChatParams |
| 36 | |
| 37 | _MODEL = "test-model" |
| 38 | _USER = [{"role": "user", "content": "Hello"}] |
| 39 | |
| 40 | _WEATHER_PARAMETERS = { |
| 41 | "type": "object", |
| 42 | "properties": {"location": {"type": "string"}}, |
| 43 | "required": ["location"], |
| 44 | } |
| 45 | |
| 46 | _CHAT_WEATHER_TOOL = { |
| 47 | "type": "function", |
| 48 | "function": { |
| 49 | "name": "get_weather", |
| 50 | "description": "Get the weather", |
| 51 | "parameters": _WEATHER_PARAMETERS, |
| 52 | }, |
| 53 | } |
| 54 | |
| 55 | _RESPONSES_WEATHER_TOOL = { |
| 56 | "type": "function", |
| 57 | "name": "get_weather", |
| 58 | "description": "Get the weather", |
| 59 | "parameters": _WEATHER_PARAMETERS, |
| 60 | } |
| 61 | |
| 62 | |
| 63 | @dataclass |
| 64 | class MockHFConfig: |
| 65 | model_type: str = "llama" |
| 66 | |
| 67 | |
| 68 | @dataclass |
| 69 | class MockModelConfig: |
| 70 | task = "generate" |
| 71 | runner_type = "generate" |
| 72 | model = _MODEL |
| 73 | tokenizer = _MODEL |
| 74 | trust_remote_code = False |
| 75 | tokenizer_mode = "auto" |
| 76 | max_model_len = 100 |
| 77 | tokenizer_revision = None |
| 78 | multimodal_config = MultiModalConfig() |
| 79 | hf_config = MockHFConfig() |
| 80 | hf_text_config = MockHFConfig() |
| 81 | logits_processors: list[str] | None = None |
| 82 | diff_sampling_param: dict | None = None |
| 83 | allowed_local_media_path: str = "" |
| 84 | allowed_media_domains: list[str] | None = None |
| 85 | encoder_config = None |
| 86 | generation_config: str = "auto" |
| 87 | override_generation_config: dict[str, Any] = field(default_factory=dict) |
| 88 | media_io_kwargs: dict[str, dict[str, Any]] = field(default_factory=dict) |
| 89 | skip_tokenizer_init: bool = False |
| 90 | is_encoder_decoder: bool = False |
| 91 | is_multimodal_model: bool = False |
| 92 | supports_multimodal_inputs: bool = False |
| 93 | renderer_num_workers: int = 1 |
| 94 | enable_prompt_embeds: bool = False |
| 95 | |
| 96 | def get_diff_sampling_param(self): |
| 97 | return self.diff_sampling_param or {} |
| 98 | |
| 99 | |
| 100 | @dataclass(frozen=True) |
| 101 | class CapturedRenderInputs: |
| 102 | """Args observed at ``render_chat_async`` (HF render boundary).""" |
| 103 | |
| 104 | messages: list[Any] |
| 105 | chat_params: ChatParams |
| 106 | |
| 107 | |
| 108 | class RenderCapture: |
| 109 | """Install a ``render_chat_async`` stub and record its HF-bound inputs.""" |
| 110 | |
| 111 | def __init__(self, online_renderer: OnlineRenderer) -> None: |
| 112 | self.online_renderer = online_renderer |
| 113 | self.captured: CapturedRenderInputs | None = None |
| 114 | |
| 115 | async def fake_render_chat_async( |
| 116 | conversations, |
| 117 | chat_params, |
| 118 | tok_params=None, |
| 119 | *, |
| 120 | prompt_extras=None, |
| 121 | skip_mm_cache=False, |
| 122 | ): |
| 123 | assert len(conversations) == 1 |
| 124 | self.captured = CapturedRenderInputs( |
| 125 | messages=list(conversations[0]), |
| 126 | chat_params=chat_params, |
| 127 | ) |
| 128 | return [list(conversations[0])], [ |
| 129 | tokens_input(prompt_token_ids=[0]), |
| 130 | ] |
| 131 | |
| 132 | online_renderer.renderer.render_chat_async = fake_render_chat_async |
| 133 | |
| 134 | def take(self) -> CapturedRenderInputs: |
| 135 | assert self.captured is not None |
| 136 | captured = self.captured |
| 137 | self.captured = None |
| 138 | return captured |
| 139 | |
| 140 | |
| 141 | async def _capture_chat( |
| 142 | online_renderer: OnlineRenderer, |
| 143 | request: ChatCompletionRequest, |
| 144 | ) -> CapturedRenderInputs: |
| 145 | capture = RenderCapture(online_renderer) |
| 146 | result = await online_renderer.render_chat(request) |
| 147 | assert not isinstance(result, ErrorResponse), result |
| 148 | return capture.take() |
| 149 | |
| 150 | |
| 151 | async def _capture_responses( |
| 152 | serving: OpenAIServingResponses, |
| 153 | request: ResponsesRequest, |
| 154 | ) -> CapturedRenderInputs: |
| 155 | capture = RenderCapture(serving.online_renderer) |
| 156 | result = await serving.online_renderer.render_responses(request) |
| 157 | assert not isinstance(result, ErrorResponse), result |
| 158 | return capture.take() |
| 159 | |
| 160 | |
| 161 | async def _assert_parity( |
| 162 | online_renderer: OnlineRenderer, |
| 163 | serving: OpenAIServingResponses, |
| 164 | *, |
| 165 | chat_kwargs: dict[str, Any], |
| 166 | responses_kwargs: dict[str, Any], |
| 167 | ) -> None: |
| 168 | """Build paired requests, capture HF render inputs, and assert equality.""" |
| 169 | chat_req = ChatCompletionRequest(model=_MODEL, **chat_kwargs) |
| 170 | responses_req = ResponsesRequest(model=_MODEL, **responses_kwargs) |
| 171 | chat = await _capture_chat(online_renderer, chat_req) |
| 172 | responses = await _capture_responses(serving, responses_req) |
| 173 | |
| 174 | assert chat.messages == responses.messages |
| 175 | |
| 176 | chat_params = chat.chat_params |
| 177 | responses_params = responses.chat_params |
| 178 | assert chat_params.chat_template == responses_params.chat_template |
| 179 | assert ( |
| 180 | chat_params.chat_template_content_format |
| 181 | == responses_params.chat_template_content_format |
| 182 | ) |
| 183 | assert chat_params.media_io_kwargs == responses_params.media_io_kwargs |
| 184 | assert chat_params.mm_processor_kwargs == responses_params.mm_processor_kwargs |
| 185 | assert dict(chat_params.chat_template_kwargs) == dict( |
| 186 | responses_params.chat_template_kwargs |
| 187 | ) |
| 188 | |
| 189 | |
| 190 | def _weather_tools( |
| 191 | *, overrides: dict[str, Any] | None = None |
| 192 | ) -> tuple[list[dict], list[dict]]: |
| 193 | """Return (chat_tools, responses_tools) for the shared weather function.""" |
| 194 | overrides = overrides or {} |
| 195 | chat_tool = { |
| 196 | "type": "function", |
| 197 | "function": { |
| 198 | "name": "get_weather", |
| 199 | "description": "Get the weather", |
| 200 | "parameters": _WEATHER_PARAMETERS, |
| 201 | **overrides, |
| 202 | }, |
| 203 | } |
| 204 | responses_tool = { |
| 205 | "type": "function", |
| 206 | "name": "get_weather", |
| 207 | "description": "Get the weather", |
| 208 | "parameters": _WEATHER_PARAMETERS, |
| 209 | **overrides, |
| 210 | } |
| 211 | return [chat_tool], [responses_tool] |
| 212 | |
| 213 | |
| 214 | @pytest.fixture |
| 215 | def model_config() -> MockModelConfig: |
| 216 | return MockModelConfig() |
| 217 | |
| 218 | |
| 219 | @pytest.fixture |
| 220 | def online_renderer(model_config: MockModelConfig, request) -> OnlineRenderer: |
| 221 | exclude_tools_when_tool_choice_none = getattr(request, "param", False) |
| 222 | renderer = MagicMock() |
| 223 | # Non-Mistral stub; only needed so render_chat / preprocess_chat can |
| 224 | # inspect tokenizer type before render_chat_async is mocked. |
| 225 | renderer.tokenizer = MagicMock() |
| 226 | |
| 227 | return OnlineRenderer( |
| 228 | model_config=model_config, |
| 229 | renderer=renderer, |
| 230 | request_logger=None, |
| 231 | chat_template=None, |
| 232 | chat_template_content_format="auto", |
| 233 | enable_auto_tools=True, |
| 234 | tool_parser="openai", |
| 235 | exclude_tools_when_tool_choice_none=exclude_tools_when_tool_choice_none, |
| 236 | trust_request_mm_kwargs=True, |
| 237 | ) |
| 238 | |
| 239 | |
| 240 | @pytest.fixture |
| 241 | def serving_responses( |
| 242 | model_config: MockModelConfig, |
| 243 | online_renderer: OnlineRenderer, |
| 244 | ) -> OpenAIServingResponses: |
| 245 | engine_client = MagicMock() |
| 246 | engine_client.model_config = model_config |
| 247 | engine_client.renderer = online_renderer.renderer |
| 248 | engine_client.input_processor = MagicMock() |
| 249 | engine_client.vllm_config = MagicMock() |
| 250 | |
| 251 | return OpenAIServingResponses( |
| 252 | engine_client=engine_client, |
| 253 | models=MagicMock(), |
| 254 | online_renderer=online_renderer, |
| 255 | request_logger=None, |
| 256 | chat_template=online_renderer.chat_template, |
| 257 | chat_template_content_format=online_renderer.chat_template_content_format, |
| 258 | enable_auto_tools=True, |
| 259 | tool_parser="openai", |
| 260 | ) |
| 261 | |
| 262 | |
| 263 | @pytest.mark.asyncio |
| 264 | class TestConversationRenderParity: |
| 265 | async def test_multiturn_tool_calling(self, online_renderer, serving_responses): |
| 266 | """System/instructions, tool-call turn, and a follow-up user message.""" |
| 267 | await _assert_parity( |
| 268 | online_renderer, |
| 269 | serving_responses, |
| 270 | chat_kwargs={ |
| 271 | "messages": [ |
| 272 | {"role": "system", "content": "Be helpful."}, |
| 273 | {"role": "user", "content": "Weather in NYC?"}, |
| 274 | { |
| 275 | "role": "assistant", |
| 276 | "tool_calls": [ |
| 277 | { |
| 278 | "id": "call_1", |
| 279 | "type": "function", |
| 280 | "function": { |
| 281 | "name": "get_weather", |
| 282 | "arguments": '{"location":"NYC"}', |
| 283 | }, |
| 284 | } |
| 285 | ], |
| 286 | }, |
| 287 | { |
| 288 | "role": "tool", |
| 289 | "tool_call_id": "call_1", |
| 290 | "content": "72F", |
| 291 | }, |
| 292 | {"role": "user", "content": "Thanks"}, |
| 293 | ], |
| 294 | "tools": [_CHAT_WEATHER_TOOL], |
| 295 | "tool_choice": "auto", |
| 296 | }, |
| 297 | responses_kwargs={ |
| 298 | "instructions": "Be helpful.", |
| 299 | "input": [ |
| 300 | {"role": "user", "content": "Weather in NYC?"}, |
| 301 | { |
| 302 | "type": "function_call", |
| 303 | "call_id": "call_1", |
| 304 | "name": "get_weather", |
| 305 | "arguments": '{"location":"NYC"}', |
| 306 | }, |
| 307 | { |
| 308 | "type": "function_call_output", |
| 309 | "call_id": "call_1", |
| 310 | "output": "72F", |
| 311 | }, |
| 312 | {"role": "user", "content": "Thanks"}, |
| 313 | ], |
| 314 | "tools": [_RESPONSES_WEATHER_TOOL], |
| 315 | "tool_choice": "auto", |
| 316 | }, |
| 317 | ) |
| 318 | |
| 319 | |
| 320 | @pytest.mark.asyncio |
| 321 | class TestToolsRenderParity: |
| 322 | @pytest.mark.parametrize( |
| 323 | "chat_tool_choice,responses_tool_choice", |
| 324 | [ |
| 325 | ("auto", "auto"), |
| 326 | ("required", "required"), |
| 327 | ( |
| 328 | {"type": "function", "function": {"name": "get_weather"}}, |
| 329 | {"type": "function", "name": "get_weather"}, |
| 330 | ), |
| 331 | ], |
| 332 | ids=["auto", "required", "named"], |
| 333 | ) |
| 334 | async def test_tools_with_tool_choice( |
| 335 | self, |
| 336 | online_renderer, |
| 337 | serving_responses, |
| 338 | chat_tool_choice, |
| 339 | responses_tool_choice, |
| 340 | ): |
| 341 | chat_tools, responses_tools = _weather_tools() |
| 342 | await _assert_parity( |
| 343 | online_renderer, |
| 344 | serving_responses, |
| 345 | chat_kwargs={ |
| 346 | "messages": _USER, |
| 347 | "tools": chat_tools, |
| 348 | "tool_choice": chat_tool_choice, |
| 349 | }, |
| 350 | responses_kwargs={ |
| 351 | "input": _USER, |
| 352 | "tools": responses_tools, |
| 353 | "tool_choice": responses_tool_choice, |
| 354 | }, |
| 355 | ) |
| 356 | |
| 357 | @pytest.mark.parametrize( |
| 358 | "online_renderer", |
| 359 | [False, True], |
| 360 | indirect=True, |
| 361 | ids=["include_tools", "exclude_tools"], |
| 362 | ) |
| 363 | async def test_tools_with_tool_choice_none( |
| 364 | self, online_renderer, serving_responses |
| 365 | ): |
| 366 | chat_tools, responses_tools = _weather_tools() |
| 367 | await _assert_parity( |
| 368 | online_renderer, |
| 369 | serving_responses, |
| 370 | chat_kwargs={ |
| 371 | "messages": _USER, |
| 372 | "tools": chat_tools, |
| 373 | "tool_choice": "none", |
| 374 | }, |
| 375 | responses_kwargs={ |
| 376 | "input": _USER, |
| 377 | "tools": responses_tools, |
| 378 | "tool_choice": "none", |
| 379 | }, |
| 380 | ) |
| 381 | |
| 382 | async def test_function_tool_optional_fields( |
| 383 | self, online_renderer, serving_responses |
| 384 | ): |
| 385 | """Optional FunctionDefinition fields dump the same on both APIs.""" |
| 386 | chat_tools, responses_tools = _weather_tools( |
| 387 | overrides={ |
| 388 | "strict": True, |
| 389 | "defer_loading": False, |
| 390 | "unrelated_extra": "should_be_ignored", |
| 391 | } |
| 392 | ) |
| 393 | await _assert_parity( |
| 394 | online_renderer, |
| 395 | serving_responses, |
| 396 | chat_kwargs={ |
| 397 | "messages": _USER, |
| 398 | "tools": chat_tools, |
| 399 | "tool_choice": "auto", |
| 400 | }, |
| 401 | responses_kwargs={ |
| 402 | "input": _USER, |
| 403 | "tools": responses_tools, |
| 404 | "tool_choice": "auto", |
| 405 | }, |
| 406 | ) |
| 407 | |
| 408 | |
| 409 | @pytest.mark.asyncio |
| 410 | class TestReasoningRenderParity: |
| 411 | @pytest.mark.parametrize( |
| 412 | "effort", |
| 413 | ["none", "minimal", "low", "medium", "high", "xhigh"], |
| 414 | ) |
| 415 | async def test_reasoning_effort( |
| 416 | self, online_renderer, serving_responses, effort: str |
| 417 | ): |
| 418 | await _assert_parity( |
| 419 | online_renderer, |
| 420 | serving_responses, |
| 421 | chat_kwargs={ |
| 422 | "messages": _USER, |
| 423 | "reasoning_effort": effort, |
| 424 | "tool_choice": "none", |
| 425 | }, |
| 426 | responses_kwargs={ |
| 427 | "input": _USER, |
| 428 | "reasoning": Reasoning(effort=effort), |
| 429 | "tool_choice": "none", |
| 430 | }, |
| 431 | ) |
| 432 | |
| 433 | async def test_reasoning_effort_none_rejected_by_harmony(self, online_renderer): |
| 434 | """Harmony rejects reasoning_effort="none" as invalid user input.""" |
| 435 | request = ChatCompletionRequest( |
| 436 | model=_MODEL, |
| 437 | messages=_USER, |
| 438 | reasoning_effort="none", |
| 439 | ) |
| 440 | with pytest.raises( |
| 441 | VLLMValidationError, match="Harmony does not support" |
| 442 | ) as exc_info: |
| 443 | online_renderer._make_request_with_harmony(request) |
| 444 | assert exc_info.value.parameter == "reasoning_effort" |
| 445 | |
| 446 | async def test_explicit_enable_thinking_not_overridden( |
| 447 | self, online_renderer, serving_responses |
| 448 | ): |
| 449 | await _assert_parity( |
| 450 | online_renderer, |
| 451 | serving_responses, |
| 452 | chat_kwargs={ |
| 453 | "messages": _USER, |
| 454 | "reasoning_effort": "high", |
| 455 | "chat_template_kwargs": {"enable_thinking": False}, |
| 456 | "tool_choice": "none", |
| 457 | }, |
| 458 | responses_kwargs={ |
| 459 | "input": _USER, |
| 460 | "reasoning": Reasoning(effort="high"), |
| 461 | "chat_template_kwargs": {"enable_thinking": False}, |
| 462 | "tool_choice": "none", |
| 463 | }, |
| 464 | ) |
| 465 | |
| 466 | |
| 467 | @pytest.mark.asyncio |
| 468 | class TestTemplateKwargsRenderParity: |
| 469 | async def test_passthrough_fields(self, online_renderer, serving_responses): |
| 470 | """chat_template_kwargs / media_io_kwargs / mm_processor_kwargs parity.""" |
| 471 | await _assert_parity( |
| 472 | online_renderer, |
| 473 | serving_responses, |
| 474 | chat_kwargs={ |
| 475 | "messages": [ |
| 476 | {"role": "system", "content": "Be helpful."}, |
| 477 | {"role": "user", "content": "Hello"}, |
| 478 | ], |
| 479 | "tools": [_CHAT_WEATHER_TOOL], |
| 480 | "tool_choice": "auto", |
| 481 | "reasoning_effort": "medium", |
| 482 | "chat_template_kwargs": {"custom_flag": True}, |
| 483 | "media_io_kwargs": {"image": {"max_pixels": 512}}, |
| 484 | "mm_processor_kwargs": {"num_crops": 2}, |
| 485 | }, |
| 486 | responses_kwargs={ |
| 487 | "instructions": "Be helpful.", |
| 488 | "input": _USER, |
| 489 | "tools": [_RESPONSES_WEATHER_TOOL], |
| 490 | "tool_choice": "auto", |
| 491 | "reasoning": Reasoning(effort="medium"), |
| 492 | "chat_template_kwargs": {"custom_flag": True}, |
| 493 | "media_io_kwargs": {"image": {"max_pixels": 512}}, |
| 494 | "mm_processor_kwargs": {"num_crops": 2}, |
| 495 | }, |
| 496 | ) |