| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | """Unit tests for ParsableContext's parsing behavior. |
| 4 | |
| 5 | These tests verify that ParsableContext correctly delegates to the unified |
| 6 | Parser (via parse) and properly builds response output items. |
| 7 | """ |
| 8 | |
| 9 | import json |
| 10 | from collections.abc import Sequence |
| 11 | from types import SimpleNamespace |
| 12 | from unittest.mock import AsyncMock, MagicMock |
| 13 | |
| 14 | import pytest |
| 15 | from openai.types.responses import ResponseFunctionToolCall |
| 16 | |
| 17 | from vllm.entrypoints.generate.base.protocol import ( |
| 18 | DeltaMessage, |
| 19 | ExtractedToolCallInformation, |
| 20 | FunctionCall, |
| 21 | ToolCall, |
| 22 | ) |
| 23 | from vllm.entrypoints.mcp.tool import HarmonyPythonTool |
| 24 | from vllm.entrypoints.openai.responses.context import ParsableContext |
| 25 | from vllm.entrypoints.openai.responses.protocol import ResponsesRequest |
| 26 | from vllm.outputs import CompletionOutput, RequestOutput |
| 27 | from vllm.parser.abstract_parser import DelegatingParser |
| 28 | |
| 29 | pytestmark = pytest.mark.skip_global_cleanup |
| 30 | |
| 31 | |
| 32 | # --------------------------------------------------------------------------- |
| 33 | # Test parser stubs |
| 34 | # --------------------------------------------------------------------------- |
| 35 | |
| 36 | |
| 37 | class _NoOpParser(DelegatingParser): |
| 38 | """Parser that extracts no reasoning and no tool calls.""" |
| 39 | |
| 40 | def is_reasoning_end(self, input_ids: list[int]) -> bool: |
| 41 | return False |
| 42 | |
| 43 | def extract_reasoning(self, model_output, request): |
| 44 | return None, model_output |
| 45 | |
| 46 | def extract_reasoning_streaming(self, *args, **kwargs): |
| 47 | return None |
| 48 | |
| 49 | def extract_tool_calls(self, model_output, request): |
| 50 | return ExtractedToolCallInformation( |
| 51 | tools_called=False, tool_calls=[], content=model_output |
| 52 | ) |
| 53 | |
| 54 | def extract_tool_calls_streaming(self, *args, **kwargs): |
| 55 | return None |
| 56 | |
| 57 | def parse_delta(self, *args, **kwargs) -> DeltaMessage | None: |
| 58 | return None |
| 59 | |
| 60 | |
| 61 | class _ReasoningOnlyParser(DelegatingParser): |
| 62 | """Parser that extracts reasoning but no tool calls.""" |
| 63 | |
| 64 | def is_reasoning_end(self, input_ids: list[int]) -> bool: |
| 65 | return False |
| 66 | |
| 67 | def extract_reasoning(self, model_output, request): |
| 68 | if "<think>" in model_output and "</think>" in model_output: |
| 69 | start = model_output.index("<think>") + len("<think>") |
| 70 | end = model_output.index("</think>") |
| 71 | reasoning = model_output[start:end] |
| 72 | content = model_output[end + len("</think>") :] |
| 73 | return reasoning, content.strip() or None |
| 74 | return None, model_output |
| 75 | |
| 76 | def extract_reasoning_streaming(self, *args, **kwargs): |
| 77 | return None |
| 78 | |
| 79 | def extract_tool_calls(self, model_output, request): |
| 80 | return ExtractedToolCallInformation( |
| 81 | tools_called=False, tool_calls=[], content=model_output |
| 82 | ) |
| 83 | |
| 84 | def extract_tool_calls_streaming(self, *args, **kwargs): |
| 85 | return None |
| 86 | |
| 87 | def parse_delta(self, *args, **kwargs) -> DeltaMessage | None: |
| 88 | return None |
| 89 | |
| 90 | |
| 91 | class _CountingParser(_NoOpParser): |
| 92 | """Parser that records the ids it counts; token id 7 is reasoning.""" |
| 93 | |
| 94 | def __init__(self, tokenizer, *args, **kwargs): |
| 95 | super().__init__(tokenizer) |
| 96 | self.counted_ids: list[list[int]] = [] |
| 97 | |
| 98 | def count_reasoning_tokens(self, token_ids: Sequence[int]) -> int: |
| 99 | self.counted_ids.append(list(token_ids)) |
| 100 | return sum(token_id == 7 for token_id in token_ids) |
| 101 | |
| 102 | |
| 103 | class _StubToolParser: |
| 104 | """Minimal tool parser stub that always returns a hardcoded tool call.""" |
| 105 | |
| 106 | supports_required_and_named = False |
| 107 | |
| 108 | def __init__(self, tokenizer=None, tools=None): |
| 109 | pass |
| 110 | |
| 111 | def extract_tool_calls(self, model_output, request): |
| 112 | return ExtractedToolCallInformation( |
| 113 | tools_called=True, |
| 114 | tool_calls=[ |
| 115 | ToolCall( |
| 116 | id="call_123", |
| 117 | type="function", |
| 118 | function=FunctionCall( |
| 119 | name="get_weather", |
| 120 | arguments='{"location": "Paris"}', |
| 121 | ), |
| 122 | ) |
| 123 | ], |
| 124 | content=None, |
| 125 | ) |
| 126 | |
| 127 | def extract_tool_calls_streaming(self, *args, **kwargs): |
| 128 | return None |
| 129 | |
| 130 | def adjust_request(self, request): |
| 131 | return request |
| 132 | |
| 133 | |
| 134 | class _ToolCallingParser(DelegatingParser): |
| 135 | """Parser that extracts a hardcoded tool call from any input.""" |
| 136 | |
| 137 | def __init__(self, tokenizer, *args, **kwargs): |
| 138 | super().__init__(tokenizer) |
| 139 | self._tool_parser = _StubToolParser() |
| 140 | |
| 141 | def is_reasoning_end(self, input_ids: list[int]) -> bool: |
| 142 | return False |
| 143 | |
| 144 | def extract_reasoning(self, model_output, request): |
| 145 | return None, model_output |
| 146 | |
| 147 | def extract_reasoning_streaming(self, *args, **kwargs): |
| 148 | return None |
| 149 | |
| 150 | def extract_tool_calls_streaming(self, *args, **kwargs): |
| 151 | return None |
| 152 | |
| 153 | def parse_delta(self, *args, **kwargs) -> DeltaMessage | None: |
| 154 | return None |
| 155 | |
| 156 | |
| 157 | # --------------------------------------------------------------------------- |
| 158 | # Helpers |
| 159 | # --------------------------------------------------------------------------- |
| 160 | |
| 161 | |
| 162 | def _make_request(**overrides) -> ResponsesRequest: |
| 163 | defaults = {"model": "test-model", "input": "test"} |
| 164 | defaults.update(overrides) |
| 165 | return ResponsesRequest.model_validate(defaults) |
| 166 | |
| 167 | |
| 168 | def _make_request_output( |
| 169 | text: str = "Hello, world!", |
| 170 | token_ids: Sequence[int] = (1, 2, 3), |
| 171 | finish_reason: str = "stop", |
| 172 | ) -> RequestOutput: |
| 173 | return RequestOutput( |
| 174 | request_id="test", |
| 175 | prompt=None, |
| 176 | prompt_token_ids=[], |
| 177 | prompt_logprobs=None, |
| 178 | outputs=[ |
| 179 | CompletionOutput( |
| 180 | index=0, |
| 181 | text=text, |
| 182 | token_ids=list(token_ids), |
| 183 | cumulative_logprob=None, |
| 184 | logprobs=None, |
| 185 | finish_reason=finish_reason, |
| 186 | ) |
| 187 | ], |
| 188 | finished=True, |
| 189 | ) |
| 190 | |
| 191 | |
| 192 | def _make_context(parser_cls, **overrides): |
| 193 | # ParsableContext no longer lazily builds a parser from ``parser_cls``; |
| 194 | # the caller (here, the serving layer in production) must supply one. |
| 195 | request = overrides.get("request", _make_request()) |
| 196 | response_parser = overrides.pop("response_parser", None) |
| 197 | if response_parser is None and parser_cls is not None: |
| 198 | response_parser = parser_cls(MagicMock(), request.tools) |
| 199 | |
| 200 | defaults = dict( |
| 201 | tokenizer=MagicMock(), |
| 202 | parser_cls=parser_cls, |
| 203 | response_parser=response_parser, |
| 204 | response_messages=[], |
| 205 | request=request, |
| 206 | available_tools=None, |
| 207 | chat_template=None, |
| 208 | chat_template_content_format="auto", |
| 209 | ) |
| 210 | defaults.update(overrides) |
| 211 | return ParsableContext(**defaults) |
| 212 | |
| 213 | |
| 214 | # --------------------------------------------------------------------------- |
| 215 | # Tests: basic text passthrough |
| 216 | # --------------------------------------------------------------------------- |
| 217 | |
| 218 | |
| 219 | def test_process_text_with_parser(): |
| 220 | """Parser with no reasoning/tools returns a single message item.""" |
| 221 | ctx = _make_context(_NoOpParser) |
| 222 | ctx.append_output(_make_request_output(text="Hello!")) |
| 223 | |
| 224 | assert len(ctx.response_messages) == 1 |
| 225 | msg = ctx.response_messages[0] |
| 226 | assert msg.type == "message" |
| 227 | assert msg.content[0].text == "Hello!" |
| 228 | |
| 229 | |
| 230 | def test_process_text_without_parser(): |
| 231 | """parser_cls=None falls back to plain text wrapping.""" |
| 232 | ctx = _make_context(None) |
| 233 | ctx.append_output(_make_request_output(text="Hello!")) |
| 234 | |
| 235 | assert len(ctx.response_messages) == 1 |
| 236 | msg = ctx.response_messages[0] |
| 237 | assert msg.type == "message" |
| 238 | assert msg.content[0].text == "Hello!" |
| 239 | |
| 240 | |
| 241 | # --------------------------------------------------------------------------- |
| 242 | # Tests: empty / whitespace output |
| 243 | # --------------------------------------------------------------------------- |
| 244 | |
| 245 | |
| 246 | def test_process_empty_text_without_parser(): |
| 247 | """Empty text with no parser produces no output items.""" |
| 248 | ctx = _make_context(None) |
| 249 | ctx.append_output(_make_request_output(text="")) |
| 250 | |
| 251 | assert len(ctx.response_messages) == 0 |
| 252 | |
| 253 | |
| 254 | def test_process_empty_text_with_parser(): |
| 255 | """Empty text with parser produces no output items.""" |
| 256 | ctx = _make_context(_NoOpParser) |
| 257 | ctx.append_output(_make_request_output(text="")) |
| 258 | |
| 259 | assert len(ctx.response_messages) == 0 |
| 260 | |
| 261 | |
| 262 | # --------------------------------------------------------------------------- |
| 263 | # Tests: reasoning extraction |
| 264 | # --------------------------------------------------------------------------- |
| 265 | |
| 266 | |
| 267 | def test_process_extracts_reasoning(): |
| 268 | """Parser that finds reasoning produces both reasoning and message items.""" |
| 269 | ctx = _make_context(_ReasoningOnlyParser) |
| 270 | ctx.append_output( |
| 271 | _make_request_output(text="<think>Let me check</think>The answer is 42") |
| 272 | ) |
| 273 | |
| 274 | types = [m.type for m in ctx.response_messages] |
| 275 | assert "reasoning" in types |
| 276 | assert "message" in types |
| 277 | |
| 278 | reasoning_item = next(m for m in ctx.response_messages if m.type == "reasoning") |
| 279 | assert reasoning_item.content[0].text == "Let me check" |
| 280 | |
| 281 | message_item = next(m for m in ctx.response_messages if m.type == "message") |
| 282 | assert message_item.content[0].text == "The answer is 42" |
| 283 | |
| 284 | |
| 285 | def test_process_reasoning_only_no_content(): |
| 286 | """When reasoning consumes all text, only a reasoning item is produced.""" |
| 287 | ctx = _make_context(_ReasoningOnlyParser) |
| 288 | ctx.append_output(_make_request_output(text="<think>Just thinking</think>")) |
| 289 | |
| 290 | types = [m.type for m in ctx.response_messages] |
| 291 | assert "reasoning" in types |
| 292 | assert "message" not in types |
| 293 | |
| 294 | |
| 295 | # --------------------------------------------------------------------------- |
| 296 | # Tests: tool call extraction |
| 297 | # --------------------------------------------------------------------------- |
| 298 | |
| 299 | |
| 300 | def test_process_extracts_tool_calls(): |
| 301 | """Parser that finds tool calls produces function_call items.""" |
| 302 | request = _make_request( |
| 303 | tool_choice="auto", |
| 304 | tools=[ |
| 305 | { |
| 306 | "type": "function", |
| 307 | "name": "get_weather", |
| 308 | "parameters": {"type": "object", "properties": {}}, |
| 309 | } |
| 310 | ], |
| 311 | ) |
| 312 | ctx = _make_context(_ToolCallingParser, request=request, enable_auto_tools=True) |
| 313 | ctx.append_output(_make_request_output(text="calling tool")) |
| 314 | |
| 315 | types = [m.type for m in ctx.response_messages] |
| 316 | assert "function_call" in types |
| 317 | |
| 318 | tool_item = next(m for m in ctx.response_messages if m.type == "function_call") |
| 319 | assert tool_item.name == "get_weather" |
| 320 | assert tool_item.arguments == '{"location": "Paris"}' |
| 321 | assert tool_item.status == "completed" |
| 322 | |
| 323 | |
| 324 | @pytest.mark.parametrize( |
| 325 | ("tool_name", "session_name", "dispatched_name", "arguments"), |
| 326 | [ |
| 327 | ("code_interpreter", "python", "python", {"code": "print(42)"}), |
| 328 | ("web_search_preview", "browser", "search", {"query": "vLLM"}), |
| 329 | ("container.exec", "container", "exec", {"cmd": ["pwd"]}), |
| 330 | ], |
| 331 | ) |
| 332 | @pytest.mark.asyncio |
| 333 | async def test_builtin_tool_output_preserves_function_call_id( |
| 334 | tool_name, session_name, dispatched_name, arguments |
| 335 | ): |
| 336 | """Built-in tool outputs remain correlated with their originating call.""" |
| 337 | tool_session = MagicMock() |
| 338 | tool_session.call_tool = AsyncMock( |
| 339 | return_value=SimpleNamespace(content=[SimpleNamespace(text="result")]) |
| 340 | ) |
| 341 | context = _make_context(None, available_tools=[session_name]) |
| 342 | tool_call = ResponseFunctionToolCall( |
| 343 | id=f"fc_{session_name}", |
| 344 | call_id=f"call_{session_name}", |
| 345 | type="function_call", |
| 346 | name=tool_name, |
| 347 | arguments=json.dumps(arguments), |
| 348 | ) |
| 349 | context.response_messages.append(tool_call) |
| 350 | context._tool_sessions[session_name] = tool_session |
| 351 | |
| 352 | output = await context.call_tool() |
| 353 | |
| 354 | tool_session.call_tool.assert_awaited_once_with(dispatched_name, arguments) |
| 355 | assert output[0].call_id == tool_call.call_id |
| 356 | |
| 357 | |
| 358 | @pytest.mark.asyncio |
| 359 | async def test_local_python_tool_output_preserves_function_call_id(): |
| 360 | """Local code-interpreter output remains correlated with its call.""" |
| 361 | |
| 362 | async def process(_): |
| 363 | yield SimpleNamespace(content=[SimpleNamespace(text="result")]) |
| 364 | |
| 365 | python_tool = object.__new__(HarmonyPythonTool) |
| 366 | python_tool.python_tool = MagicMock(process=process) |
| 367 | context = _make_context(None, available_tools=["python"]) |
| 368 | tool_call = ResponseFunctionToolCall( |
| 369 | id="fc_python", |
| 370 | call_id="call_python", |
| 371 | type="function_call", |
| 372 | name="code_interpreter", |
| 373 | arguments='{"code": "print(42)"}', |
| 374 | ) |
| 375 | context.response_messages.append(tool_call) |
| 376 | context._tool_sessions["python"] = python_tool |
| 377 | |
| 378 | output = await context.call_tool() |
| 379 | |
| 380 | assert output[0].call_id == tool_call.call_id |
| 381 | |
| 382 | |
| 383 | # --------------------------------------------------------------------------- |
| 384 | # Tests: finish_reason tracking |
| 385 | # --------------------------------------------------------------------------- |
| 386 | |
| 387 | |
| 388 | def test_finish_reason_tracked(): |
| 389 | """finish_reason from CompletionOutput is stored on the context.""" |
| 390 | ctx = _make_context(_NoOpParser) |
| 391 | assert ctx.finish_reason is None |
| 392 | |
| 393 | ctx.append_output(_make_request_output(finish_reason="stop")) |
| 394 | assert ctx.finish_reason == "stop" |
| 395 | |
| 396 | ctx.append_output(_make_request_output(finish_reason="length")) |
| 397 | assert ctx.finish_reason == "length" |
| 398 | |
| 399 | |
| 400 | # --------------------------------------------------------------------------- |
| 401 | # Tests: multi-turn accumulation |
| 402 | # --------------------------------------------------------------------------- |
| 403 | |
| 404 | |
| 405 | def test_multi_turn_accumulation(): |
| 406 | """Multiple append_output() calls accumulate response_messages.""" |
| 407 | ctx = _make_context(_NoOpParser) |
| 408 | |
| 409 | ctx.append_output(_make_request_output(text="First turn")) |
| 410 | ctx.append_output(_make_request_output(text="Second turn")) |
| 411 | |
| 412 | assert len(ctx.response_messages) == 2 |
| 413 | texts = [m.content[0].text for m in ctx.response_messages] |
| 414 | assert texts == ["First turn", "Second turn"] |
| 415 | |
| 416 | |
| 417 | def test_reasoning_tokens_counted_per_round(): |
| 418 | """Each round is counted on its own ids, not the concatenated rounds.""" |
| 419 | ctx = _make_context(_CountingParser) |
| 420 | |
| 421 | ctx.append_output(_make_request_output(token_ids=[7, 7, 1])) |
| 422 | ctx.append_output(_make_request_output(token_ids=[7, 2])) |
| 423 | |
| 424 | assert ctx.response_parser.counted_ids == [[7, 7, 1], [7, 2]] |
| 425 | assert ctx.num_reasoning_tokens == 3 |
| 426 | |
| 427 | |
| 428 | def test_reasoning_tokens_zero_without_parser(): |
| 429 | ctx = _make_context(None) |
| 430 | |
| 431 | ctx.append_output(_make_request_output(token_ids=[7, 7, 1])) |
| 432 | |
| 433 | assert ctx.num_reasoning_tokens == 0 |
| 434 | |
| 435 | |
| 436 | def test_num_init_messages_offset(): |
| 437 | """Initial messages are preserved and offset works correctly.""" |
| 438 | init_messages = [MagicMock(type="message")] |
| 439 | ctx = _make_context(_NoOpParser, response_messages=init_messages) |
| 440 | |
| 441 | assert ctx.num_init_messages == 1 |
| 442 | |
| 443 | ctx.append_output(_make_request_output(text="New output")) |
| 444 | |
| 445 | assert len(ctx.response_messages) == 2 |
| 446 | items = ctx.make_response_output_items() |
| 447 | assert len(items) == 1 |
| 448 | assert items[0].type == "message" |