| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | """Tests for include_reasoning suppression in the unified Parser interface. |
| 4 | |
| 5 | Covers non-streaming (parser.parse() + build_response_output_items), |
| 6 | streaming (parse_delta), and ParsableContext.append_output() paths. |
| 7 | """ |
| 8 | |
| 9 | import json |
| 10 | |
| 11 | import pytest |
| 12 | |
| 13 | from vllm.entrypoints.generate.base.protocol import DeltaMessage |
| 14 | from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest |
| 15 | from vllm.entrypoints.openai.responses.protocol import ResponsesRequest |
| 16 | from vllm.parser.abstract_parser import DelegatingParser |
| 17 | from vllm.reasoning.basic_parsers import BaseThinkingReasoningParser |
| 18 | from vllm.tool_parsers.hermes_tool_parser import Hermes2ProToolParser |
| 19 | |
| 20 | |
| 21 | class ThinkReasoningParser(BaseThinkingReasoningParser): |
| 22 | @property |
| 23 | def start_token(self) -> str: |
| 24 | return "<think>" |
| 25 | |
| 26 | @property |
| 27 | def end_token(self) -> str: |
| 28 | return "</think>" |
| 29 | |
| 30 | |
| 31 | MODEL_OUTPUT_REASONING_AND_CONTENT = ( |
| 32 | "<think>let me think about this</think>The answer is 42." |
| 33 | ) |
| 34 | |
| 35 | MODEL_OUTPUT_REASONING_AND_TOOL = ( |
| 36 | "<think>I need to call a tool</think>" |
| 37 | '<tool_call>\n{"name": "get_weather", ' |
| 38 | '"arguments": {"city": "Dallas"}}\n</tool_call>' |
| 39 | ) |
| 40 | |
| 41 | MODEL_OUTPUT_CONTENT_ONLY = "The answer is 42." |
| 42 | |
| 43 | |
| 44 | @pytest.fixture(scope="module") |
| 45 | def tokenizer(): |
| 46 | from vllm.tokenizers import get_tokenizer |
| 47 | |
| 48 | return get_tokenizer("Qwen/Qwen3-32B") |
| 49 | |
| 50 | |
| 51 | def make_responses_request(**kwargs) -> ResponsesRequest: |
| 52 | defaults = dict(model="test-model", input="test input") |
| 53 | defaults.update(kwargs) |
| 54 | return ResponsesRequest(**defaults) |
| 55 | |
| 56 | |
| 57 | def make_chat_request(**kwargs) -> ChatCompletionRequest: |
| 58 | defaults = dict( |
| 59 | model="test-model", |
| 60 | messages=[{"role": "user", "content": "hi"}], |
| 61 | ) |
| 62 | defaults.update(kwargs) |
| 63 | return ChatCompletionRequest(**defaults) |
| 64 | |
| 65 | |
| 66 | def make_parser(tokenizer, reasoning=False, tool=False): |
| 67 | class TestParser(DelegatingParser): |
| 68 | reasoning_parser_cls = ThinkReasoningParser if reasoning else None |
| 69 | tool_parser_cls = Hermes2ProToolParser if tool else None |
| 70 | |
| 71 | return TestParser(tokenizer) |
| 72 | |
| 73 | |
| 74 | # ── Non-streaming: parser.parse() + build_response_output_items ────── |
| 75 | |
| 76 | |
| 77 | def parse_and_build(parser, request, model_output, enable_auto_tools=False): |
| 78 | """Mirrors the non-streaming path in _make_response_output_items / |
| 79 | ParsableContext.append_output(): parse → suppress reasoning → build items. |
| 80 | """ |
| 81 | from vllm.entrypoints.openai.responses.utils import ( |
| 82 | build_response_output_items, |
| 83 | ) |
| 84 | |
| 85 | reasoning, content, tool_calls = parser.parse( |
| 86 | model_output, request, enable_auto_tools=enable_auto_tools |
| 87 | ) |
| 88 | if not request.include_reasoning: |
| 89 | reasoning = None |
| 90 | return build_response_output_items( |
| 91 | reasoning=reasoning, |
| 92 | content=content, |
| 93 | tool_calls=tool_calls, |
| 94 | ) |
| 95 | |
| 96 | |
| 97 | class TestNonStreamingIncludeReasoning: |
| 98 | def test_include_reasoning_true_has_reasoning_item(self, tokenizer): |
| 99 | """Default: reasoning items appear in output.""" |
| 100 | parser = make_parser(tokenizer, reasoning=True) |
| 101 | request = make_responses_request(include_reasoning=True) |
| 102 | |
| 103 | outputs = parse_and_build(parser, request, MODEL_OUTPUT_REASONING_AND_CONTENT) |
| 104 | |
| 105 | types = [o.type for o in outputs] |
| 106 | assert "reasoning" in types |
| 107 | assert "message" in types |
| 108 | |
| 109 | def test_include_reasoning_false_no_reasoning_item(self, tokenizer): |
| 110 | """Reasoning item is suppressed when include_reasoning=False.""" |
| 111 | parser = make_parser(tokenizer, reasoning=True) |
| 112 | request = make_responses_request(include_reasoning=False) |
| 113 | |
| 114 | outputs = parse_and_build(parser, request, MODEL_OUTPUT_REASONING_AND_CONTENT) |
| 115 | |
| 116 | types = [o.type for o in outputs] |
| 117 | assert "reasoning" not in types |
| 118 | assert "message" in types |
| 119 | assert outputs[0].content[0].text == "The answer is 42." |
| 120 | |
| 121 | def test_include_reasoning_false_content_preserved(self, tokenizer): |
| 122 | """Content is extracted correctly even when reasoning is suppressed.""" |
| 123 | parser = make_parser(tokenizer, reasoning=True) |
| 124 | request = make_responses_request(include_reasoning=False) |
| 125 | |
| 126 | outputs = parse_and_build(parser, request, MODEL_OUTPUT_REASONING_AND_CONTENT) |
| 127 | |
| 128 | message = next(o for o in outputs if o.type == "message") |
| 129 | assert message.content[0].text == "The answer is 42." |
| 130 | |
| 131 | def test_include_reasoning_false_tool_calls_preserved(self, tokenizer): |
| 132 | """Tool calls still work when reasoning is suppressed.""" |
| 133 | parser = make_parser(tokenizer, reasoning=True, tool=True) |
| 134 | request = make_responses_request( |
| 135 | include_reasoning=False, |
| 136 | tools=[ |
| 137 | { |
| 138 | "type": "function", |
| 139 | "name": "get_weather", |
| 140 | "parameters": { |
| 141 | "type": "object", |
| 142 | "properties": {"city": {"type": "string"}}, |
| 143 | }, |
| 144 | } |
| 145 | ], |
| 146 | ) |
| 147 | |
| 148 | outputs = parse_and_build( |
| 149 | parser, |
| 150 | request, |
| 151 | MODEL_OUTPUT_REASONING_AND_TOOL, |
| 152 | enable_auto_tools=True, |
| 153 | ) |
| 154 | |
| 155 | types = [o.type for o in outputs] |
| 156 | assert "reasoning" not in types |
| 157 | assert "function_call" in types |
| 158 | fc = next(o for o in outputs if o.type == "function_call") |
| 159 | assert fc.name == "get_weather" |
| 160 | assert json.loads(fc.arguments) == {"city": "Dallas"} |
| 161 | |
| 162 | def test_no_reasoning_parser_include_false_is_noop(self, tokenizer): |
| 163 | """include_reasoning=False is harmless when no reasoning parser.""" |
| 164 | parser = make_parser(tokenizer, reasoning=False) |
| 165 | request = make_responses_request(include_reasoning=False) |
| 166 | |
| 167 | outputs = parse_and_build(parser, request, MODEL_OUTPUT_CONTENT_ONLY) |
| 168 | |
| 169 | assert len(outputs) == 1 |
| 170 | assert outputs[0].type == "message" |
| 171 | assert outputs[0].content[0].text == MODEL_OUTPUT_CONTENT_ONLY |
| 172 | |
| 173 | def test_default_include_reasoning_is_true(self, tokenizer): |
| 174 | """ResponsesRequest defaults to include_reasoning=True.""" |
| 175 | request = make_responses_request() |
| 176 | assert request.include_reasoning is True |
| 177 | |
| 178 | def test_include_reasoning_false_suppresses_all_reasoning(self, tokenizer): |
| 179 | """Reasoning is suppressed regardless of request type.""" |
| 180 | parser = make_parser(tokenizer, reasoning=True) |
| 181 | request = make_responses_request(include_reasoning=False) |
| 182 | |
| 183 | outputs = parse_and_build(parser, request, MODEL_OUTPUT_REASONING_AND_CONTENT) |
| 184 | |
| 185 | assert all(o.type != "reasoning" for o in outputs) |
| 186 | |
| 187 | |
| 188 | # ── Streaming: parse_delta ─────────────────────────────────────────── |
| 189 | |
| 190 | |
| 191 | def stream_text(parser, tokenizer, text, request, prompt_token_ids=None): |
| 192 | token_ids = tokenizer.encode(text, add_special_tokens=False) |
| 193 | results: list[DeltaMessage | None] = [] |
| 194 | for i, tid in enumerate(token_ids): |
| 195 | delta_text = tokenizer.decode([tid]) |
| 196 | is_last = i == len(token_ids) - 1 |
| 197 | result = parser.parse_delta( |
| 198 | delta_text, |
| 199 | [tid], |
| 200 | request, |
| 201 | prompt_token_ids=prompt_token_ids, |
| 202 | finished=is_last, |
| 203 | ) |
| 204 | prompt_token_ids = None |
| 205 | results.append(result) |
| 206 | return results |
| 207 | |
| 208 | |
| 209 | def collect_fields(results): |
| 210 | all_reasoning = "".join(r.reasoning for r in results if r and r.reasoning) |
| 211 | all_content = "".join(r.content for r in results if r and r.content) |
| 212 | all_tool_calls = [tc for r in results if r and r.tool_calls for tc in r.tool_calls] |
| 213 | return all_reasoning, all_content, all_tool_calls |
| 214 | |
| 215 | |
| 216 | class TestParseDeltaIncludeReasoning: |
| 217 | def test_streaming_include_true_emits_reasoning(self, tokenizer): |
| 218 | """With include_reasoning=True, reasoning deltas are emitted.""" |
| 219 | parser = make_parser(tokenizer, reasoning=True) |
| 220 | request = make_responses_request(include_reasoning=True) |
| 221 | |
| 222 | results = stream_text( |
| 223 | parser, |
| 224 | tokenizer, |
| 225 | MODEL_OUTPUT_REASONING_AND_CONTENT, |
| 226 | request, |
| 227 | prompt_token_ids=[], |
| 228 | ) |
| 229 | reasoning, content, _ = collect_fields(results) |
| 230 | |
| 231 | assert "let me think about this" in reasoning |
| 232 | assert "42" in content |
| 233 | |
| 234 | def test_streaming_include_false_suppresses_reasoning(self, tokenizer): |
| 235 | """With include_reasoning=False, no reasoning deltas are emitted.""" |
| 236 | parser = make_parser(tokenizer, reasoning=True) |
| 237 | request = make_responses_request(include_reasoning=False) |
| 238 | |
| 239 | results = stream_text( |
| 240 | parser, |
| 241 | tokenizer, |
| 242 | MODEL_OUTPUT_REASONING_AND_CONTENT, |
| 243 | request, |
| 244 | prompt_token_ids=[], |
| 245 | ) |
| 246 | reasoning, content, _ = collect_fields(results) |
| 247 | |
| 248 | assert reasoning == "" |
| 249 | assert "42" in content |
| 250 | |
| 251 | def test_streaming_include_false_content_still_works(self, tokenizer): |
| 252 | """Content is correctly extracted in streaming even with suppression.""" |
| 253 | parser = make_parser(tokenizer, reasoning=True) |
| 254 | request = make_responses_request(include_reasoning=False) |
| 255 | |
| 256 | results = stream_text( |
| 257 | parser, |
| 258 | tokenizer, |
| 259 | MODEL_OUTPUT_REASONING_AND_CONTENT, |
| 260 | request, |
| 261 | prompt_token_ids=[], |
| 262 | ) |
| 263 | _, content, _ = collect_fields(results) |
| 264 | |
| 265 | assert "The answer is 42" in content |
| 266 | |
| 267 | def test_streaming_include_false_tool_calls_preserved(self, tokenizer): |
| 268 | """Tool calls stream correctly when reasoning is suppressed.""" |
| 269 | parser = make_parser(tokenizer, reasoning=True, tool=True) |
| 270 | request = make_responses_request( |
| 271 | include_reasoning=False, |
| 272 | tools=[ |
| 273 | { |
| 274 | "type": "function", |
| 275 | "name": "get_weather", |
| 276 | "parameters": { |
| 277 | "type": "object", |
| 278 | "properties": {"city": {"type": "string"}}, |
| 279 | }, |
| 280 | } |
| 281 | ], |
| 282 | ) |
| 283 | |
| 284 | results = stream_text( |
| 285 | parser, |
| 286 | tokenizer, |
| 287 | MODEL_OUTPUT_REASONING_AND_TOOL, |
| 288 | request, |
| 289 | prompt_token_ids=[], |
| 290 | ) |
| 291 | reasoning, content, tool_calls = collect_fields(results) |
| 292 | |
| 293 | assert reasoning == "" |
| 294 | assert len(tool_calls) > 0 |
| 295 | assert tool_calls[0].function.name == "get_weather" |
| 296 | tool_args = "".join( |
| 297 | tc.function.arguments for tc in tool_calls if tc.function.arguments |
| 298 | ) |
| 299 | assert json.loads(tool_args) == {"city": "Dallas"} |
| 300 | |
| 301 | def test_streaming_no_reasoning_parser_include_false(self, tokenizer): |
| 302 | """No crash when reasoning parser absent and include_reasoning=False.""" |
| 303 | parser = make_parser(tokenizer, reasoning=False) |
| 304 | request = make_responses_request(include_reasoning=False) |
| 305 | |
| 306 | results = stream_text( |
| 307 | parser, |
| 308 | tokenizer, |
| 309 | MODEL_OUTPUT_CONTENT_ONLY, |
| 310 | request, |
| 311 | prompt_token_ids=[], |
| 312 | ) |
| 313 | reasoning, content, _ = collect_fields(results) |
| 314 | |
| 315 | assert reasoning == "" |
| 316 | assert "42" in content |
| 317 | |
| 318 | def test_streaming_chat_completion_include_false(self, tokenizer): |
| 319 | """parse_delta also respects ChatCompletionRequest.include_reasoning.""" |
| 320 | parser = make_parser(tokenizer, reasoning=True) |
| 321 | request = make_chat_request(include_reasoning=False) |
| 322 | |
| 323 | results = stream_text( |
| 324 | parser, |
| 325 | tokenizer, |
| 326 | MODEL_OUTPUT_REASONING_AND_CONTENT, |
| 327 | request, |
| 328 | prompt_token_ids=[], |
| 329 | ) |
| 330 | reasoning, content, _ = collect_fields(results) |
| 331 | |
| 332 | assert reasoning == "" |
| 333 | assert "42" in content |
| 334 | |
| 335 | def test_streaming_reasoning_only_deltas_become_none(self, tokenizer): |
| 336 | """Deltas that carry only reasoning become None (not empty).""" |
| 337 | parser = make_parser(tokenizer, reasoning=True) |
| 338 | request = make_responses_request(include_reasoning=False) |
| 339 | |
| 340 | results = stream_text( |
| 341 | parser, |
| 342 | tokenizer, |
| 343 | MODEL_OUTPUT_REASONING_AND_CONTENT, |
| 344 | request, |
| 345 | prompt_token_ids=[], |
| 346 | ) |
| 347 | |
| 348 | for r in results: |
| 349 | if r is not None: |
| 350 | assert r.reasoning is None |
| 351 | |
| 352 | |
| 353 | # ── ParsableContext.append_output() ─────────────────────────────────── |
| 354 | |
| 355 | |
| 356 | class TestParsableContextIncludeReasoning: |
| 357 | def _make_context(self, tokenizer, request): |
| 358 | from vllm.entrypoints.openai.responses.context import ParsableContext |
| 359 | |
| 360 | class TestParser(DelegatingParser): |
| 361 | reasoning_parser_cls = ThinkReasoningParser |
| 362 | tool_parser_cls = None |
| 363 | |
| 364 | return ParsableContext( |
| 365 | tokenizer=tokenizer, |
| 366 | parser_cls=TestParser, |
| 367 | response_messages=[], |
| 368 | request=request, |
| 369 | available_tools=None, |
| 370 | chat_template=None, |
| 371 | chat_template_content_format="auto", |
| 372 | ) |
| 373 | |
| 374 | def test_process_include_false_suppresses_reasoning(self, tokenizer): |
| 375 | """ParsableContext.process() suppresses reasoning items.""" |
| 376 | from vllm.outputs import CompletionOutput, RequestOutput |
| 377 | |
| 378 | request = make_responses_request(include_reasoning=False) |
| 379 | ctx = self._make_context(tokenizer, request) |
| 380 | |
| 381 | output = RequestOutput( |
| 382 | request_id="test", |
| 383 | prompt=None, |
| 384 | prompt_token_ids=[], |
| 385 | prompt_logprobs=None, |
| 386 | outputs=[ |
| 387 | CompletionOutput( |
| 388 | index=0, |
| 389 | text=MODEL_OUTPUT_REASONING_AND_CONTENT, |
| 390 | token_ids=tokenizer.encode( |
| 391 | MODEL_OUTPUT_REASONING_AND_CONTENT, |
| 392 | add_special_tokens=False, |
| 393 | ), |
| 394 | cumulative_logprob=None, |
| 395 | logprobs=None, |
| 396 | finish_reason="stop", |
| 397 | ) |
| 398 | ], |
| 399 | finished=True, |
| 400 | ) |
| 401 | |
| 402 | ctx.append_output(output) |
| 403 | |
| 404 | types = [getattr(m, "type", None) for m in ctx.response_messages] |
| 405 | assert "reasoning" not in types |
| 406 | assert "message" in types |
| 407 | |
| 408 | def test_process_include_true_has_reasoning(self, tokenizer): |
| 409 | """ParsableContext.process() includes reasoning by default.""" |
| 410 | from vllm.outputs import CompletionOutput, RequestOutput |
| 411 | |
| 412 | request = make_responses_request(include_reasoning=True) |
| 413 | ctx = self._make_context(tokenizer, request) |
| 414 | |
| 415 | output = RequestOutput( |
| 416 | request_id="test", |
| 417 | prompt=None, |
| 418 | prompt_token_ids=[], |
| 419 | prompt_logprobs=None, |
| 420 | outputs=[ |
| 421 | CompletionOutput( |
| 422 | index=0, |
| 423 | text=MODEL_OUTPUT_REASONING_AND_CONTENT, |
| 424 | token_ids=tokenizer.encode( |
| 425 | MODEL_OUTPUT_REASONING_AND_CONTENT, |
| 426 | add_special_tokens=False, |
| 427 | ), |
| 428 | cumulative_logprob=None, |
| 429 | logprobs=None, |
| 430 | finish_reason="stop", |
| 431 | ) |
| 432 | ], |
| 433 | finished=True, |
| 434 | ) |
| 435 | |
| 436 | ctx.append_output(output) |
| 437 | |
| 438 | types = [getattr(m, "type", None) for m in ctx.response_messages] |
| 439 | assert "reasoning" in types |
| 440 | assert "message" in types |