| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | import json |
| 5 | from types import SimpleNamespace |
| 6 | |
| 7 | import pytest |
| 8 | |
| 9 | from vllm.entrypoints.generate.base.protocol import DeltaMessage |
| 10 | from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest |
| 11 | from vllm.parser.abstract_parser import DelegatingParser |
| 12 | from vllm.parser.engine.registered_adapters import Qwen3ParserReasoningAdapter |
| 13 | from vllm.reasoning.basic_parsers import BaseThinkingReasoningParser |
| 14 | from vllm.tool_parsers.hermes_tool_parser import Hermes2ProToolParser |
| 15 | |
| 16 | |
| 17 | @pytest.fixture(autouse=True) |
| 18 | def enable_hermes_required_named_parsing(monkeypatch): |
| 19 | # With VLLM_ENFORCE_STRICT_TOOL_CALLING (default on), Hermes sets |
| 20 | # supports_required_and_named=False and uses structural-tag guided tool |
| 21 | # calling. Force it True to test the non-guided JSON required/named parse path. |
| 22 | monkeypatch.setattr(Hermes2ProToolParser, "supports_required_and_named", True) |
| 23 | |
| 24 | |
| 25 | class ThinkReasoningParser(BaseThinkingReasoningParser): |
| 26 | @property |
| 27 | def start_token(self) -> str: |
| 28 | return "<think>" |
| 29 | |
| 30 | @property |
| 31 | def end_token(self) -> str: |
| 32 | return "</think>" |
| 33 | |
| 34 | |
| 35 | MODEL_OUTPUT = ( |
| 36 | "<think>let me think about this</think>" |
| 37 | '<tool_call>\n{"name": "get_weather", ' |
| 38 | '"arguments": {"city": "Dallas"}}\n</tool_call>' |
| 39 | ) |
| 40 | |
| 41 | |
| 42 | @pytest.fixture(scope="module") |
| 43 | def tokenizer(): |
| 44 | from vllm.tokenizers import get_tokenizer |
| 45 | |
| 46 | return get_tokenizer("Qwen/Qwen3-32B") |
| 47 | |
| 48 | |
| 49 | TOOLS = [ |
| 50 | { |
| 51 | "type": "function", |
| 52 | "function": { |
| 53 | "name": "get_weather", |
| 54 | "parameters": {"type": "object", "properties": {}}, |
| 55 | }, |
| 56 | } |
| 57 | ] |
| 58 | |
| 59 | |
| 60 | KIMI_K2_MODEL_CONFIG = SimpleNamespace( |
| 61 | hf_text_config=SimpleNamespace(model_type="kimi_k2"), |
| 62 | hf_overrides=None, |
| 63 | ) |
| 64 | |
| 65 | HISTORY_MESSAGES = [ |
| 66 | {"role": "user", "content": "first"}, |
| 67 | { |
| 68 | "role": "assistant", |
| 69 | "content": None, |
| 70 | "tool_calls": [ |
| 71 | { |
| 72 | "id": "functions.get_current_weather:0", |
| 73 | "type": "function", |
| 74 | "function": { |
| 75 | "name": "get_current_weather", |
| 76 | "arguments": "{}", |
| 77 | }, |
| 78 | } |
| 79 | ], |
| 80 | }, |
| 81 | { |
| 82 | "role": "tool", |
| 83 | "tool_call_id": "functions.get_current_weather:0", |
| 84 | "content": "{}", |
| 85 | }, |
| 86 | {"role": "user", "content": "again"}, |
| 87 | ] |
| 88 | |
| 89 | |
| 90 | @pytest.fixture |
| 91 | def request_obj(): |
| 92 | return ChatCompletionRequest( |
| 93 | model="test-model", |
| 94 | messages=[{"role": "user", "content": "hi"}], |
| 95 | tools=TOOLS, |
| 96 | tool_choice="auto", |
| 97 | ) |
| 98 | |
| 99 | |
| 100 | def make_parser(tokenizer, reasoning=False, tool=False, **kwargs): |
| 101 | class TestParser(DelegatingParser): |
| 102 | reasoning_parser_cls = ThinkReasoningParser if reasoning else None |
| 103 | tool_parser_cls = Hermes2ProToolParser if tool else None |
| 104 | |
| 105 | return TestParser(tokenizer, **kwargs) |
| 106 | |
| 107 | |
| 108 | def stream_text(parser, tokenizer, text, request, prompt_token_ids=None): |
| 109 | token_ids = tokenizer.encode(text, add_special_tokens=False) |
| 110 | results: list[DeltaMessage | None] = [] |
| 111 | for tid in token_ids: |
| 112 | delta_text = tokenizer.decode([tid]) |
| 113 | result = parser.parse_delta( |
| 114 | delta_text, |
| 115 | [tid], |
| 116 | request, |
| 117 | prompt_token_ids=prompt_token_ids, |
| 118 | finished=False, |
| 119 | ) |
| 120 | prompt_token_ids = None |
| 121 | results.append(result) |
| 122 | return results |
| 123 | |
| 124 | |
| 125 | def collect_fields(results): |
| 126 | all_reasoning = "".join(r.reasoning for r in results if r and r.reasoning) |
| 127 | all_content = "".join(r.content for r in results if r and r.content) |
| 128 | all_tool_calls = [tc for r in results if r and r.tool_calls for tc in r.tool_calls] |
| 129 | return all_reasoning, all_content, all_tool_calls |
| 130 | |
| 131 | |
| 132 | def test_parse_delta_neither_parser(tokenizer, request_obj): |
| 133 | parser = make_parser(tokenizer, reasoning=False, tool=False) |
| 134 | results = stream_text( |
| 135 | parser, tokenizer, MODEL_OUTPUT, request_obj, prompt_token_ids=[] |
| 136 | ) |
| 137 | reasoning, content, tool_calls = collect_fields(results) |
| 138 | |
| 139 | assert reasoning == "" |
| 140 | assert len(tool_calls) == 0 |
| 141 | assert "<think>" in content |
| 142 | assert "let me think about this" in content |
| 143 | assert "<tool_call>" in content |
| 144 | assert "get_weather" in content |
| 145 | |
| 146 | |
| 147 | def test_parse_delta_tool_parser_only(tokenizer, request_obj): |
| 148 | parser = make_parser(tokenizer, reasoning=False, tool=True) |
| 149 | results = stream_text( |
| 150 | parser, tokenizer, MODEL_OUTPUT, request_obj, prompt_token_ids=[] |
| 151 | ) |
| 152 | reasoning, content, tool_calls = collect_fields(results) |
| 153 | |
| 154 | assert reasoning == "" |
| 155 | assert "<think>" in content |
| 156 | assert "let me think about this" in content |
| 157 | assert "</think>" in content |
| 158 | |
| 159 | assert len(tool_calls) > 0 |
| 160 | assert tool_calls[0].function.name == "get_weather" |
| 161 | tool_args = "".join( |
| 162 | tc.function.arguments for tc in tool_calls if tc.function.arguments |
| 163 | ) |
| 164 | assert json.loads(tool_args) == {"city": "Dallas"} |
| 165 | |
| 166 | |
| 167 | def test_parse_delta_reasoning_parser_only(tokenizer, request_obj): |
| 168 | parser = make_parser(tokenizer, reasoning=True, tool=False) |
| 169 | results = stream_text( |
| 170 | parser, tokenizer, MODEL_OUTPUT, request_obj, prompt_token_ids=[] |
| 171 | ) |
| 172 | reasoning, content, tool_calls = collect_fields(results) |
| 173 | |
| 174 | assert "let me think about this" in reasoning |
| 175 | assert len(tool_calls) == 0 |
| 176 | assert "<tool_call>" in content |
| 177 | assert "get_weather" in content |
| 178 | assert "</tool_call>" in content |
| 179 | |
| 180 | |
| 181 | def test_parse_delta_both_parsers(tokenizer, request_obj): |
| 182 | parser = make_parser(tokenizer, reasoning=True, tool=True) |
| 183 | results = stream_text( |
| 184 | parser, tokenizer, MODEL_OUTPUT, request_obj, prompt_token_ids=[] |
| 185 | ) |
| 186 | reasoning, content, tool_calls = collect_fields(results) |
| 187 | |
| 188 | assert "let me think about this" in reasoning |
| 189 | assert content == "" |
| 190 | |
| 191 | assert len(tool_calls) > 0 |
| 192 | assert tool_calls[0].function.name == "get_weather" |
| 193 | tool_args = "".join( |
| 194 | tc.function.arguments for tc in tool_calls if tc.function.arguments |
| 195 | ) |
| 196 | assert json.loads(tool_args) == {"city": "Dallas"} |
| 197 | |
| 198 | |
| 199 | def stream_chunks(parser, tokenizer, chunks, request_obj): |
| 200 | """Stream pre-split token-ID chunks through the parser.""" |
| 201 | results: list[DeltaMessage | None] = [] |
| 202 | prompt_token_ids: list[int] | None = [] |
| 203 | for chunk in chunks: |
| 204 | delta_text = tokenizer.decode(chunk) |
| 205 | result = parser.parse_delta( |
| 206 | delta_text, |
| 207 | chunk, |
| 208 | request_obj, |
| 209 | prompt_token_ids=prompt_token_ids, |
| 210 | finished=False, |
| 211 | ) |
| 212 | prompt_token_ids = None |
| 213 | results.append(result) |
| 214 | return results |
| 215 | |
| 216 | |
| 217 | def _boundary_chunks(tokenizer, parser, end_token_id=None): |
| 218 | """Split MODEL_OUTPUT into 3 chunks that straddle the </think> boundary.""" |
| 219 | token_ids = tokenizer.encode(MODEL_OUTPUT, add_special_tokens=False) |
| 220 | if end_token_id is None: |
| 221 | end_token_id = parser._reasoning_parser.end_token_id |
| 222 | end_idx = token_ids.index(end_token_id) |
| 223 | return [ |
| 224 | token_ids[: end_idx - 1], |
| 225 | token_ids[end_idx - 1 : end_idx + 2], |
| 226 | token_ids[end_idx + 2 :], |
| 227 | ] |
| 228 | |
| 229 | |
| 230 | def test_parse_delta_reasoning_not_dropped_on_boundary(tokenizer, request_obj): |
| 231 | """Regression: reasoning must not be lost when a multi-token delta |
| 232 | spans the reasoning/tool-call boundary.""" |
| 233 | parser = make_parser(tokenizer, reasoning=True, tool=True) |
| 234 | chunks = _boundary_chunks(tokenizer, parser) |
| 235 | results = stream_chunks(parser, tokenizer, chunks, request_obj) |
| 236 | reasoning, content, tool_calls = collect_fields(results) |
| 237 | |
| 238 | assert "think about this" in reasoning |
| 239 | assert content == "" |
| 240 | assert len(tool_calls) > 0 |
| 241 | assert tool_calls[0].function.name == "get_weather" |
| 242 | tool_args = "".join( |
| 243 | tc.function.arguments for tc in tool_calls if tc.function.arguments |
| 244 | ) |
| 245 | assert json.loads(tool_args) == {"city": "Dallas"} |
| 246 | |
| 247 | |
| 248 | def test_parse_delta_reasoning_boundary_no_tool_parser(tokenizer, request_obj): |
| 249 | """When no tool parser is active, boundary-spanning chunks must still |
| 250 | preserve reasoning and pass post-</think> text as content.""" |
| 251 | parser = make_parser(tokenizer, reasoning=True, tool=False) |
| 252 | chunks = _boundary_chunks(tokenizer, parser) |
| 253 | results = stream_chunks(parser, tokenizer, chunks, request_obj) |
| 254 | reasoning, content, tool_calls = collect_fields(results) |
| 255 | |
| 256 | assert "think about this" in reasoning |
| 257 | assert len(tool_calls) == 0 |
| 258 | assert "<tool_call>" in content |
| 259 | assert "get_weather" in content |
| 260 | |
| 261 | |
| 262 | def test_parse_delta_reasoning_only_no_think_leak(tokenizer, request_obj): |
| 263 | """Regression: </think> must not leak into content when streaming |
| 264 | token-by-token with reasoning=True, tool=False.""" |
| 265 | parser = make_parser(tokenizer, reasoning=True, tool=False) |
| 266 | results = stream_text( |
| 267 | parser, tokenizer, MODEL_OUTPUT, request_obj, prompt_token_ids=[] |
| 268 | ) |
| 269 | reasoning, content, tool_calls = collect_fields(results) |
| 270 | |
| 271 | assert "let me think about this" in reasoning |
| 272 | assert "</think>" not in content |
| 273 | assert "<think>" not in content |
| 274 | |
| 275 | |
| 276 | def test_parse_delta_reasoning_only_thinking_disabled(tokenizer, request_obj): |
| 277 | """Regression test for vllm-project/vllm#40466. |
| 278 | |
| 279 | When enable_thinking=False, the chat template places <think>\\n\\n</think> |
| 280 | in the prompt. The model then generates pure content (no think tokens). |
| 281 | All streaming output must go to delta.content, not delta.reasoning. |
| 282 | """ |
| 283 | parser = make_parser(tokenizer, reasoning=True, tool=False) |
| 284 | |
| 285 | end_token_id = parser._reasoning_parser.end_token_id |
| 286 | prompt_token_ids = [1, 2, end_token_id, 3] |
| 287 | |
| 288 | content_text = "Hello! How can I assist you today?" |
| 289 | results = stream_text( |
| 290 | parser, |
| 291 | tokenizer, |
| 292 | content_text, |
| 293 | request_obj, |
| 294 | prompt_token_ids=prompt_token_ids, |
| 295 | ) |
| 296 | reasoning, content, tool_calls = collect_fields(results) |
| 297 | |
| 298 | assert reasoning == "", f"Expected no reasoning, got: {reasoning!r}" |
| 299 | assert "Hello" in content |
| 300 | assert "assist" in content |
| 301 | assert len(tool_calls) == 0 |
| 302 | |
| 303 | |
| 304 | def test_parse_delta_finished_no_flush_without_tool_call_delta(tokenizer, request_obj): |
| 305 | """When finished=True but the final parse_delta produces no |
| 306 | tool-call delta, unstreamed args are not flushed.""" |
| 307 | parser = make_parser(tokenizer, reasoning=False, tool=True) |
| 308 | |
| 309 | results = stream_text( |
| 310 | parser, tokenizer, MODEL_OUTPUT, request_obj, prompt_token_ids=[] |
| 311 | ) |
| 312 | _, _, tool_calls = collect_fields(results) |
| 313 | assert len(tool_calls) > 0 |
| 314 | |
| 315 | streamed = parser._tool_parser.streamed_args_for_tool[0] |
| 316 | assert len(streamed) > 5 |
| 317 | parser._tool_parser.streamed_args_for_tool[0] = streamed[:-5] |
| 318 | |
| 319 | # Prevent normal extraction from catching the gap — without a |
| 320 | # tool-call delta to merge into, the flush is skipped. |
| 321 | parser._tool_parser.extract_tool_calls_streaming = lambda *a, **kw: None |
| 322 | |
| 323 | flush_result = parser.parse_delta("", [], request_obj, finished=True) |
| 324 | assert flush_result is None or flush_result.tool_calls is None |
| 325 | |
| 326 | |
| 327 | def test_parse_delta_finished_no_extra_args_when_fully_streamed(tokenizer, request_obj): |
| 328 | """When all args have been streamed, finished=True must not |
| 329 | produce extra or duplicate arguments.""" |
| 330 | parser = make_parser(tokenizer, reasoning=False, tool=True) |
| 331 | results = stream_text( |
| 332 | parser, tokenizer, MODEL_OUTPUT, request_obj, prompt_token_ids=[] |
| 333 | ) |
| 334 | _, _, tool_calls = collect_fields(results) |
| 335 | |
| 336 | assert len(tool_calls) > 0 |
| 337 | assert tool_calls[0].function.name == "get_weather" |
| 338 | tool_args = "".join( |
| 339 | tc.function.arguments for tc in tool_calls if tc.function.arguments |
| 340 | ) |
| 341 | assert json.loads(tool_args) == {"city": "Dallas"} |
| 342 | |
| 343 | flush_result = parser.parse_delta("", [], request_obj, finished=True) |
| 344 | assert flush_result is None or flush_result.tool_calls is None |
| 345 | |
| 346 | |
| 347 | def test_parse_delta_finished_appends_remaining_args(tokenizer, request_obj): |
| 348 | """When finished=True and the tool parser has unstreamed args, |
| 349 | parse_delta appends the remaining arguments to the tool-call delta.""" |
| 350 | parser = make_parser(tokenizer, reasoning=False, tool=True) |
| 351 | token_ids = tokenizer.encode(MODEL_OUTPUT, add_special_tokens=False) |
| 352 | |
| 353 | remainder = ',"unit":"celsius"}' |
| 354 | prompt_ids: list[int] | None = [] |
| 355 | results: list[DeltaMessage | None] = [] |
| 356 | for i, tid in enumerate(token_ids): |
| 357 | prev = results[-1] if results else None |
| 358 | prev_had_args = ( |
| 359 | prev |
| 360 | and prev.tool_calls |
| 361 | and any(tc.function and tc.function.arguments for tc in prev.tool_calls) |
| 362 | ) |
| 363 | |
| 364 | if prev_had_args: |
| 365 | parser._tool_parser.get_remaining_unstreamed_args = lambda: remainder |
| 366 | |
| 367 | result = parser.parse_delta( |
| 368 | tokenizer.decode([tid]), |
| 369 | [tid], |
| 370 | request_obj, |
| 371 | prompt_token_ids=prompt_ids, |
| 372 | finished=prev_had_args, |
| 373 | ) |
| 374 | prompt_ids = None |
| 375 | results.append(result) |
| 376 | |
| 377 | if prev_had_args: |
| 378 | break |
| 379 | |
| 380 | _, _, tool_calls = collect_fields(results) |
| 381 | tool_args = "".join( |
| 382 | tc.function.arguments for tc in tool_calls if tc.function.arguments |
| 383 | ) |
| 384 | assert tool_args.endswith(remainder) |
| 385 | |
| 386 | |
| 387 | def test_parse_delta_tool_choice_none(tokenizer, request_obj): |
| 388 | parser = make_parser(tokenizer, reasoning=False, tool=True) |
| 389 | request = request_obj.model_copy(update={"tool_choice": "none"}) |
| 390 | results = stream_text(parser, tokenizer, MODEL_OUTPUT, request, prompt_token_ids=[]) |
| 391 | reasoning, content, tool_calls = collect_fields(results) |
| 392 | |
| 393 | assert reasoning == "" |
| 394 | assert len(tool_calls) == 0 |
| 395 | assert "<tool_call>" in content |
| 396 | assert "get_weather" in content |
| 397 | |
| 398 | |
| 399 | def test_parse_delta_tool_choice_none_with_reasoning(tokenizer, request_obj): |
| 400 | parser = make_parser(tokenizer, reasoning=True, tool=True) |
| 401 | request = request_obj.model_copy(update={"tool_choice": "none"}) |
| 402 | results = stream_text(parser, tokenizer, MODEL_OUTPUT, request, prompt_token_ids=[]) |
| 403 | reasoning, content, tool_calls = collect_fields(results) |
| 404 | |
| 405 | assert "let me think about this" in reasoning |
| 406 | assert len(tool_calls) == 0 |
| 407 | assert "<tool_call>" in content |
| 408 | assert "get_weather" in content |
| 409 | |
| 410 | |
| 411 | def test_parse_delta_required_tool_choice_kimi_k2_ids(tokenizer, request_obj): |
| 412 | parser = make_parser( |
| 413 | tokenizer, reasoning=False, tool=True, model_config=KIMI_K2_MODEL_CONFIG |
| 414 | ) |
| 415 | request = request_obj.model_copy(update={"tool_choice": "required"}) |
| 416 | output = json.dumps( |
| 417 | [ |
| 418 | { |
| 419 | "name": "get_current_weather", |
| 420 | "parameters": {"city": "Dallas"}, |
| 421 | } |
| 422 | ] |
| 423 | ) |
| 424 | |
| 425 | results: list[DeltaMessage | None] = [] |
| 426 | prompt_token_ids: list[int] | None = [] |
| 427 | for i in range(0, len(output), 3): |
| 428 | chunk = output[i : i + 3] |
| 429 | results.append( |
| 430 | parser.parse_delta( |
| 431 | chunk, |
| 432 | [], |
| 433 | request, |
| 434 | prompt_token_ids=prompt_token_ids, |
| 435 | finished=False, |
| 436 | ) |
| 437 | ) |
| 438 | prompt_token_ids = None |
| 439 | |
| 440 | _, content, tool_calls = collect_fields(results) |
| 441 | assert content == "" |
| 442 | assert any(tc.id == "functions.get_current_weather:0" for tc in tool_calls) |
| 443 | assert all(tc.id in (None, "functions.get_current_weather:0") for tc in tool_calls) |
| 444 | |
| 445 | |
| 446 | def test_parse_delta_required_tool_choice_kimi_k2_ids_after_history( |
| 447 | tokenizer, request_obj |
| 448 | ): |
| 449 | parser = make_parser( |
| 450 | tokenizer, reasoning=False, tool=True, model_config=KIMI_K2_MODEL_CONFIG |
| 451 | ) |
| 452 | request = request_obj.model_copy( |
| 453 | update={"messages": HISTORY_MESSAGES, "tool_choice": "required"} |
| 454 | ) |
| 455 | output = json.dumps( |
| 456 | [ |
| 457 | { |
| 458 | "name": "get_current_weather", |
| 459 | "parameters": {"city": "Dallas"}, |
| 460 | } |
| 461 | ] |
| 462 | ) |
| 463 | |
| 464 | results: list[DeltaMessage | None] = [] |
| 465 | prompt_token_ids: list[int] | None = [] |
| 466 | for i in range(0, len(output), 3): |
| 467 | chunk = output[i : i + 3] |
| 468 | results.append( |
| 469 | parser.parse_delta( |
| 470 | chunk, |
| 471 | [], |
| 472 | request, |
| 473 | prompt_token_ids=prompt_token_ids, |
| 474 | finished=False, |
| 475 | ) |
| 476 | ) |
| 477 | prompt_token_ids = None |
| 478 | |
| 479 | _, _, tool_calls = collect_fields(results) |
| 480 | assert any(tc.id == "functions.get_current_weather:1" for tc in tool_calls) |
| 481 | assert all(tc.id in (None, "functions.get_current_weather:1") for tc in tool_calls) |
| 482 | |
| 483 | |
| 484 | # ── Engine-based reasoning + non-engine tool parser (Qwen3 + Hermes) ── |
| 485 | |
| 486 | |
| 487 | class Qwen3ReasoningHermesToolParser(DelegatingParser): |
| 488 | reasoning_parser_cls = Qwen3ParserReasoningAdapter |
| 489 | tool_parser_cls = Hermes2ProToolParser |
| 490 | |
| 491 | |
| 492 | def test_engine_reasoning_hermes_tool_token_by_token(tokenizer, request_obj): |
| 493 | """Qwen3 engine reasoning + Hermes tool parser, token-by-token. |
| 494 | |
| 495 | Sanity check that the mixed engine/non-engine configuration works |
| 496 | when tokens arrive one at a time (no deferred content).""" |
| 497 | parser = Qwen3ReasoningHermesToolParser(tokenizer) |
| 498 | |
| 499 | assert parser._reasoning_parser.engine_based_streaming is True |
| 500 | assert parser._tool_parser.engine_based_streaming is False |
| 501 | assert parser._engine_based is False |
| 502 | |
| 503 | results = stream_text( |
| 504 | parser, tokenizer, MODEL_OUTPUT, request_obj, prompt_token_ids=[] |
| 505 | ) |
| 506 | reasoning, content, tool_calls = collect_fields(results) |
| 507 | |
| 508 | assert "let me think about this" in reasoning |
| 509 | assert content == "" |
| 510 | assert len(tool_calls) > 0 |
| 511 | assert tool_calls[0].function.name == "get_weather" |
| 512 | tool_args = "".join( |
| 513 | tc.function.arguments for tc in tool_calls if tc.function.arguments |
| 514 | ) |
| 515 | assert json.loads(tool_args) == {"city": "Dallas"} |
| 516 | |
| 517 | |
| 518 | def test_engine_reasoning_hermes_tool_boundary(tokenizer, request_obj): |
| 519 | """Qwen3 engine reasoning + Hermes tool parser, boundary chunks. |
| 520 | |
| 521 | When </think> and <tool_call> are in the same chunk with aligned |
| 522 | text and token IDs, the engine processes both terminals and returns |
| 523 | the <tool_call> text as content.""" |
| 524 | parser = Qwen3ReasoningHermesToolParser(tokenizer) |
| 525 | end_token_id = parser._reasoning_parser._parser_engine._reasoning_end_token_id |
| 526 | chunks = _boundary_chunks(tokenizer, parser, end_token_id=end_token_id) |
| 527 | results = stream_chunks(parser, tokenizer, chunks, request_obj) |
| 528 | reasoning, content, tool_calls = collect_fields(results) |
| 529 | |
| 530 | assert "think about this" in reasoning |
| 531 | assert content == "" |
| 532 | assert len(tool_calls) > 0 |
| 533 | assert tool_calls[0].function.name == "get_weather" |
| 534 | tool_args = "".join( |
| 535 | tc.function.arguments for tc in tool_calls if tc.function.arguments |
| 536 | ) |
| 537 | assert json.loads(tool_args) == {"city": "Dallas"} |
| 538 | assert "tool_call" not in content |
| 539 | |
| 540 | |
| 541 | def test_engine_reasoning_hermes_tool_text_holdback(tokenizer, request_obj): |
| 542 | """Qwen3 engine reasoning + Hermes tool parser with engine holdback. |
| 543 | |
| 544 | Simulates stream_interval > 1 where a batched delta contains |
| 545 | '</think><'. The '<' is a regular character token — not the |
| 546 | <tool_call> special token — so the engine's incremental lexer |
| 547 | buffers it (it could be the start of a text terminal like |
| 548 | <tool_call>). The buffered '<' is only recoverable via |
| 549 | finish_streaming(). |
| 550 | |
| 551 | Without the fix, finish_streaming() is never called at the |
| 552 | reasoning->tool transition when _engine_based is False, so the '<' |
| 553 | is lost and the Hermes parser sees 'tool_call>...' instead of |
| 554 | '<tool_call>...'.""" |
| 555 | parser = Qwen3ReasoningHermesToolParser(tokenizer) |
| 556 | vocab = tokenizer.get_vocab() |
| 557 | think_end_id = vocab["</think>"] |
| 558 | lt_id = vocab["<"] |
| 559 | |
| 560 | token_ids = tokenizer.encode(MODEL_OUTPUT, add_special_tokens=False) |
| 561 | end_idx = token_ids.index(think_end_id) |
| 562 | |
| 563 | # Reasoning tokens (aligned text + IDs) |
| 564 | pre_ids = token_ids[:end_idx] |
| 565 | pre_text = tokenizer.decode(pre_ids) |
| 566 | |
| 567 | # Batched delta: '</think><' — the engine recognises </think> as |
| 568 | # THINK_END but the trailing '<' is consumed by the engine's lexer |
| 569 | # and held back (it could be the start of <tool_call>). The '<' |
| 570 | # is NOT in delta_message.content; it is only in the engine's |
| 571 | # internal buffer, recoverable via finish_streaming(). |
| 572 | holdback_ids = [think_end_id, lt_id] |
| 573 | holdback_text = "</think><" |
| 574 | |
| 575 | # Remaining text: 'tool_call>\n{...}\n</tool_call>' — the model |
| 576 | # generated <tool_call> as character tokens (not the special token), |
| 577 | # and the '<' was consumed above. Encode separately to get the |
| 578 | # correct token IDs for this substring. |
| 579 | rest_text = ( |
| 580 | 'tool_call>\n{"name": "get_weather", ' |
| 581 | '"arguments": {"city": "Dallas"}}\n</tool_call>' |
| 582 | ) |
| 583 | rest_ids = tokenizer.encode(rest_text, add_special_tokens=False) |
| 584 | |
| 585 | results: list[DeltaMessage | None] = [] |
| 586 | results.append( |
| 587 | parser.parse_delta( |
| 588 | pre_text, |
| 589 | pre_ids, |
| 590 | request_obj, |
| 591 | prompt_token_ids=[], |
| 592 | finished=False, |
| 593 | ) |
| 594 | ) |
| 595 | results.append( |
| 596 | parser.parse_delta( |
| 597 | holdback_text, |
| 598 | holdback_ids, |
| 599 | request_obj, |
| 600 | finished=False, |
| 601 | ) |
| 602 | ) |
| 603 | results.append( |
| 604 | parser.parse_delta( |
| 605 | rest_text, |
| 606 | rest_ids, |
| 607 | request_obj, |
| 608 | finished=False, |
| 609 | ) |
| 610 | ) |
| 611 | |
| 612 | reasoning, content, tool_calls = collect_fields(results) |
| 613 | |
| 614 | assert "let me think about this" in reasoning |
| 615 | assert len(tool_calls) > 0, ( |
| 616 | "Tool calls lost at engine-reasoning -> tool transition. " |
| 617 | "finish_streaming() not called when _engine_based is False." |
| 618 | ) |
| 619 | assert tool_calls[0].function.name == "get_weather" |
| 620 | tool_args = "".join( |
| 621 | tc.function.arguments for tc in tool_calls if tc.function.arguments |
| 622 | ) |
| 623 | assert json.loads(tool_args) == {"city": "Dallas"} |
| 624 | assert "tool_call" not in content |
| 625 | |
| 626 | |
| 627 | # ── Engine-based reasoning WITHOUT a tool parser (Qwen3 only) ── |
| 628 | |
| 629 | |
| 630 | class Qwen3ReasoningNoToolParser(DelegatingParser): |
| 631 | reasoning_parser_cls = Qwen3ParserReasoningAdapter |
| 632 | tool_parser_cls = None |
| 633 | |
| 634 | |
| 635 | def test_engine_reasoning_no_tool_batched_content_passthrough(tokenizer, request_obj): |
| 636 | """Qwen3 engine reasoning with NO tool parser, batched boundary. |
| 637 | |
| 638 | The three mixed-parser tests above all pair the engine reasoning |
| 639 | parser with Hermes; none exercise the engine-reasoning-only path |
| 640 | through the hoisted finish_streaming() transition (where |
| 641 | ``_engine_based`` is True and there is no tool parser). A single |
| 642 | batched delta carries ``</think>`` plus the following content |
| 643 | (as happens with stream_interval > 1). The post-``</think>`` text |
| 644 | must be emitted as content -- not dropped, not reclassified as |
| 645 | reasoning -- and the ``</think>`` marker must not leak either way.""" |
| 646 | parser = Qwen3ReasoningNoToolParser(tokenizer) |
| 647 | assert parser._reasoning_parser.engine_based_streaming is True |
| 648 | assert parser._tool_parser is None |
| 649 | |
| 650 | model_output = "<think>let me think about this</think>The answer is 42." |
| 651 | end_token_id = parser._reasoning_parser._parser_engine._reasoning_end_token_id |
| 652 | token_ids = tokenizer.encode(model_output, add_special_tokens=False) |
| 653 | end_idx = token_ids.index(end_token_id) |
| 654 | chunks = [token_ids[:end_idx], token_ids[end_idx:]] |
| 655 | |
| 656 | results = stream_chunks(parser, tokenizer, chunks, request_obj) |
| 657 | reasoning, content, tool_calls = collect_fields(results) |
| 658 | |
| 659 | assert "let me think about this" in reasoning |
| 660 | assert content == "The answer is 42." |
| 661 | assert "</think>" not in content |
| 662 | assert "</think>" not in reasoning |
| 663 | assert len(tool_calls) == 0 |
| 664 | |
| 665 | |
| 666 | def _decode_stream_deltas(tokenizer, groups): |
| 667 | """Decode token-ID groups into ``(delta_text, group)`` pairs via the real |
| 668 | incremental ``DecodeStream``. |
| 669 | |
| 670 | This mirrors how vLLM's detokenizer feeds ``parse_delta`` in |
| 671 | production: byte-level UTF-8 hold-back means a character whose bytes |
| 672 | span multiple tokens is only surfaced once complete (a naive |
| 673 | per-token ``decode`` would instead emit U+FFFD replacement chars).""" |
| 674 | from tokenizers.decoders import DecodeStream |
| 675 | |
| 676 | stream = DecodeStream(skip_special_tokens=False) |
| 677 | inner = tokenizer._tokenizer |
| 678 | pairs = [] |
| 679 | for group in groups: |
| 680 | text = "" |
| 681 | for token_id in group: |
| 682 | piece = stream.step(inner, token_id) |
| 683 | if piece: |
| 684 | text += piece |
| 685 | pairs.append((text, group)) |
| 686 | return pairs |
| 687 | |
| 688 | |
| 689 | def test_engine_reasoning_hermes_tool_multibyte_holdback(tokenizer, request_obj): |
| 690 | """Multi-token character across the reasoning->tool boundary. |
| 691 | |
| 692 | Extends the ASCII '<' hold-back guard with bbrowning's multi-token |
| 693 | *character* concern. Two things must both hold: |
| 694 | |
| 695 | 1. The batched ``</think><`` delta relies on the hoisted |
| 696 | finish_streaming() to recover the engine-buffered '<'. Without |
| 697 | the fix the Hermes parser never sees ``<tool_call>`` and emits no |
| 698 | tool call at all. |
| 699 | 2. The tool-call arguments carry ``東京🧑\u200d🚀``; the astronaut |
| 700 | ZWJ sequence's bytes span multiple Qwen3 tokens, so the rest of |
| 701 | the stream is fed one token at a time through the real |
| 702 | ``DecodeStream``. Its UTF-8 hold-back yields the correct |
| 703 | character round-trip (a naive per-token decode would corrupt it), |
| 704 | verifying the boundary stays byte-safe for multi-token |
| 705 | characters.""" |
| 706 | parser = Qwen3ReasoningHermesToolParser(tokenizer) |
| 707 | vocab = tokenizer.get_vocab() |
| 708 | think_end_id = vocab["</think>"] |
| 709 | lt_id = vocab["<"] |
| 710 | |
| 711 | city = "東京🧑\u200d🚀" |
| 712 | # Faithfulness precondition: the emoji really is a multi-token char. |
| 713 | assert len(tokenizer.encode("🧑\u200d🚀", add_special_tokens=False)) > 1 |
| 714 | |
| 715 | model_output = ( |
| 716 | "<think>let me think about this</think>" |
| 717 | '<tool_call>\n{"name": "get_weather", ' |
| 718 | f'"arguments": {{"city": "{city}"}}}}\n</tool_call>' |
| 719 | ) |
| 720 | token_ids = tokenizer.encode(model_output, add_special_tokens=False) |
| 721 | end_idx = token_ids.index(think_end_id) |
| 722 | pre_ids = token_ids[:end_idx] |
| 723 | |
| 724 | rest_text = ( |
| 725 | 'tool_call>\n{"name": "get_weather", ' |
| 726 | f'"arguments": {{"city": "{city}"}}}}\n</tool_call>' |
| 727 | ) |
| 728 | rest_ids = tokenizer.encode(rest_text, add_special_tokens=False) |
| 729 | |
| 730 | # Deltas: reasoning, then a batched '</think><' (the engine buffers |
| 731 | # the '<'), then the remaining tokens one at a time so the |
| 732 | # multi-token character is genuinely split across deltas by the |
| 733 | # detokenizer. |
| 734 | groups = [pre_ids, [think_end_id, lt_id]] + [[tid] for tid in rest_ids] |
| 735 | pairs = _decode_stream_deltas(tokenizer, groups) |
| 736 | |
| 737 | results: list[DeltaMessage | None] = [] |
| 738 | prompt_token_ids: list[int] | None = [] |
| 739 | for delta_text, group in pairs: |
| 740 | results.append( |
| 741 | parser.parse_delta( |
| 742 | delta_text, |
| 743 | group, |
| 744 | request_obj, |
| 745 | prompt_token_ids=prompt_token_ids, |
| 746 | finished=False, |
| 747 | ) |
| 748 | ) |
| 749 | prompt_token_ids = None |
| 750 | |
| 751 | reasoning, content, tool_calls = collect_fields(results) |
| 752 | |
| 753 | assert "let me think about this" in reasoning |
| 754 | assert len(tool_calls) > 0, ( |
| 755 | "Tool call lost at engine-reasoning -> tool transition; the " |
| 756 | "buffered '<' was not recovered by finish_streaming()." |
| 757 | ) |
| 758 | assert tool_calls[0].function.name == "get_weather" |
| 759 | tool_args = "".join( |
| 760 | tc.function.arguments for tc in tool_calls if tc.function.arguments |
| 761 | ) |
| 762 | assert json.loads(tool_args) == {"city": city} |
| 763 | assert content == "" |