@tduncan / Nvidia-Optimization-Stack-VLLM / commits / 550218b136

[Bugfix][Frontend] Flush engine reasoning parser at engine-reasoning → tool streaming boundary (#47606)

Signed-off-by: Aakif Nawaz <aakif.nawaz@amd.com> Co-authored-by: Ben Browning <56071+bbrowning@users.noreply.github.com>

akii96 committed Jul 13, 2026 at 21:06 UTC 550218b13670fd6708916a645d8f538d773409c3

2 files changed +298 -11

tests/parser/test_streaming.py

+286 -2

@@ -9,6 +9,7 @@ import pytest

9 from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest

10 from vllm.entrypoints.openai.engine.protocol import DeltaMessage

11 from vllm.parser.abstract_parser import DelegatingParser

12 +from vllm.parser.engine.registered_adapters import Qwen3ParserReasoningAdapter

13 from vllm.reasoning.basic_parsers import BaseThinkingReasoningParser

14 from vllm.tool_parsers.hermes_tool_parser import Hermes2ProToolParser

@@ -205,10 +206,11 @@ def stream_chunks(parser, tokenizer, chunks, request_obj):

206 return results

207

208

208 -def _boundary_chunks(tokenizer, parser):

209 +def _boundary_chunks(tokenizer, parser, end_token_id=None):

210 """Split MODEL_OUTPUT into 3 chunks that straddle the </think> boundary."""

211 token_ids = tokenizer.encode(MODEL_OUTPUT, add_special_tokens=False)

211 - end_token_id = parser._reasoning_parser.end_token_id

212 + if end_token_id is None:

213 + end_token_id = parser._reasoning_parser.end_token_id

214 end_idx = token_ids.index(end_token_id)

215 return [

216 token_ids[: end_idx - 1],

@@ -469,3 +471,285 @@ def test_parse_delta_required_tool_choice_kimi_k2_ids_after_history(

471 _, _, tool_calls = collect_fields(results)

472 assert any(tc.id == "functions.get_current_weather:1" for tc in tool_calls)

473 assert all(tc.id in (None, "functions.get_current_weather:1") for tc in tool_calls)

474 +

475 +

476 +# ── Engine-based reasoning + non-engine tool parser (Qwen3 + Hermes) ──

477 +

478 +

479 +class Qwen3ReasoningHermesToolParser(DelegatingParser):

480 + reasoning_parser_cls = Qwen3ParserReasoningAdapter

481 + tool_parser_cls = Hermes2ProToolParser

482 +

483 +

484 +def test_engine_reasoning_hermes_tool_token_by_token(tokenizer, request_obj):

485 + """Qwen3 engine reasoning + Hermes tool parser, token-by-token.

486 +

487 + Sanity check that the mixed engine/non-engine configuration works

488 + when tokens arrive one at a time (no deferred content)."""

489 + parser = Qwen3ReasoningHermesToolParser(tokenizer)

490 +

491 + assert parser._reasoning_parser.engine_based_streaming is True

492 + assert parser._tool_parser.engine_based_streaming is False

493 + assert parser._engine_based is False

494 +

495 + results = stream_text(

496 + parser, tokenizer, MODEL_OUTPUT, request_obj, prompt_token_ids=[]

497 + )

498 + reasoning, content, tool_calls = collect_fields(results)

499 +

500 + assert "let me think about this" in reasoning

501 + assert content == ""

502 + assert len(tool_calls) > 0

503 + assert tool_calls[0].function.name == "get_weather"

504 + tool_args = "".join(

505 + tc.function.arguments for tc in tool_calls if tc.function.arguments

506 + )

507 + assert json.loads(tool_args) == {"city": "Dallas"}

508 +

509 +

510 +def test_engine_reasoning_hermes_tool_boundary(tokenizer, request_obj):

511 + """Qwen3 engine reasoning + Hermes tool parser, boundary chunks.

512 +

513 + When </think> and <tool_call> are in the same chunk with aligned

514 + text and token IDs, the engine processes both terminals and returns

515 + the <tool_call> text as content."""

516 + parser = Qwen3ReasoningHermesToolParser(tokenizer)

517 + end_token_id = parser._reasoning_parser._parser_engine._reasoning_end_token_id

518 + chunks = _boundary_chunks(tokenizer, parser, end_token_id=end_token_id)

519 + results = stream_chunks(parser, tokenizer, chunks, request_obj)

520 + reasoning, content, tool_calls = collect_fields(results)

521 +

522 + assert "think about this" in reasoning

523 + assert content == ""

524 + assert len(tool_calls) > 0

525 + assert tool_calls[0].function.name == "get_weather"

526 + tool_args = "".join(

527 + tc.function.arguments for tc in tool_calls if tc.function.arguments

528 + )

529 + assert json.loads(tool_args) == {"city": "Dallas"}

530 + assert "tool_call" not in content

531 +

532 +

533 +def test_engine_reasoning_hermes_tool_text_holdback(tokenizer, request_obj):

534 + """Qwen3 engine reasoning + Hermes tool parser with engine holdback.

535 +

536 + Simulates stream_interval > 1 where a batched delta contains

537 + '</think><'. The '<' is a regular character token — not the

538 + <tool_call> special token — so the engine's incremental lexer

539 + buffers it (it could be the start of a text terminal like

540 + <tool_call>). The buffered '<' is only recoverable via

541 + finish_streaming().

542 +

543 + Without the fix, finish_streaming() is never called at the

544 + reasoning->tool transition when _engine_based is False, so the '<'

545 + is lost and the Hermes parser sees 'tool_call>...' instead of

546 + '<tool_call>...'."""

547 + parser = Qwen3ReasoningHermesToolParser(tokenizer)

548 + vocab = tokenizer.get_vocab()

549 + think_end_id = vocab["</think>"]

550 + lt_id = vocab["<"]

551 +

552 + token_ids = tokenizer.encode(MODEL_OUTPUT, add_special_tokens=False)

553 + end_idx = token_ids.index(think_end_id)

554 +

555 + # Reasoning tokens (aligned text + IDs)

556 + pre_ids = token_ids[:end_idx]

557 + pre_text = tokenizer.decode(pre_ids)

558 +

559 + # Batched delta: '</think><' — the engine recognises </think> as

560 + # THINK_END but the trailing '<' is consumed by the engine's lexer

561 + # and held back (it could be the start of <tool_call>). The '<'

562 + # is NOT in delta_message.content; it is only in the engine's

563 + # internal buffer, recoverable via finish_streaming().

564 + holdback_ids = [think_end_id, lt_id]

565 + holdback_text = "</think><"

566 +

567 + # Remaining text: 'tool_call>\n{...}\n</tool_call>' — the model

568 + # generated <tool_call> as character tokens (not the special token),

569 + # and the '<' was consumed above. Encode separately to get the

570 + # correct token IDs for this substring.

571 + rest_text = (

572 + 'tool_call>\n{"name": "get_weather", '

573 + '"arguments": {"city": "Dallas"}}\n</tool_call>'

574 + )

575 + rest_ids = tokenizer.encode(rest_text, add_special_tokens=False)

576 +

577 + results: list[DeltaMessage | None] = []

578 + results.append(

579 + parser.parse_delta(

580 + pre_text,

581 + pre_ids,

582 + request_obj,

583 + prompt_token_ids=[],

584 + finished=False,

585 + )

586 + )

587 + results.append(

588 + parser.parse_delta(

589 + holdback_text,

590 + holdback_ids,

591 + request_obj,

592 + finished=False,

593 + )

594 + )

595 + results.append(

596 + parser.parse_delta(

597 + rest_text,

598 + rest_ids,

599 + request_obj,

600 + finished=False,

601 + )

602 + )

603 +

604 + reasoning, content, tool_calls = collect_fields(results)

605 +

606 + assert "let me think about this" in reasoning

607 + assert len(tool_calls) > 0, (

608 + "Tool calls lost at engine-reasoning -> tool transition. "

609 + "finish_streaming() not called when _engine_based is False."

610 + )

611 + assert tool_calls[0].function.name == "get_weather"

612 + tool_args = "".join(

613 + tc.function.arguments for tc in tool_calls if tc.function.arguments

614 + )

615 + assert json.loads(tool_args) == {"city": "Dallas"}

616 + assert "tool_call" not in content

617 +

618 +

619 +# ── Engine-based reasoning WITHOUT a tool parser (Qwen3 only) ──

620 +

621 +

622 +class Qwen3ReasoningNoToolParser(DelegatingParser):

623 + reasoning_parser_cls = Qwen3ParserReasoningAdapter

624 + tool_parser_cls = None

625 +

626 +

627 +def test_engine_reasoning_no_tool_batched_content_passthrough(tokenizer, request_obj):

628 + """Qwen3 engine reasoning with NO tool parser, batched boundary.

629 +

630 + The three mixed-parser tests above all pair the engine reasoning

631 + parser with Hermes; none exercise the engine-reasoning-only path

632 + through the hoisted finish_streaming() transition (where

633 + ``_engine_based`` is True and there is no tool parser). A single

634 + batched delta carries ``</think>`` plus the following content

635 + (as happens with stream_interval > 1). The post-``</think>`` text

636 + must be emitted as content -- not dropped, not reclassified as

637 + reasoning -- and the ``</think>`` marker must not leak either way."""

638 + parser = Qwen3ReasoningNoToolParser(tokenizer)

639 + assert parser._reasoning_parser.engine_based_streaming is True

640 + assert parser._tool_parser is None

641 +

642 + model_output = "<think>let me think about this</think>The answer is 42."

643 + end_token_id = parser._reasoning_parser._parser_engine._reasoning_end_token_id

644 + token_ids = tokenizer.encode(model_output, add_special_tokens=False)

645 + end_idx = token_ids.index(end_token_id)

646 + chunks = [token_ids[:end_idx], token_ids[end_idx:]]

647 +

648 + results = stream_chunks(parser, tokenizer, chunks, request_obj)

649 + reasoning, content, tool_calls = collect_fields(results)

650 +

651 + assert "let me think about this" in reasoning

652 + assert content == "The answer is 42."

653 + assert "</think>" not in content

654 + assert "</think>" not in reasoning

655 + assert len(tool_calls) == 0

656 +

657 +

658 +def _decode_stream_deltas(tokenizer, groups):

659 + """Decode token-ID groups into ``(delta_text, group)`` pairs via the real

660 + incremental ``DecodeStream``.

661 +

662 + This mirrors how vLLM's detokenizer feeds ``parse_delta`` in

663 + production: byte-level UTF-8 hold-back means a character whose bytes

664 + span multiple tokens is only surfaced once complete (a naive

665 + per-token ``decode`` would instead emit U+FFFD replacement chars)."""

666 + from tokenizers.decoders import DecodeStream

667 +

668 + stream = DecodeStream(skip_special_tokens=False)

669 + inner = tokenizer._tokenizer

670 + pairs = []

671 + for group in groups:

672 + text = ""

673 + for token_id in group:

674 + piece = stream.step(inner, token_id)

675 + if piece:

676 + text += piece

677 + pairs.append((text, group))

678 + return pairs

679 +

680 +

681 +def test_engine_reasoning_hermes_tool_multibyte_holdback(tokenizer, request_obj):

682 + """Multi-token character across the reasoning->tool boundary.

683 +

684 + Extends the ASCII '<' hold-back guard with bbrowning's multi-token

685 + *character* concern. Two things must both hold:

686 +

687 + 1. The batched ``</think><`` delta relies on the hoisted

688 + finish_streaming() to recover the engine-buffered '<'. Without

689 + the fix the Hermes parser never sees ``<tool_call>`` and emits no

690 + tool call at all.

691 + 2. The tool-call arguments carry ``東京🧑\u200d🚀``; the astronaut

692 + ZWJ sequence's bytes span multiple Qwen3 tokens, so the rest of

693 + the stream is fed one token at a time through the real

694 + ``DecodeStream``. Its UTF-8 hold-back yields the correct

695 + character round-trip (a naive per-token decode would corrupt it),

696 + verifying the boundary stays byte-safe for multi-token

697 + characters."""

698 + parser = Qwen3ReasoningHermesToolParser(tokenizer)

699 + vocab = tokenizer.get_vocab()

700 + think_end_id = vocab["</think>"]

701 + lt_id = vocab["<"]

702 +

703 + city = "東京🧑\u200d🚀"

704 + # Faithfulness precondition: the emoji really is a multi-token char.

705 + assert len(tokenizer.encode("🧑\u200d🚀", add_special_tokens=False)) > 1

706 +

707 + model_output = (

708 + "<think>let me think about this</think>"

709 + '<tool_call>\n{"name": "get_weather", '

710 + f'"arguments": {{"city": "{city}"}}}}\n</tool_call>'

711 + )

712 + token_ids = tokenizer.encode(model_output, add_special_tokens=False)

713 + end_idx = token_ids.index(think_end_id)

714 + pre_ids = token_ids[:end_idx]

715 +

716 + rest_text = (

717 + 'tool_call>\n{"name": "get_weather", '

718 + f'"arguments": {{"city": "{city}"}}}}\n</tool_call>'

719 + )

720 + rest_ids = tokenizer.encode(rest_text, add_special_tokens=False)

721 +

722 + # Deltas: reasoning, then a batched '</think><' (the engine buffers

723 + # the '<'), then the remaining tokens one at a time so the

724 + # multi-token character is genuinely split across deltas by the

725 + # detokenizer.

726 + groups = [pre_ids, [think_end_id, lt_id]] + [[tid] for tid in rest_ids]

727 + pairs = _decode_stream_deltas(tokenizer, groups)

728 +

729 + results: list[DeltaMessage | None] = []

730 + prompt_token_ids: list[int] | None = []

731 + for delta_text, group in pairs:

732 + results.append(

733 + parser.parse_delta(

734 + delta_text,

735 + group,

736 + request_obj,

737 + prompt_token_ids=prompt_token_ids,

738 + finished=False,

739 + )

740 + )

741 + prompt_token_ids = None

742 +

743 + reasoning, content, tool_calls = collect_fields(results)

744 +

745 + assert "let me think about this" in reasoning

746 + assert len(tool_calls) > 0, (

747 + "Tool call lost at engine-reasoning -> tool transition; the "

748 + "buffered '<' was not recovered by finish_streaming()."

749 + )

750 + assert tool_calls[0].function.name == "get_weather"

751 + tool_args = "".join(

752 + tc.function.arguments for tc in tool_calls if tc.function.arguments

753 + )

754 + assert json.loads(tool_args) == {"city": city}

755 + assert content == ""

vllm/parser/abstract_parser.py

+12 -9

@@ -851,19 +851,22 @@ class DelegatingParser(Parser):

851 state.reasoning_ended = True

852 reasoning_transitioned = True

853 current_token_ids = self.extract_content_ids(delta_token_ids)

854 + # Flush whenever the reasoning parser is engine-based (not only

855 + # when _engine_based is True): it buffers the post-marker text

856 + # (e.g. the "<" of "<tool_call>"), surfaced via finish_streaming().

857 + flush_delta = (

858 + reasoning_parser.finish_streaming() # type: ignore[union-attr, attr-defined]

859 + if reasoning_parser is not None

860 + and reasoning_parser.engine_based_streaming

861 + else None

862 + )

863 + current_text = (

864 + (delta_message.content if delta_message else None) or ""

865 + ) + ((flush_delta.content if flush_delta else None) or "")

866 if self._engine_based:

855 - flush_delta = reasoning_parser.finish_streaming() # type: ignore[union-attr, attr-defined]

856 - current_text = (

857 - (delta_message.content if delta_message else None) or ""

858 - ) + ((flush_delta.content if flush_delta else None) or "")

867 if delta_message and self._tool_parser is not None:

868 delta_message.content = None

869 else:

862 - current_text = (

863 - delta_message.content

864 - if delta_message and delta_message.content

865 - else ""

866 - )

870 delta_text = current_text

871

872 # Tool call extraction