[Bugfix] Gemma4 parser: classify channel-less output consistently in streaming and non-streaming (#48262)
Signed-off-by: Adhithya Balakrishnan <adhithya.b2004@gmail.com> Co-authored-by: Ben Browning <56071+bbrowning@users.noreply.github.com>
adhi29 committed
Jul 14, 2026 at 19:00 UTC
af453e5647773b6edb89230148b9fabeea0ace21
2 files changed
+124
-25
tests/parser/engine/test_gemma4_streaming_reasoning.py
+87
-16
@@ -378,15 +378,14 @@ _PRE_INIT_THOUGHT_GEN_SEQUENCE: list[tuple[int, str]] = [
378
379
class TestGemma4PreInitReasoningRobustness:
380
"""Tests for the ``(REASONING, THINK_START)`` no-op transition and
381
- cooperating ``thought\\n`` prefix stripping when the engine has been
382
- pre-initialised to ``REASONING`` from the prompt.
383
-
384
- These cover the case the reviewer raised: prompt ends with
385
- ``<|turn>model\\n`` (``is_reasoning_end`` returns ``False`` because
386
- thinking is enabled, so the engine is pre-initialised), but the model
387
- still emits its own ``<|channel>thought\\n…<channel|>content``. The
388
- ``thought\\n`` prefix must be stripped, the ``<|channel>`` must not
389
- leak as text, and the post-``<channel|>`` text must appear as content.
381
+ cooperating ``thought\\n`` prefix stripping.
382
+
383
+ Covers both engine start states: a new-turn prompt (engine starts in
384
+ ``CONTENT`` and the model emits its own ``<|channel>`` opener) and a
385
+ prompt ending inside an open ``<|channel>`` block (engine
386
+ pre-initialised to ``REASONING``). In both cases the ``thought\\n``
387
+ prefix must be stripped, the ``<|channel>`` must not leak as text,
388
+ and the post-``<channel|>`` text must appear as content.
389
"""
390
391
@pytest.fixture
@@ -397,13 +396,12 @@ class TestGemma4PreInitReasoningRobustness:
396
def pre_init_parser(self, pre_init_tokenizer):
397
return Gemma4Parser(pre_init_tokenizer)
398
400
- def test_redundant_channel_open_swallowed_after_new_turn(
399
+ def test_model_emitted_channel_open_after_new_turn(
400
self, pre_init_parser, pre_init_tokenizer, request_obj
401
):
403
- # Prompt ends with ``<|turn>model\n``-style sentinel. With
404
- # ``enable_thinking=True`` (the default), ``is_reasoning_end``
405
- # returns ``False`` for a ``<|turn>`` tail, so the engine is
406
- # pre-initialised to ``REASONING``.
402
+ # Prompt ends with a ``<|turn>model\n``-style sentinel, i.e. not
403
+ # inside an open ``<|channel>`` block, so the engine starts in
404
+ # ``CONTENT`` and the model's own opener drives the transition.
405
results = _stream_tokens_batched(
406
pre_init_parser,
407
pre_init_tokenizer,
@@ -414,8 +412,7 @@ class TestGemma4PreInitReasoningRobustness:
412
413
reasoning, content, _ = _collect_fields(results)
414
417
- # ``thought\n`` prefix must be stripped from reasoning even though
418
- # the engine was pre-initialised to REASONING.
415
+ # ``thought\n`` prefix must be stripped from reasoning.
416
assert reasoning.startswith("Reason"), (
417
f"thought\\n prefix leaked into reasoning: {reasoning!r}"
418
)
@@ -464,6 +461,80 @@ class TestGemma4PreInitReasoningRobustness:
461
assert "Final content" in content
462
463
464
+# ── Channel-less direct answer (no reasoning markers at all) ──────────
465
+
466
+_PLAIN_ANSWER_TEXT = "This is a direct final answer without channel markers."
467
+_PLAIN_ANSWER_TOKENS: list[tuple[int, str]] = []
468
+for _i, _word in enumerate(_PLAIN_ANSWER_TEXT.split(" ")):
469
+ _prefix = " " if _i > 0 else ""
470
+ _PLAIN_ANSWER_TOKENS.append((7000 + _i, _prefix + _word))
471
+
472
+
473
+class TestGemma4ChannelLessOutputConsistency:
474
+ """Streaming and non-streaming must classify channel-less output
475
+ identically.
476
+
477
+ With ``enable_thinking=True``, a prompt that merely starts a new model
478
+ turn (ends with ``<|turn>model\\n``) used to pre-initialise the
479
+ streaming engine to ``REASONING``, so a direct answer containing no
480
+ channel markers was streamed entirely as ``reasoning`` while the
481
+ non-streaming path returned the same text as ``content``.
482
+
483
+ Regression test for vllm-project/vllm#48217.
484
+ """
485
+
486
+ @pytest.fixture
487
+ def plain_tokenizer(self):
488
+ return _make_tokenizer(_PLAIN_ANSWER_TOKENS)
489
+
490
+ @pytest.fixture
491
+ def plain_parser(self, plain_tokenizer):
492
+ return Gemma4Parser(plain_tokenizer)
493
+
494
+ def test_streaming_channel_less_output_is_content(
495
+ self, plain_parser, plain_tokenizer, request_obj
496
+ ):
497
+ results = _stream_tokens_batched(
498
+ plain_parser,
499
+ plain_tokenizer,
500
+ request_obj,
501
+ batch_size=1,
502
+ # New-turn tail: ``<|turn>model\n``.
503
+ prompt_token_ids=[NEW_TURN_ID, 9100, 9101],
504
+ )
505
+
506
+ reasoning, content, _ = _collect_fields(results)
507
+
508
+ assert content == _PLAIN_ANSWER_TEXT, (
509
+ f"Channel-less output missing from content: {content!r}"
510
+ )
511
+ assert not reasoning, (
512
+ f"Channel-less output misclassified as reasoning: {reasoning!r}"
513
+ )
514
+
515
+ def test_streaming_matches_non_streaming(
516
+ self, plain_parser, plain_tokenizer, request_obj
517
+ ):
518
+ reasoning, content = plain_parser.extract_reasoning(
519
+ _PLAIN_ANSWER_TEXT, request_obj
520
+ )
521
+ assert reasoning is None
522
+ assert content == _PLAIN_ANSWER_TEXT
523
+
524
+ stream_parser = Gemma4Parser(plain_tokenizer)
525
+ results = _stream_tokens_batched(
526
+ stream_parser,
527
+ plain_tokenizer,
528
+ request_obj,
529
+ batch_size=1,
530
+ prompt_token_ids=[NEW_TURN_ID, 9100, 9101],
531
+ )
532
+ stream_reasoning, stream_content, _ = _collect_fields(results)
533
+
534
+ assert (stream_reasoning or None) == reasoning
535
+ assert (stream_content or None) == content
536
+
537
+
538
# ── Second model output: two tool calls with holdback ────────────────
539
540
REASONING_TEXT_2 = (
vllm/parser/gemma4.py
+37
-9
@@ -479,19 +479,47 @@ class Gemma4Parser(ParserEngine):
479
return True
480
return True
481
482
+ def _prompt_ends_in_open_reasoning(self, prompt_token_ids: Sequence[int]) -> bool:
483
+ """Whether the prompt tail is inside an open ``<|channel>`` block.
484
+
485
+ Scans backwards: a ``<|channel>`` start token seen before any
486
+ closing or turn-boundary token means the block is still open.
487
+ """
488
+ start_id = self._reasoning_start_token_id
489
+ if start_id is None:
490
+ return False
491
+ boundary_ids = {
492
+ tid
493
+ for tid in (
494
+ self._reasoning_end_token_id,
495
+ self._tool_call_token_id,
496
+ self._new_turn_token_id,
497
+ self._tool_response_token_id,
498
+ )
499
+ if tid is not None
500
+ }
501
+ for tid in reversed(prompt_token_ids):
502
+ if tid == start_id:
503
+ return True
504
+ if tid in boundary_ids:
505
+ return False
506
+ return False
507
+
508
def adjust_initial_state_from_prompt(self, prompt_token_ids: Sequence[int]) -> None:
483
- """Pre-initialise the engine to ``REASONING`` when the prompt does
484
- not already end with reasoning concluded.
509
+ """Pre-initialise the engine to ``REASONING`` when the prompt ends
510
+ inside an open ``<|channel>`` block.
511
512
This covers the post-tool-response continuation case where the chat
487
- template leaves the prompt ending inside an open ``<|channel>``
488
- block (issue #45834). It is also safe in the common new-turn case
489
- where the model itself emits ``<|channel>`` first: the no-op
490
- ``(REASONING, THINK_START)`` transition swallows it, and the
491
- ``thought\n`` prefix in the first reasoning chunk is stripped by
492
- ``_events_to_delta`` as it already is in the default flow.
513
+ template leaves the prompt ending with ``<|channel>thought\n``
514
+ (issue #45834). A prompt that merely starts a new model turn must
515
+ not pre-initialise reasoning: the model may answer directly without
516
+ emitting any channel markers, and the non-streaming path classifies
517
+ such output as content (issue #48217). When the model does open its
518
+ own ``<|channel>``, the ``(CONTENT, THINK_START)`` transition
519
+ handles it, and the ``thought\n`` prefix in the first reasoning
520
+ chunk is stripped by ``_events_to_delta`` as in the default flow.
521
"""
494
- if self.is_reasoning_end(list(prompt_token_ids)):
522
+ if not self._prompt_ends_in_open_reasoning(prompt_token_ids):
523
return
524
self._engine.reset(initial_state=ParserState.REASONING)
525
# Prevent a later default ``initialize_streaming()`` (e.g. from