@tduncan / Nvidia-Optimization-Stack-VLLM / commits / 25671cb520

[Parser][Bugfix] Ensure tool call or other special tokens don't leak in non-streaming tool parsing (#46875)

Signed-off-by: Ben Browning <bbrownin@redhat.com> Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>

Ben Browning committed Jun 30, 2026 at 13:46 UTC 25671cb520e01c7b1c0eabeb65f38a6cf9052b4c

14 files changed +783 -219

tests/parser/engine/conftest.py

+14 -2

@@ -15,8 +15,17 @@ def should_do_global_cleanup_after_test() -> bool:

15 return False

18 -def make_mock_tokenizer(vocab: dict[str, int]) -> MagicMock:

19 - """Create a mock tokenizer with the given special-token vocabulary.

18 +def make_mock_tokenizer(

19 + vocab: dict[str, int],

20 + special_tokens: list[str] | None = None,

21 +) -> MagicMock:

22 + """Create a mock tokenizer with the given vocabulary.

23 +

24 + Args:

25 + vocab: Mapping of token text to token ID.

26 + special_tokens: Which tokens to mark as special. When ``None``

27 + (the default), every key in *vocab* is treated as special —

28 + convenient when the vocab only contains delimiter tokens.

30 The returned mock supports get_vocab(), encode(), and decode().

31 decode() maps known token IDs back to their text and falls back to

@@ -29,6 +38,9 @@ def make_mock_tokenizer(vocab: dict[str, int]) -> MagicMock:

38 tokenizer.decode.side_effect = lambda ids: "".join(

39 id_to_text.get(i, chr(i) if i < 128 else f"<{i}>") for i in ids

40 )

41 + st = special_tokens if special_tokens is not None else list(vocab.keys())

42 + tokenizer.all_special_tokens = st

43 + tokenizer.all_special_ids = [vocab[t] for t in st if t in vocab]

44 return tokenizer

tests/parser/engine/replay_harness.py

+41

@@ -62,6 +62,7 @@ class MockTokenizer:

62 "eos_token_id",

63 "bos_token_id",

64 "pad_token_id",

65 + "_all_special_tokens",

66 )

68 def __init__(

@@ -73,6 +74,7 @@ class MockTokenizer:

74 self._token_ids = [tid for tid, _ in tokens]

75 self._token_decode_map = {tid: text for tid, text in tokens}

76 self._special_ids = set(vocab.values())

77 + self._all_special_tokens = list(vocab.keys())

78 self.eos_token_id = None

79 self.bos_token_id = None

80 self.pad_token_id = None

@@ -83,6 +85,14 @@ class MockTokenizer:

85 def get_vocab(self) -> dict[str, int]:

86 return self._vocab

88 + @property

89 + def all_special_tokens(self) -> list[str]:

90 + return self._all_special_tokens

91 +

92 + @property

93 + def all_special_ids(self) -> list[int]:

94 + return [self._vocab[t] for t in self._all_special_tokens if t in self._vocab]

95 +

96 def encode(self, text: str, **kwargs) -> list[int]:

97 return self._token_ids

@@ -117,6 +127,37 @@ def _test_request(

127 )

128

129

130 +DUMMY_TOOLS = [

131 + {

132 + "type": "function",

133 + "function": {"name": "stub", "parameters": {"type": "object"}},

134 + },

135 +]

136 +

137 +

138 +def parse_non_streaming(

139 + parser,

140 + sample: Sample,

141 + request: ChatCompletionRequest,

142 +) -> ParseOutput:

143 + """Run ``parser.parse()`` and return a :class:`ParseOutput`."""

144 + full_text = "".join(text for _, text in sample.tokens)

145 + reasoning, content, tool_calls = parser.parse(

146 + full_text,

147 + request,

148 + enable_auto_tools=True,

149 + )

150 + tc_list: list[dict] = []

151 + if tool_calls:

152 + for tc in tool_calls:

153 + tc_list.append({"name": tc.name, "arguments": tc.arguments})

154 + return ParseOutput(

155 + reasoning=reasoning or "",

156 + content=content or "",

157 + tool_calls=tc_list,

158 + )

159 +

160 +

161 def replay_streaming(

162 parser,

163 tokens: list[tuple[int, str]],

tests/parser/engine/test_delegating_replay.py

+52

@@ -20,10 +20,14 @@ from pydantic import TypeAdapter

21 from tests.parser.engine.replay_harness import (

22 CHUNK_SIZES,

23 + DUMMY_TOOLS,

24 MockTokenizer,

25 + _test_request,

26 + assert_no_terminal_leakage,

27 assert_parse_output,

28 collect_output,

29 make_mock_tokenizer,

30 + parse_non_streaming,

31 replay_streaming,

32 )

33 from tests.parser.engine.trace_builder import _BUILDERS, build_samples

@@ -142,3 +146,51 @@ def test_delegating_replay(parser_cls, sample, chunk_size):

146 )

147 output = collect_output(deltas)

148 assert_parse_output(output, sample)

149 +

150 +

151 +_TOOL_CALL_SAMPLES = [

152 + (p.parser_cls, p.name, s)

153 + for p in _PAIRINGS

154 + for s in p.samples

155 + if s.expected_tool_calls

156 +]

157 +

158 +

159 +@pytest.mark.parametrize(

160 + "parser_cls,parser_name,sample",

161 + _TOOL_CALL_SAMPLES,

162 + ids=lambda v: v.id if hasattr(v, "id") else "",

163 +)

164 +def test_delegating_parse_tool_choice_none(parser_cls, parser_name, sample):

165 + """Non-streaming parse() with tool_choice='none' via DelegatingParser

166 + must not leak special tokens into content."""

167 + tokenizer = make_mock_tokenizer(sample)

168 + validated_tools = (

169 + _TOOLS_VALIDATOR.validate_python(sample.tools) if sample.tools else None

170 + )

171 + parser = parser_cls(

172 + tokenizer,

173 + validated_tools,

174 + chat_template_kwargs=sample.chat_template_kwargs,

175 + )

176 +

177 + request = _test_request(tools=DUMMY_TOOLS)

178 + request.tool_choice = "none"

179 +

180 + output = parse_non_streaming(parser, sample, request)

181 +

182 + assert output.tool_calls == [], (

183 + f"Expected no tool calls but got {output.tool_calls}"

184 + )

185 +

186 + cfg = parser._tool_parser._parser_engine.parser_engine_config

187 + terminals = sorted(

188 + v

189 + for v in set(cfg.terminals.values()) | set(cfg.token_id_terminals.values())

190 + if len(v) > 1

191 + )

192 + assert_no_terminal_leakage(

193 + output,

194 + terminals,

195 + context=f"parser={parser_name}",

196 + )

tests/parser/engine/test_gemma4_streaming_reasoning.py

+141 -12

@@ -7,6 +7,7 @@ from unittest.mock import MagicMock

8 import pytest

10 +from tests.parser.engine.conftest import make_mock_tokenizer

11 from tests.parser.engine.streaming_helpers import (

12 collect_content,

13 collect_function_name,

@@ -57,6 +58,8 @@ def _make_tokenizer(sequence: list[tuple[int, str]]) -> MagicMock:

58 return "".join(parts)

60 tokenizer.decode.side_effect = decode

61 + tokenizer.all_special_tokens = list(SPECIAL_TOKEN_MAP.values())

62 + tokenizer.all_special_ids = list(SPECIAL_TOKEN_MAP.keys())

63 return tokenizer

@@ -657,19 +660,15 @@ class TestGemma4ReasoningTruncationWithHoldback:

660 @pytest.fixture

661 def tool_call_tokenizer():

662 """Mock tokenizer with Gemma4 special token vocab."""

660 - tokenizer = MagicMock()

661 - tokenizer.encode.return_value = [1, 2, 3]

662 - tokenizer.get_vocab.return_value = {

663 - "<|tool_call>": TOOL_CALL_START_ID,

664 - "<tool_call|>": TOOL_CALL_END_ID,

665 - "<|channel>": CHANNEL_START_ID,

666 - "<channel|>": CHANNEL_END_ID,

667 - '<|"|>': QUOTED_ID,

668 - }

669 - tokenizer.decode.side_effect = lambda ids: "".join(

670 - SPECIAL_TOKEN_MAP.get(i, chr(i) if i < 128 else f"<{i}>") for i in ids

663 + return make_mock_tokenizer(

664 + vocab={

665 + "<|tool_call>": TOOL_CALL_START_ID,

666 + "<tool_call|>": TOOL_CALL_END_ID,

667 + "<|channel>": CHANNEL_START_ID,

668 + "<channel|>": CHANNEL_END_ID,

669 + '<|"|>': QUOTED_ID,

670 + },

671 )

672 - return tokenizer

672

673

674 @pytest.fixture

@@ -1407,3 +1406,133 @@ class TestBareThoughtWithoutChannelOpener:

1406 assert reasoning == ""

1407 assert content == ""

1408 assert len(tool_calls) == 0

1409 +

1410 +

1411 +# ── Regression: commas inside <|"|>-delimited string values ─────────

1412 +#

1413 +# _make_tokenizer sets all_special_tokens, which activates the auto-drop

1414 +# mechanism in _build_drop_info. If <|"|> is not in configured_texts,

1415 +# it gets silently dropped and commas inside string values become field

1416 +# separators, e.g. "San Francisco, CA" → {"location": "San Francisco"}.

1417 +

1418 +

1419 +COMMA_TOKEN_SEQUENCE: list[tuple[int, str]] = [

1420 + (TOOL_CALL_START_ID, "<|tool_call>"),

1421 + (4000, "call"),

1422 + (4001, ":"),

1423 + (4002, "get_weather"),

1424 + (4003, "{"),

1425 + (4004, "location"),

1426 + (4005, ":"),

1427 + (QUOTED_ID, '<|"|>'),

1428 + (4006, "San Francisco"),

1429 + (4007, ", CA"),

1430 + (QUOTED_ID, '<|"|>'),

1431 + (4008, ","),

1432 + (4009, "unit"),

1433 + (4010, ":"),

1434 + (QUOTED_ID, '<|"|>'),

1435 + (4011, "celsius"),

1436 + (QUOTED_ID, '<|"|>'),

1437 + (4012, "}"),

1438 + (TOOL_CALL_END_ID, "<tool_call|>"),

1439 +]

1440 +

1441 +MULTI_COMMA_TOKEN_SEQUENCE: list[tuple[int, str]] = [

1442 + (TOOL_CALL_START_ID, "<|tool_call>"),

1443 + (4000, "call"),

1444 + (4001, ":"),

1445 + (4020, "send_message"),

1446 + (4003, "{"),

1447 + (4021, "destination"),

1448 + (4005, ":"),

1449 + (QUOTED_ID, '<|"|>'),

1450 + (4022, "456 Oakwood Avenue"),

1451 + (4023, ", Rivermist"),

1452 + (4024, ", 83214"),

1453 + (QUOTED_ID, '<|"|>'),

1454 + (4012, "}"),

1455 + (TOOL_CALL_END_ID, "<tool_call|>"),

1456 +]

1457 +

1458 +

1459 +class TestCommaInStringValueRegression:

1460 + """Regression: <|"|> delimiters must not be auto-dropped.

1461 +

1462 + When _build_drop_info discovers <|"|> as a special token and it is

1463 + not in configured_texts, the delimiter is silently removed. Without

1464 + it, _parse_gemma4_args treats commas inside string values as field

1465 + separators.

1466 + """

1467 +

1468 + @pytest.fixture

1469 + def comma_tokenizer(self):

1470 + return _make_tokenizer(COMMA_TOKEN_SEQUENCE)

1471 +

1472 + @pytest.fixture

1473 + def comma_parser(self, comma_tokenizer):

1474 + return Gemma4Parser(comma_tokenizer)

1475 +

1476 + @pytest.fixture

1477 + def multi_comma_tokenizer(self):

1478 + return _make_tokenizer(MULTI_COMMA_TOKEN_SEQUENCE)

1479 +

1480 + @pytest.fixture

1481 + def multi_comma_parser(self, multi_comma_tokenizer):

1482 + return Gemma4Parser(multi_comma_tokenizer)

1483 +

1484 + def test_batched_streaming_comma_in_value(

1485 + self, comma_parser, comma_tokenizer, request_obj

1486 + ):

1487 + results = _stream_tokens_batched(

1488 + comma_parser,

1489 + comma_tokenizer,

1490 + request_obj,

1491 + batch_size=1,

1492 + prompt_token_ids=[],

1493 + )

1494 + _, _, tool_calls = _collect_fields(results)

1495 + assert len(tool_calls) > 0

1496 + args_text = "".join(

1497 + tc.function.arguments

1498 + for tc in tool_calls

1499 + if tc.function and tc.function.arguments

1500 + )

1501 + parsed = json.loads(args_text)

1502 + assert parsed["location"] == "San Francisco, CA"

1503 + assert parsed["unit"] == "celsius"

1504 +

1505 + def test_batched_streaming_multiple_commas(

1506 + self, multi_comma_parser, multi_comma_tokenizer, request_obj

1507 + ):

1508 + results = _stream_tokens_batched(

1509 + multi_comma_parser,

1510 + multi_comma_tokenizer,

1511 + request_obj,

1512 + batch_size=1,

1513 + prompt_token_ids=[],

1514 + )

1515 + _, _, tool_calls = _collect_fields(results)

1516 + assert len(tool_calls) > 0

1517 + args_text = "".join(

1518 + tc.function.arguments

1519 + for tc in tool_calls

1520 + if tc.function and tc.function.arguments

1521 + )

1522 + parsed = json.loads(args_text)

1523 + assert parsed["destination"] == "456 Oakwood Avenue, Rivermist, 83214"

1524 +

1525 + def test_non_streaming_comma_in_value(self, comma_parser, request_obj):

1526 + text = "".join(text for _, text in COMMA_TOKEN_SEQUENCE)

1527 + result = comma_parser.extract_tool_calls(text, request_obj)

1528 + assert result.tools_called is True

1529 + args = json.loads(result.tool_calls[0].function.arguments)

1530 + assert args["location"] == "San Francisco, CA"

1531 + assert args["unit"] == "celsius"

1532 +

1533 + def test_non_streaming_multiple_commas(self, multi_comma_parser, request_obj):

1534 + text = "".join(text for _, text in MULTI_COMMA_TOKEN_SEQUENCE)

1535 + result = multi_comma_parser.extract_tool_calls(text, request_obj)

1536 + assert result.tools_called is True

1537 + args = json.loads(result.tool_calls[0].function.arguments)

1538 + assert args["destination"] == "456 Oakwood Avenue, Rivermist, 83214"

tests/parser/engine/test_parser_engine.py

+182 -1

@@ -119,8 +119,10 @@ def _hermes_config() -> ParserEngineConfig:

119 def _make_engine(

120 config: ParserEngineConfig | None = None,

121 tools: list | None = None,

122 + vocab: dict[str, int] | None = None,

123 + special_tokens: list[str] | None = None,

124 ) -> ParserEngine:

123 - tokenizer = make_mock_tokenizer(_VOCAB)

125 + tokenizer = make_mock_tokenizer(vocab or _VOCAB, special_tokens=special_tokens)

126 cfg = config or _combined_config()

127 return ParserEngine(

128 tokenizer,

@@ -843,6 +845,7 @@ class TestParseTokenIdPassthrough:

845

846 assert content is not None

847 assert "<tool_call>" in content

848 + assert content == "Use <tool_call> to call tools."

849 assert tool_calls is not None

850 assert len(tool_calls) == 1

851 assert tool_calls[0].name == "f"

@@ -1490,3 +1493,181 @@ class TestCoercionInstabilityRegression:

1493 assert parsed["val"] == "4e"

1494 assert isinstance(parsed["val"], str)

1495 assert parsed["extra"] == "ok"

1496 +

1497 +

1498 +_DROP_VOCAB: dict[str, int] = {

1499 + **_VOCAB,

1500 + "<bos>": 204,

1501 + "<eos>": 205,

1502 +}

1503 +

1504 +

1505 +class TestDropSpecialTokens:

1506 + """Special token dropping via the __DROP__ terminal mechanism."""

1507 +

1508 + def test_drops_special_token_by_id_from_content(self):

1509 + """A special token (not a configured terminal) is dropped when

1510 + it arrives as its actual token ID."""

1511 + config = ParserEngineConfig(

1512 + name="drop_content_test",

1513 + terminals={},

1514 + token_id_terminals={},

1515 + transitions={},

1516 + initial_state=ParserState.CONTENT,

1517 + content_events={ParserState.CONTENT: EventType.TEXT_CHUNK},

1518 + )

1519 + engine = _make_engine(

1520 + config=config,

1521 + vocab=_DROP_VOCAB,

1522 + special_tokens=list(_DROP_VOCAB.keys()),

1523 + )

1524 + engine._engine.reset()

1525 + events = engine._engine.feed("hello<bos>world", [72, 204, 73])

1526 + delta = engine._events_to_delta(events)

1527 + assert delta is not None

1528 + assert "<bos>" not in delta.content

1529 + assert delta.content == "helloworld"

1530 +

1531 + def test_drops_special_token_by_id_from_reasoning(self):

1532 + engine = _make_engine(

1533 + vocab=_DROP_VOCAB,

1534 + special_tokens=list(_DROP_VOCAB.keys()),

1535 + )

1536 + engine._engine.reset()

1537 + events = engine._engine.feed("thinking<eos>more", [72, 205, 73])

1538 + delta = engine._events_to_delta(events)

1539 + assert delta is not None

1540 + assert "<eos>" not in delta.reasoning

1541 + assert delta.reasoning == "thinkingmore"

1542 +

1543 + def test_drops_via_text_fallback_when_no_token_ids(self):

1544 + """When no token IDs are provided, text-based lexer catches

1545 + drop tokens as a fallback."""

1546 + engine = _make_engine(

1547 + vocab=_DROP_VOCAB,

1548 + special_tokens=list(_DROP_VOCAB.keys()),

1549 + )

1550 + engine._engine.reset()

1551 + events = engine._engine.feed("hello<bos>world", [])

1552 + delta = engine._events_to_delta(events)

1553 + assert delta is not None

1554 + assert "<bos>" not in delta.reasoning

1555 +

1556 + def test_regular_tokens_spelling_special_survive(self):

1557 + """Regular tokens that spell out a drop-token string survive

1558 + when token IDs prove they are not the special token."""

1559 + vocab = {**_DROP_VOCAB, "h": 72, "<": 73, "bos": 74, ">": 75, "w": 76}

1560 + engine = _make_engine(

1561 + vocab=vocab,

1562 + special_tokens=list(_DROP_VOCAB.keys()),

1563 + )

1564 + engine._engine.reset()

1565 + # Feed regular token IDs (73, 74, 75) that spell <bos>,

1566 + # not the special token ID 204.

1567 + events = engine._engine.feed("h<bos>w", [72, 73, 74, 75, 76])

1568 + delta = engine._events_to_delta(events)

1569 + assert delta is not None

1570 + assert "<bos>" in delta.reasoning

1571 +

1572 + def test_configured_terminal_not_treated_as_drop(self):

1573 + """Tokens already in config.terminals (like <think>) are handled

1574 + by the state machine, not the drop mechanism."""

1575 + engine = _make_engine(

1576 + vocab=_DROP_VOCAB,

1577 + special_tokens=list(_DROP_VOCAB.keys()),

1578 + )

1579 + # Start from CONTENT so <think> triggers REASONING_START

1580 + engine._engine.reset(initial_state=ParserState.CONTENT)

1581 + events = engine._engine.feed("<think>", [200])

1582 + has_reasoning_start = any(e.type == EventType.REASONING_START for e in events)

1583 + assert has_reasoning_start

1584 +

1585 + def test_tool_name_not_affected_by_drops(self):

1586 + engine = _make_engine(

1587 + vocab=_DROP_VOCAB,

1588 + special_tokens=list(_DROP_VOCAB.keys()),

1589 + )

1590 + engine._engine.reset()

1591 + events = [

1592 + SemanticEvent(EventType.TOOL_CALL_START, tool_index=0),

1593 + SemanticEvent(EventType.TOOL_NAME, "get_weather", tool_index=0),

1594 + SemanticEvent(EventType.ARG_VALUE_CHUNK, '{"city": "NYC"}', tool_index=0),

1595 + SemanticEvent(EventType.TOOL_CALL_END, tool_index=0),

1596 + ]

1597 + delta = engine._events_to_delta(events)

1598 + assert delta is not None

1599 + assert delta.tool_calls is not None

1600 + names = [

1601 + tc.function.name

1602 + for tc in delta.tool_calls

1603 + if tc.function and tc.function.name

1604 + ]

1605 + assert "get_weather" in names

1606 +

1607 + def test_adjacent_drop_tokens_by_id(self):

1608 + engine = _make_engine(

1609 + vocab=_DROP_VOCAB,

1610 + special_tokens=list(_DROP_VOCAB.keys()),

1611 + )

1612 + engine._engine.reset()

1613 + events = engine._engine.feed("<bos><eos>", [204, 205])

1614 + delta = engine._events_to_delta(events)

1615 + assert delta is None

1616 +

1617 + def test_no_special_tokens_means_no_drops(self):

1618 + """Engine with no special tokens passes all text through."""

1619 + engine = _make_engine(special_tokens=[])

1620 + engine._engine.reset()

1621 + events = engine._engine.feed("hello<bos>world", [])

1622 + delta = engine._events_to_delta(events)

1623 + assert delta is not None

1624 + assert "<bos>" in delta.reasoning

1625 +

1626 + def test_drops_suppressed_with_skip_tool_parsing(self):

1627 + """When skip_tool_parsing is active, drop tokens are preserved

1628 + as content so a later tool-call pass can see them."""

1629 + engine = _make_engine(

1630 + vocab=_DROP_VOCAB,

1631 + special_tokens=list(_DROP_VOCAB.keys()),

1632 + )

1633 + engine._engine.skip_tool_parsing = True

1634 + engine._engine.reset()

1635 + events = engine._engine.feed("hello<bos>world", [72, 204, 73])

1636 + delta = engine._events_to_delta(events)

1637 + assert delta is not None

1638 + assert "<bos>" in delta.reasoning

1639 +

1640 + def test_drops_in_tool_args_state(self):

1641 + """Drop tokens in TOOL_ARGS state are silently discarded."""

1642 + vocab = {**_DROP_VOCAB, '{"a":1}': 300}

1643 + engine = _make_engine(

1644 + vocab=vocab,

1645 + special_tokens=list(_DROP_VOCAB.keys()),

1646 + )

1647 + engine._engine.reset(initial_state=ParserState.CONTENT)

1648 + events = engine._engine.feed(

1649 + '<tool_call>{"a":1}<bos></tool_call>',

1650 + [202, 300, 204, 203],

1651 + )

1652 + arg_chunks = [e.value for e in events if e.type == EventType.ARG_VALUE_CHUNK]

1653 + combined = "".join(arg_chunks)

1654 + assert "<bos>" not in combined

1655 + assert '{"a":1}' in combined

1656 +

1657 + def test_mixed_configured_and_drop_terminals(self):

1658 + """Configured terminals trigger transitions while drop terminals

1659 + are silently removed in the same stream."""

1660 + engine = _make_engine(

1661 + vocab=_DROP_VOCAB,

1662 + special_tokens=list(_DROP_VOCAB.keys()),

1663 + )

1664 + engine._engine.reset()

1665 + events = engine._engine.feed("thought<bos></think>answer", [72, 204, 201, 73])

1666 + types = [e.type for e in events]

1667 + assert EventType.REASONING_CHUNK in types

1668 + assert EventType.REASONING_END in types

1669 + assert EventType.TEXT_CHUNK in types

1670 + reasoning_text = "".join(

1671 + e.value for e in events if e.type == EventType.REASONING_CHUNK

1672 + )

1673 + assert "<bos>" not in reasoning_text

tests/parser/engine/test_replay.py

+217 -29

@@ -18,18 +18,21 @@ from typing import NamedTuple

18 import pytest

20 from tests.parser.engine.replay_harness import (

21 + DUMMY_TOOLS,

22 MockTokenizer,

23 _test_request,

24 assert_no_terminal_leakage,

25 assert_parse_output,

26 collect_output,

27 make_mock_tokenizer,

28 + parse_non_streaming,

29 replay_streaming,

30 replay_with_text_holdback,

31 )

32 from tests.parser.engine.trace_builder import _BUILDERS, build_samples

33 from vllm.parser.engine import registered_adapters as _adapters_mod

34 from vllm.parser.engine.parser_engine import ParserEngine

35 +from vllm.parser.engine.parser_engine_config import ParserState

37 # ── Parser discovery ─────────────────────────────────────────────────

@@ -78,7 +81,11 @@ def _discover_parsers() -> list[_ParserInfo]:

81 terminals=sorted(v for v in all_vals if len(v) > 1),

82 tool_end=tool_end,

83 think_end=cfg.terminals.get("THINK_END", ""),

81 - tool_start=cfg.terminals.get("TOOL_START", ""),

84 + tool_start=(

85 + cfg.terminals["TOOL_SECTION_START"]

86 + if (ParserState.CONTENT, "TOOL_SECTION_START") in cfg.transitions

87 + else cfg.terminals.get("TOOL_START", "")

88 + ),

89 )

90 )

91 if missing_builders:

@@ -287,16 +294,18 @@ _TOOL_CALL_SAMPLES = [

294 ]

295

296

290 -def _suppressed_expectations(

291 - sample, think_end: str, tool_start: str

297 +def _tool_suppression_expectations(

298 + sample, think_end: str, tool_start: str, *, include_tool_block: bool

299 ) -> tuple[str, str]:

293 - """Compute expected (reasoning, content) when tools are suppressed.

300 + """Expected (reasoning, content) when tool calls are not extracted.

301

295 - When an explicit reasoning-end delimiter is present, reasoning ends

296 - there and the tool call block becomes content. When reasoning ends

297 - implicitly (the tool-start token triggers both REASONING_END and

298 - TOOL_CALL_START), reasoning still ends at the tool start and the raw

299 - tool call block becomes content text.

302 + With ``include_tool_block=True`` (skip_tool_parsing / reasoning

303 + adapter first pass), tool terminal text is preserved as content so

304 + a second-pass parser can see it.

305 +

306 + With ``include_tool_block=False`` (_suppress_tool_calls /

307 + tool_choice='none'), the state machine consumes tool blocks and

308 + only non-tool content survives.

309 """

310 full_text = "".join(text for _, text in sample.tokens)

311 reasoning = sample.expected_reasoning

@@ -307,36 +316,42 @@ def _suppressed_expectations(

316 if think_end:

317 pos = after_reasoning.find(think_end)

318 if pos >= 0:

310 - return (reasoning, after_reasoning[pos + len(think_end) :])

319 + if include_tool_block:

320 + return (reasoning, after_reasoning[pos + len(think_end) :])

321 + after_reasoning = after_reasoning[pos + len(think_end) :]

322 if tool_start:

323 pos = after_reasoning.find(tool_start)

324 if pos >= 0:

314 - return (reasoning, after_reasoning[pos:])

315 - return (full_text, "")

316 -

317 -

318 -_DUMMY_TOOLS = [

319 - {

320 - "type": "function",

321 - "function": {"name": "stub", "parameters": {"type": "object"}},

322 - }

323 -]

325 + if include_tool_block:

326 + return (reasoning, after_reasoning[pos:])

327 + return (reasoning, after_reasoning[:pos])

328 + if include_tool_block:

329 + return (full_text, "")

330 + return (reasoning, after_reasoning)

331

332

333 @pytest.mark.parametrize("chunk_size", [1, 5, None], ids=lambda c: f"chunk{c}")

334 +@pytest.mark.parametrize(

335 + "mode",

336 + ["skip_tool_parsing", "suppress_tool_calls"],

337 + ids=["skip_tool_parsing", "suppress_tool_calls"],

338 +)

339 @pytest.mark.parametrize(

340 "parser_cls,sample,think_end,tool_start",

341 _TOOL_CALL_SAMPLES,

342 ids=lambda v: v.id if hasattr(v, "id") else getattr(v, "__name__", ""),

343 )

332 -class TestSkipToolParsingReplay:

333 - """Replay with skip_tool_parsing=True (tool_choice='none').

344 +class TestToolCallFilteringReplay:

345 + """Replay with tool calls not extracted, in both filtering modes.

346 +

347 + ``skip_tool_parsing`` (reasoning adapter first pass): tool terminal

348 + text is preserved as content for a second-pass tool parser.

349

335 - Verifies that reasoning is extracted normally and the raw tool call

336 - block appears as content text with no tool calls parsed.

350 + ``suppress_tool_calls`` (tool_choice='none'): tool call blocks are

351 + consumed by the state machine and do not leak into content.

352 """

353

339 - def test_replay(self, parser_cls, sample, think_end, tool_start, chunk_size):

354 + def test_replay(self, parser_cls, sample, think_end, tool_start, mode, chunk_size):

355 tokenizer = make_mock_tokenizer(sample)

356 kwargs = {}

357 if sample.chat_template_kwargs:

@@ -344,8 +359,11 @@ class TestSkipToolParsingReplay:

359 parser = parser_cls(tokenizer, **kwargs)

360

361 request = _test_request()

347 - request.tool_choice = "none"

348 - request.tools = _DUMMY_TOOLS

362 + request.tools = DUMMY_TOOLS

363 + if mode == "skip_tool_parsing":

364 + parser.skip_tool_parsing = True

365 + else:

366 + request.tool_choice = "none"

367

368 all_ids = [tid for tid, _ in sample.tokens]

369 all_texts = [text for _, text in sample.tokens]

@@ -370,10 +388,51 @@ class TestSkipToolParsingReplay:

388

389 output = collect_output(results)

390

373 - expected_reasoning, expected_content = _suppressed_expectations(

374 - sample, think_end, tool_start

391 + include_block = mode == "skip_tool_parsing"

392 + expected_reasoning, expected_content = _tool_suppression_expectations(

393 + sample, think_end, tool_start, include_tool_block=include_block

394 )

395

396 + assert output.reasoning == expected_reasoning, (

397 + f"Reasoning mismatch (mode={mode}):\n"

398 + f" expected: {expected_reasoning!r}\n"

399 + f" actual: {output.reasoning!r}"

400 + )

401 + assert output.tool_calls == [], (

402 + f"Expected no tool calls (mode={mode}) but got {output.tool_calls}"

403 + )

404 + assert output.content == expected_content, (

405 + f"Content mismatch (mode={mode}):\n"

406 + f" expected: {expected_content!r}\n"

407 + f" actual: {output.content!r}"

408 + )

409 +

410 +

411 +@pytest.mark.parametrize(

412 + "parser_cls,sample,think_end,tool_start",

413 + _TOOL_CALL_SAMPLES,

414 + ids=lambda v: v.id if hasattr(v, "id") else getattr(v, "__name__", ""),

415 +)

416 +class TestToolCallFilteringNonStreaming:

417 + """Non-streaming parse() with tool_choice='none' must suppress tool

418 + calls and not leak special tokens into content."""

419 +

420 + def test_parse(self, parser_cls, sample, think_end, tool_start):

421 + tokenizer = make_mock_tokenizer(sample)

422 + kwargs = {}

423 + if sample.chat_template_kwargs:

424 + kwargs["chat_template_kwargs"] = sample.chat_template_kwargs

425 + parser = parser_cls(tokenizer, **kwargs)

426 +

427 + request = _test_request()

428 + request.tools = DUMMY_TOOLS

429 + request.tool_choice = "none"

430 +

431 + output = parse_non_streaming(parser, sample, request)

432 +

433 + expected_reasoning, expected_content = _tool_suppression_expectations(

434 + sample, think_end, tool_start, include_tool_block=False

435 + )

436 assert output.reasoning == expected_reasoning, (

437 f"Reasoning mismatch:\n"

438 f" expected: {expected_reasoning!r}\n"

@@ -389,6 +448,135 @@ class TestSkipToolParsingReplay:

448 )

449

450

451 +_WS_TOOL_SAMPLES = [(t[0], t[1]) for t in _TOOL_CALL_SAMPLES if "whitespace" in t[1].id]

452 +

453 +

454 +@pytest.mark.parametrize(

455 + "parser_cls,sample",

456 + _WS_TOOL_SAMPLES,

457 + ids=lambda v: v.id if hasattr(v, "id") else getattr(v, "__name__", ""),

458 +)

459 +class TestToolChoiceNoneStreamingParity:

460 + """Streaming and non-streaming must return the same content

461 + when tool_choice='none' suppresses tool calls."""

462 +

463 + def test_content_matches(self, parser_cls, sample):

464 + tokenizer = make_mock_tokenizer(sample)

465 + kwargs = {}

466 + if sample.chat_template_kwargs:

467 + kwargs["chat_template_kwargs"] = sample.chat_template_kwargs

468 + request = _test_request()

469 + request.tools = DUMMY_TOOLS

470 + request.tool_choice = "none"

471 +

472 + ns_output = parse_non_streaming(

473 + parser_cls(tokenizer, **kwargs),

474 + sample,

475 + request,

476 + )

477 +

478 + s_parser = parser_cls(tokenizer, **kwargs)

479 + results = []

480 + for i, (tid, text) in enumerate(sample.tokens):

481 + is_last = i == len(sample.tokens) - 1

482 + results.append(

483 + s_parser.parse_delta(

484 + text,

485 + [tid],

486 + request,

487 + prompt_token_ids=(sample.prompt_token_ids or [])

488 + if i == 0

489 + else None,

490 + finished=is_last,

491 + )

492 + )

493 + s_output = collect_output(results)

494 +

495 + assert ns_output.content == s_output.content, (

496 + f"Streaming/non-streaming content mismatch:\n"

497 + f" streaming: {s_output.content!r}\n"

498 + f" non-streaming: {ns_output.content!r}"

499 + )

500 +

501 +

502 +_DROP_TOKENS = {"<bos>": 99990, "<eos>": 99991}

503 +

504 +

505 +def _inject_drop_tokens(sample):

506 + """Insert <bos> at stream start and <eos> between the first two tokens."""

507 + new_vocab = {**sample.vocab, **_DROP_TOKENS}

508 + tokens = list(sample.tokens)

509 + tokens.insert(0, (99990, "<bos>"))

510 + if len(tokens) >= 3:

511 + tokens.insert(2, (99991, "<eos>"))

512 + else:

513 + tokens.append((99991, "<eos>"))

514 + return dataclasses.replace(sample, vocab=new_vocab, tokens=tokens)

515 +

516 +

517 +class TestDropTokenReplay:

518 + """Verify unconfigured special tokens are silently dropped across

519 + all parsers and chunk sizes."""

520 +

521 + @pytest.mark.parametrize(

522 + "parser_info",

523 + _PARSERS,

524 + ids=[p.name for p in _PARSERS],

525 + )

526 + @pytest.mark.parametrize("chunk_size", [1, 3, None])

527 + def test_drop_tokens_removed_from_output(self, parser_info, chunk_size):

528 + for sample in parser_info.samples:

529 + injected = _inject_drop_tokens(sample)

530 + tokenizer = make_mock_tokenizer(injected)

531 + parser = parser_info.parser_cls(

532 + tokenizer,

533 + tools=sample.tools,

534 + )

535 +

536 + results = replay_streaming(

537 + parser,

538 + injected.tokens,

539 + chunk_size=chunk_size,

540 + tools=sample.tools,

541 + prompt_token_ids=sample.prompt_token_ids,

542 + )

543 + output = collect_output(results)

544 +

545 + assert_no_terminal_leakage(

546 + output,

547 + list(_DROP_TOKENS.keys()),

548 + context=f"parser={parser_info.name}, chunk={chunk_size}",

549 + )

550 + assert_parse_output(output, sample)

551 +

552 +

553 +class TestDropTokenNonStreaming:

554 + """Non-streaming parse() must also strip unconfigured special tokens."""

555 +

556 + @pytest.mark.parametrize(

557 + "parser_info",

558 + _PARSERS,

559 + ids=[p.name for p in _PARSERS],

560 + )

561 + def test_drop_tokens_removed_from_output(self, parser_info):

562 + for sample in parser_info.samples:

563 + injected = _inject_drop_tokens(sample)

564 + tokenizer = make_mock_tokenizer(injected)

565 + parser = parser_info.parser_cls(

566 + tokenizer,

567 + tools=sample.tools,

568 + )

569 +

570 + request = _test_request(tools=sample.tools)

571 + output = parse_non_streaming(parser, injected, request)

572 +

573 + assert_no_terminal_leakage(

574 + output,

575 + list(_DROP_TOKENS.keys()),

576 + context=f"parser={parser_info.name}",

577 + )

578 +

579 +

580 class TestAdapterReferences:

581 """Verify make_adapters sets reasoning/tool parser class refs on parser engine

582 parser classes so the serving layer finds them and calls adjust_request."""

tests/parser/engine/test_token_id_scanner.py

+1 -38

@@ -57,7 +57,7 @@ class TestJoinDecodedTextReturnsStr:

58 @pytest.fixture

59 def bare_scanner(self):

60 - return TokenIDScanner({}, tokenizer=None, drop_token_ids=set())

60 + return TokenIDScanner({}, tokenizer=None)

62 def test_mixed_items(self, bare_scanner):

63 items = [

@@ -265,43 +265,6 @@ class TestHoldbackTextRecovery:

265 assert "reasoning end." in combined

266

267

268 -class TestDropTokens:

269 - def test_drop_token_with_holdback(self, tokenizer):

270 - """Drop tokens stripped; hold-back text preserved."""

271 - drop_id = 300

272 - tokenizer.decode.side_effect = lambda ids: {

273 - CHANNEL_END_ID: CHANNEL_END,

274 - drop_id: "<eos>",

275 - }.get(ids[0], "?")

276 -

277 - scanner = TokenIDScanner(

278 - token_id_to_terminal={CHANNEL_END_ID: "THINK_END"},

279 - tokenizer=tokenizer,

280 - drop_token_ids={drop_id},

281 - )

282 -

283 - result = scanner.scan(

284 - delta_text="holdback<eos>",

285 - delta_token_ids=[drop_id, CHANNEL_END_ID],

286 - )

287 -

288 - assert len(result) == 0

289 -

290 - result2 = scanner.scan(

291 - delta_text="<channel|>content",

292 - delta_token_ids=[20],

293 - )

294 - pre_lexed = [r for r in result2 if isinstance(r, PreLexedTerminal)]

295 - assert len(pre_lexed) == 1

296 - assert pre_lexed[0].terminal == "THINK_END"

297 - texts = [r.text for r in result2 if isinstance(r, TextChunk)]

298 - combined = "".join(texts)

299 - assert "holdback" in combined

300 - assert "<eos>" not in combined

301 -

302 - assert len(scanner.flush_pending()) == 0

303 -

304 -

268 class TestEndToEndReasoningHoldback:

269 """End-to-end engine tests with detokenizer hold-back."""

270

tests/tool_use/test_gemma4_responses_adjust_request.py

+22 -12

@@ -100,14 +100,24 @@ def _build_chat_request(

100 class _StubTokenizer:

101 """Minimal tokenizer stub to satisfy ``Gemma4EngineToolParser.__init__``."""

102

103 + _VOCAB: dict[str, int] = {

104 + "<|tool_call>": 256_000,

105 + "<tool_call|>": 256_001,

106 + '<|"|>': 52,

107 + "<|channel>": 256_002,

108 + "<channel|>": 256_003,

109 + }

110 +

111 def get_vocab(self) -> dict[str, int]:

104 - return {

105 - "<|tool_call>": 256_000,

106 - "<tool_call|>": 256_001,

107 - '<|"|>': 52,

108 - "<|channel>": 256_002,

109 - "<channel|>": 256_003,

110 - }

112 + return dict(self._VOCAB)

113 +

114 + @property

115 + def all_special_tokens(self) -> list[str]:

116 + return list(self._VOCAB.keys())

117 +

118 + @property

119 + def all_special_ids(self) -> list[int]:

120 + return list(self._VOCAB.values())

121

122

123 def test_gemma4_adjust_request_sets_skip_special_tokens_on_responses() -> None:

@@ -235,10 +245,10 @@ def test_gemma4_keeps_special_tokens_with_tools_thinking_disabled() -> None:

245 assert request.skip_special_tokens is False

246

247

238 -def test_gemma4_strips_special_tokens_when_nothing_to_preserve() -> None:

239 - """No active tools + thinking disabled: keep the default

240 - (``skip_special_tokens=True``) so stray delimiters do not leak into

241 - content.

248 +def test_gemma4_keeps_skip_special_tokens_false_when_nothing_to_preserve() -> None:

249 + """No active tools + thinking disabled: ``skip_special_tokens`` stays

250 + ``False`` because the parser engine's ``__DROP__`` terminal mechanism

251 + strips unconfigured special tokens automatically.

252 """

253 parser = Gemma4ToolParser(_StubTokenizer())

254 request = _build_chat_request(

@@ -247,4 +257,4 @@ def test_gemma4_strips_special_tokens_when_nothing_to_preserve() -> None:

257

258 parser.adjust_request(request)

259

250 - assert request.skip_special_tokens is True

260 + assert request.skip_special_tokens is False

vllm/parser/abstract_parser.py

@@ -424,6 +424,12 @@ class DelegatingParser(Parser):

424 if tool_parser is None:

425 return [], content

426

427 + if request.tool_choice == "none":

428 + if self._engine_based:

429 + result = self.extract_tool_calls(content or "", request=request)

430 + return [], result.content

431 + return [], content

432 +

433 supports_required_and_named = tool_parser.supports_required_and_named

434 is_named_tool_choice = request.tool_choice and isinstance(

435 request.tool_choice,

vllm/parser/engine/parser_engine.py

+17 -11

@@ -116,6 +116,7 @@ class ParserEngine(Parser):

116 self._deferred_content: str = ""

117 self._deferred_reasoning: str = ""

118 self._content_has_nonws: bool = False

119 + self._suppress_tool_calls: bool = False

120

121 self._arg_converter = parser_engine_config.arg_converter

122 self._arg_structural_chars = parser_engine_config.arg_structural_chars

@@ -398,10 +399,10 @@ class ParserEngine(Parser):

399 tools = getattr(request, "tools", None)

400 if tools:

401 self._tools = tools

401 - if not self.skip_tool_parsing:

402 + if not self.skip_tool_parsing and not self._suppress_tool_calls:

403 tool_choice = getattr(request, "tool_choice", None)

404 if tool_choice == "none" and tools:

404 - self.skip_tool_parsing = True

405 + self._suppress_tool_calls = True

406

407 def _strip_content_whitespace(

408 self,

@@ -642,7 +643,7 @@ class ParserEngine(Parser):

643 events = self._feed(text, token_ids)

644 events.extend(self._engine.finish())

645

645 - delta = self._events_to_delta(events)

646 + delta = self._events_to_delta(events, finished=True)

647 tool_call_info = self._build_extracted_result()

648

649 reasoning = delta.reasoning if delta else None

@@ -701,6 +702,7 @@ class ParserEngine(Parser):

702 reasoning_parts: list[str] = []

703

704 seen_tool_event = False

705 + suppress = self._suppress_tool_calls

706 for event in events:

707 match event.type:

708 case EventType.TEXT_CHUNK:

@@ -713,17 +715,21 @@ class ParserEngine(Parser):

715 case EventType.REASONING_END:

716 self._reasoning_ended = True

717 case EventType.TOOL_CALL_START:

716 - seen_tool_event = True

717 - self._ensure_slot(event.tool_index)

718 + if not suppress:

719 + seen_tool_event = True

720 + self._ensure_slot(event.tool_index)

721 case EventType.TOOL_NAME:

719 - seen_tool_event = True

720 - self._handle_tool_name(event)

722 + if not suppress:

723 + seen_tool_event = True

724 + self._handle_tool_name(event)

725 case EventType.ARG_VALUE_CHUNK:

722 - seen_tool_event = True

723 - self._handle_arg_chunk(event, tool_call_deltas)

726 + if not suppress:

727 + seen_tool_event = True

728 + self._handle_arg_chunk(event, tool_call_deltas)

729 case EventType.TOOL_CALL_END:

725 - seen_tool_event = True

726 - self._handle_tool_end(event, tool_call_deltas)

730 + if not suppress:

731 + seen_tool_event = True

732 + self._handle_tool_end(event, tool_call_deltas)

733 case EventType.REASONING_START:

734 pass # no delta-level effect

735

vllm/parser/engine/parser_engine_config.py

+3 -12

@@ -24,16 +24,6 @@ from functools import cached_property

25 from vllm.parser.engine.events import EventType

27 -STRUCTURAL_DROP_TOKENS: frozenset[str] = frozenset(

28 - {

29 - "<eos>",

30 - "<bos>",

31 - "<pad>",

32 - "<unk>",

33 - "<mask>",

34 - }

35 -)

36 -

28 class ParserState(Enum):

29 CONTENT = auto()

@@ -90,6 +80,9 @@ class ParserEngineConfig:

81 arg_structural_chars: frozenset[str] | None = None

83 + # Special tokens exempt from auto-drop but not state-machine terminals.

84 + preserve_tokens: frozenset[str] = field(default_factory=frozenset)

85 +

86 # Prevents trailing-whitespace accumulation across multi-turn conversations.

87 strip_trailing_reasoning_whitespace: bool = True

@@ -102,8 +95,6 @@ class ParserEngineConfig:

95 # Reject tool calls whose names are absent from the request tools.

96 validate_tool_names: bool = False

105 - drop_tokens: frozenset[str] = field(default_factory=frozenset)

106 -

98 @cached_property

99 def terminal_defs(self):

100 from vllm.parser.engine.incremental_lexer import terminals_from_literals

vllm/parser/engine/streaming_parser_engine.py

+82 -17

@@ -6,15 +6,17 @@ incremental lexing, and state-machine-driven semantic event emission."""

6 from __future__ import annotations

8 from collections.abc import Sequence

9 +from dataclasses import dataclass

11 from vllm.parser.engine.events import EventType, SemanticEvent

12 from vllm.parser.engine.incremental_lexer import (

13 CONTENT_TERMINAL,

14 IncrementalLexer,

15 + LexerShape,

16 LexToken,

17 + TerminalDef,

18 )

19 from vllm.parser.engine.parser_engine_config import (

17 - STRUCTURAL_DROP_TOKENS,

20 ParserEngineConfig,

21 ParserState,

22 Transition,

@@ -26,6 +28,64 @@ from vllm.parser.engine.token_id_scanner import (

28 TokenIDScanner,

29 )

31 +DROP_TERMINAL = "__DROP__"

32 +

33 +

34 +@dataclass(slots=True)

35 +class _DropInfo:

36 + lexer_shape: LexerShape

37 + extra_token_ids: dict[int, str]

38 +

39 +

40 +def _build_drop_info(

41 + config: ParserEngineConfig,

42 + tokenizer,

43 +) -> _DropInfo | None:

44 + try:

45 + special_tokens: list[str] = list(tokenizer.all_special_tokens)

46 + special_ids: list[int] = list(tokenizer.all_special_ids)

47 + except (AttributeError, NotImplementedError):

48 + return None

49 +

50 + if not special_tokens:

51 + return None

52 +

53 + configured_texts = (

54 + set(config.token_id_terminals.values())

55 + | set(config.terminals.values())

56 + | config.preserve_tokens

57 + )

58 +

59 + extra_token_ids: dict[int, str] = {}

60 + drop_texts: set[str] = set()

61 + for text, tid in zip(special_tokens, special_ids):

62 + if text not in configured_texts:

63 + extra_token_ids[tid] = DROP_TERMINAL

64 + drop_texts.add(text)

65 +

66 + if not drop_texts:

67 + return None

68 +

69 + import regex as re

70 +

71 + drop_terminal_defs = [

72 + TerminalDef(

73 + name=DROP_TERMINAL,

74 + pattern=re.compile(re.escape(text)),

75 + is_literal=True,

76 + literal=text,

77 + )

78 + for text in drop_texts

79 + ]

80 +

81 + all_terminal_defs = list(config.terminal_defs) + drop_terminal_defs

82 + lexer_shape = LexerShape(all_terminal_defs)

83 +

84 + return _DropInfo(

85 + lexer_shape=lexer_shape,

86 + extra_token_ids=extra_token_ids,

87 + )

88 +

90 class StreamingParserEngine:

91 """Consumes ``(delta_text, delta_token_ids)`` pairs and produces a

@@ -60,7 +120,6 @@ class StreamingParserEngine:

120 self.config = config

121

122 resolved_token_ids: dict[int, str] = {}

63 - drop_token_ids: set[int] = set()

123 if tokenizer is not None:

124 if vocab is None:

125 vocab = tokenizer.get_vocab()

@@ -69,32 +128,29 @@ class StreamingParserEngine:

128 tid = vocab.get(token_text)

129 if tid is not None:

130 resolved_token_ids[tid] = terminal_name

72 - all_drop = config.drop_tokens | STRUCTURAL_DROP_TOKENS

73 - for token_text in all_drop:

74 - tid = vocab.get(token_text)

75 - if tid is not None:

76 - drop_token_ids.add(tid)

77 - for attr in ("eos_token_id", "bos_token_id", "pad_token_id"):

78 - tid = getattr(tokenizer, attr, None)

79 - if tid is not None:

80 - drop_token_ids.add(tid)

131 +

132 + drop_info: _DropInfo | None = None

133 + if tokenizer is not None:

134 + drop_info = _build_drop_info(config, tokenizer)

135 +

136 + lexer_shape = config.lexer_shape

137 + if drop_info is not None:

138 + resolved_token_ids.update(drop_info.extra_token_ids)

139 + lexer_shape = drop_info.lexer_shape

140

141 self._resolved_token_ids = resolved_token_ids

83 - self._drop_token_ids = drop_token_ids

142 + self._has_drops = drop_info is not None

143

144 self._scanner = TokenIDScanner(

145 resolved_token_ids,

146 tokenizer,

88 - drop_token_ids,

147 )

148

149 self._token_id_terminal_names: frozenset[str] = frozenset(

150 resolved_token_ids.values()

151 )

152

95 - self._lexer = IncrementalLexer(

96 - config.lexer_shape, content_terminal=CONTENT_TERMINAL

97 - )

153 + self._lexer = IncrementalLexer(lexer_shape, content_terminal=CONTENT_TERMINAL)

154

155 self._tool_terminals: frozenset[str] = frozenset(

156 terminal

@@ -150,7 +206,7 @@ class StreamingParserEngine:

206 ):

207 has_special = False

208 for tid in delta_token_ids:

153 - if tid in self._resolved_token_ids or tid in self._drop_token_ids:

209 + if tid in self._resolved_token_ids:

210 has_special = True

211 break

212 if not has_special:

@@ -247,6 +303,15 @@ class StreamingParserEngine:

303 transition = self.config.transitions.get(key)

304

305 if transition is None:

306 + if (

307 + self._has_drops

308 + and terminal == DROP_TERMINAL

309 + # Preserve drop tokens when skip_tool_parsing is active so

310 + # the reasoning pass doesn't silently remove tokens that a

311 + # later tool-call pass might need to see.

312 + and not self.skip_tool_parsing

313 + ):

314 + return []

315 return self._emit_for_state(value)

316

317 if self.skip_tool_parsing and terminal in self._tool_terminals:

vllm/parser/engine/token_id_scanner.py

+4 -27

@@ -42,12 +42,10 @@ class TokenIDScanner:

42 self,

43 token_id_to_terminal: dict[int, str],

44 tokenizer,

45 - drop_token_ids: set[int] | None = None,

45 ) -> None:

46 self.token_id_to_terminal = token_id_to_terminal

47 self.tokenizer = tokenizer

48 self._token_text_cache: dict[int, str] = {}

50 - self._drop_token_ids = drop_token_ids or set()

49 self._deferred_terminals: list[PreLexedTerminal] = []

50 self._deferred_post_text: str = ""

@@ -74,22 +72,19 @@ class TokenIDScanner:

72 if self._deferred_terminals:

73 prefix_items, effective_text = self._resolve_deferred(delta_text)

77 - if not self.token_id_to_terminal and not self._drop_token_ids:

75 + if not self.token_id_to_terminal:

76 if effective_text:

77 prefix_items.append(TextChunk(effective_text))

78 return prefix_items

80 has_special = False

83 - has_drop = False

81 token_id_to_terminal = self.token_id_to_terminal

85 - drop_token_ids = self._drop_token_ids

82 for tid in delta_token_ids:

83 if tid in token_id_to_terminal:

84 has_special = True

89 - if tid in drop_token_ids:

90 - has_drop = True

85 + break

92 - if not has_special and not has_drop:

87 + if not has_special:

88 if effective_text:

89 if not prefix_items:

90 return [TextChunk(effective_text)]

@@ -102,8 +97,6 @@ class TokenIDScanner:

97 text_accum: list[str] = []

99 for idx, tid in enumerate(delta_token_ids):

105 - if tid in self._drop_token_ids:

106 - continue

100 terminal = self.token_id_to_terminal.get(tid)

101 if terminal is not None:

102 if text_accum:

@@ -121,23 +114,7 @@ class TokenIDScanner:

114 results.append(TextChunk(joined))

115

116 if effective_text:

124 - if has_drop:

125 - clean_delta = effective_text

126 - for idx, tid in enumerate(delta_token_ids):

127 - if tid in self._drop_token_ids:

128 - dropped = token_texts[idx]

129 - pos = clean_delta.find(dropped)

130 - if pos >= 0:

131 - clean_delta = (

132 - clean_delta[:pos] + clean_delta[pos + len(dropped) :]

133 - )

134 - if clean_delta:

135 - if results:

136 - results = self._recover_holdback_text(clean_delta, results)

137 - else:

138 - results = [TextChunk(clean_delta)]

139 - else:

140 - results = self._recover_holdback_text(effective_text, results)

117 + results = self._recover_holdback_text(effective_text, results)

118 else:

119 # No detokenizer text to validate against — individually-decoded

120 # TextChunks are unreliable (context-dependent decoding).

vllm/parser/gemma4.py

+1 -58

@@ -35,34 +35,6 @@ if TYPE_CHECKING:

35 from vllm.tokenizers import TokenizerLike

36 from vllm.tool_parsers.abstract_tool_parser import Tool

38 -# Tokens the model generates that must not leak into response content.

39 -_GEMMA4_MODEL_DROP_TOKENS: set[str] = {

40 - # Turn boundaries

41 - "<|turn>",

42 - "<turn|>",

43 - # Channel / reasoning

44 - "<|channel>",

45 - "<channel|>",

46 - # Tool protocol tokens

47 - "<|tool>",

48 - "<tool|>",

49 - "<|tool_call>",

50 - "<tool_call|>",

51 - "<|tool_response>",

52 - "<tool_response|>",

53 - '<|"|>',

54 - # Thinking

55 - "<|think|>",

56 - # Multi-modal (defensive — not expected during text completion)

57 - "<|image>",

58 - "<|image|>",

59 - "<image|>",

60 - "<|audio>",

61 - "<|audio|>",

62 - "<audio|>",

63 - "<|video|>",

64 -}

65 -

38 CHANNEL_START = "<|channel>"

39 CHANNEL_END = "<channel|>"

40 TOOL_CALL_START = "<|tool_call>"

@@ -322,14 +294,6 @@ def _gemma4_arg_converter(raw_args: str, partial: bool) -> str:

294

295 @functools.cache

296 def gemma4_config() -> ParserEngineConfig:

325 - used_tokens = {

326 - CHANNEL_START,

327 - CHANNEL_END,

328 - TOOL_CALL_START,

329 - TOOL_CALL_END,

330 - '<|"|>',

331 - }

332 -

297 return ParserEngineConfig(

298 name="gemma4",

299 initial_state=ParserState.CONTENT,

@@ -412,7 +376,7 @@ def gemma4_config() -> ParserEngineConfig:

376 arg_converter=_gemma4_arg_converter,

377 tool_args_json=False,

378 arg_structural_chars=frozenset(",:{}[]<"),

415 - drop_tokens=frozenset(_GEMMA4_MODEL_DROP_TOKENS - used_tokens),

379 + preserve_tokens=frozenset({STRING_DELIM}),

380 )

381

382

@@ -451,27 +415,6 @@ class Gemma4Parser(ParserEngine):

415 self._prefix_stripped: bool = False

416 self._is_first_feed: bool = True

417

454 - def adjust_request(

455 - self,

456 - request: ChatCompletionRequest | ResponsesRequest,

457 - ) -> ChatCompletionRequest | ResponsesRequest:

458 - """Keep special tokens when thinking or tool calls need them.

459 -

460 - ``skip_special_tokens`` must stay ``False`` when there is something to

461 - preserve: reasoning channel tokens (thinking enabled) or tool-call

462 - delimiters (tools active). Otherwise keep the default so stray

463 - delimiters do not leak into content (e.g. ``tool_choice="none"`` with

464 - thinking disabled).

465 - """

466 - request = super().adjust_request(request)

467 - chat_template_kwargs = getattr(request, "chat_template_kwargs", None) or {}

468 - enable_thinking = chat_template_kwargs.get("enable_thinking", True)

469 - has_tools = bool(getattr(request, "tools", None))

470 - tools_active = has_tools and request.tool_choice != "none"

471 - if not enable_thinking and not tools_active:

472 - request.skip_special_tokens = True

473 - return request

474 -

418 def _reset(self, initial_state=None) -> None:

419 super()._reset(initial_state=initial_state)

420 self._reasoning_text = ""